@aztec/prover-client 0.0.1-commit.c7c42ec → 0.0.1-commit.c80b6263

package/dest/orchestrator/orchestrator.js

@@ -1,9 +1,376 @@
-function _ts_decorate(decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
+function applyDecs2203RFactory() {
+    function createAddInitializerMethod(initializers, decoratorFinishedRef) {
+        return function addInitializer(initializer) {
+            assertNotFinished(decoratorFinishedRef, "addInitializer");
+            assertCallable(initializer, "An initializer");
+            initializers.push(initializer);
+        };
+    }
+    function memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value) {
+        var kindStr;
+        switch(kind){
+            case 1:
+                kindStr = "accessor";
+                break;
+            case 2:
+                kindStr = "method";
+                break;
+            case 3:
+                kindStr = "getter";
+                break;
+            case 4:
+                kindStr = "setter";
+                break;
+            default:
+                kindStr = "field";
+        }
+        var ctx = {
+            kind: kindStr,
+            name: isPrivate ? "#" + name : name,
+            static: isStatic,
+            private: isPrivate,
+            metadata: metadata
+        };
+        var decoratorFinishedRef = {
+            v: false
+        };
+        ctx.addInitializer = createAddInitializerMethod(initializers, decoratorFinishedRef);
+        var get, set;
+        if (kind === 0) {
+            if (isPrivate) {
+                get = desc.get;
+                set = desc.set;
+            } else {
+                get = function() {
+                    return this[name];
+                };
+                set = function(v) {
+                    this[name] = v;
+                };
+            }
+        } else if (kind === 2) {
+            get = function() {
+                return desc.value;
+            };
+        } else {
+            if (kind === 1 || kind === 3) {
+                get = function() {
+                    return desc.get.call(this);
+                };
+            }
+            if (kind === 1 || kind === 4) {
+                set = function(v) {
+                    desc.set.call(this, v);
+                };
+            }
+        }
+        ctx.access = get && set ? {
+            get: get,
+            set: set
+        } : get ? {
+            get: get
+        } : {
+            set: set
+        };
+        try {
+            return dec(value, ctx);
+        } finally{
+            decoratorFinishedRef.v = true;
+        }
+    }
+    function assertNotFinished(decoratorFinishedRef, fnName) {
+        if (decoratorFinishedRef.v) {
+            throw new Error("attempted to call " + fnName + " after decoration was finished");
+        }
+    }
+    function assertCallable(fn, hint) {
+        if (typeof fn !== "function") {
+            throw new TypeError(hint + " must be a function");
+        }
+    }
+    function assertValidReturnValue(kind, value) {
+        var type = typeof value;
+        if (kind === 1) {
+            if (type !== "object" || value === null) {
+                throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0");
+            }
+            if (value.get !== undefined) {
+                assertCallable(value.get, "accessor.get");
+            }
+            if (value.set !== undefined) {
+                assertCallable(value.set, "accessor.set");
+            }
+            if (value.init !== undefined) {
+                assertCallable(value.init, "accessor.init");
+            }
+        } else if (type !== "function") {
+            var hint;
+            if (kind === 0) {
+                hint = "field";
+            } else if (kind === 10) {
+                hint = "class";
+            } else {
+                hint = "method";
+            }
+            throw new TypeError(hint + " decorators must return a function or void 0");
+        }
+    }
+    function applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata) {
+        var decs = decInfo[0];
+        var desc, init, value;
+        if (isPrivate) {
+            if (kind === 0 || kind === 1) {
+                desc = {
+                    get: decInfo[3],
+                    set: decInfo[4]
+                };
+            } else if (kind === 3) {
+                desc = {
+                    get: decInfo[3]
+                };
+            } else if (kind === 4) {
+                desc = {
+                    set: decInfo[3]
+                };
+            } else {
+                desc = {
+                    value: decInfo[3]
+                };
+            }
+        } else if (kind !== 0) {
+            desc = Object.getOwnPropertyDescriptor(base, name);
+        }
+        if (kind === 1) {
+            value = {
+                get: desc.get,
+                set: desc.set
+            };
+        } else if (kind === 2) {
+            value = desc.value;
+        } else if (kind === 3) {
+            value = desc.get;
+        } else if (kind === 4) {
+            value = desc.set;
+        }
+        var newValue, get, set;
+        if (typeof decs === "function") {
+            newValue = memberDec(decs, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
+            if (newValue !== void 0) {
+                assertValidReturnValue(kind, newValue);
+                if (kind === 0) {
+                    init = newValue;
+                } else if (kind === 1) {
+                    init = newValue.init;
+                    get = newValue.get || value.get;
+                    set = newValue.set || value.set;
+                    value = {
+                        get: get,
+                        set: set
+                    };
+                } else {
+                    value = newValue;
+                }
+            }
+        } else {
+            for(var i = decs.length - 1; i >= 0; i--){
+                var dec = decs[i];
+                newValue = memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
+                if (newValue !== void 0) {
+                    assertValidReturnValue(kind, newValue);
+                    var newInit;
+                    if (kind === 0) {
+                        newInit = newValue;
+                    } else if (kind === 1) {
+                        newInit = newValue.init;
+                        get = newValue.get || value.get;
+                        set = newValue.set || value.set;
+                        value = {
+                            get: get,
+                            set: set
+                        };
+                    } else {
+                        value = newValue;
+                    }
+                    if (newInit !== void 0) {
+                        if (init === void 0) {
+                            init = newInit;
+                        } else if (typeof init === "function") {
+                            init = [
+                                init,
+                                newInit
+                            ];
+                        } else {
+                            init.push(newInit);
+                        }
+                    }
+                }
+            }
+        }
+        if (kind === 0 || kind === 1) {
+            if (init === void 0) {
+                init = function(instance, init) {
+                    return init;
+                };
+            } else if (typeof init !== "function") {
+                var ownInitializers = init;
+                init = function(instance, init) {
+                    var value = init;
+                    for(var i = 0; i < ownInitializers.length; i++){
+                        value = ownInitializers[i].call(instance, value);
+                    }
+                    return value;
+                };
+            } else {
+                var originalInitializer = init;
+                init = function(instance, init) {
+                    return originalInitializer.call(instance, init);
+                };
+            }
+            ret.push(init);
+        }
+        if (kind !== 0) {
+            if (kind === 1) {
+                desc.get = value.get;
+                desc.set = value.set;
+            } else if (kind === 2) {
+                desc.value = value;
+            } else if (kind === 3) {
+                desc.get = value;
+            } else if (kind === 4) {
+                desc.set = value;
+            }
+            if (isPrivate) {
+                if (kind === 1) {
+                    ret.push(function(instance, args) {
+                        return value.get.call(instance, args);
+                    });
+                    ret.push(function(instance, args) {
+                        return value.set.call(instance, args);
+                    });
+                } else if (kind === 2) {
+                    ret.push(value);
+                } else {
+                    ret.push(function(instance, args) {
+                        return value.call(instance, args);
+                    });
+                }
+            } else {
+                Object.defineProperty(base, name, desc);
+            }
+        }
+    }
+    function applyMemberDecs(Class, decInfos, metadata) {
+        var ret = [];
+        var protoInitializers;
+        var staticInitializers;
+        var existingProtoNonFields = new Map();
+        var existingStaticNonFields = new Map();
+        for(var i = 0; i < decInfos.length; i++){
+            var decInfo = decInfos[i];
+            if (!Array.isArray(decInfo)) continue;
+            var kind = decInfo[1];
+            var name = decInfo[2];
+            var isPrivate = decInfo.length > 3;
+            var isStatic = kind >= 5;
+            var base;
+            var initializers;
+            if (isStatic) {
+                base = Class;
+                kind = kind - 5;
+                staticInitializers = staticInitializers || [];
+                initializers = staticInitializers;
+            } else {
+                base = Class.prototype;
+                protoInitializers = protoInitializers || [];
+                initializers = protoInitializers;
+            }
+            if (kind !== 0 && !isPrivate) {
+                var existingNonFields = isStatic ? existingStaticNonFields : existingProtoNonFields;
+                var existingKind = existingNonFields.get(name) || 0;
+                if (existingKind === true || existingKind === 3 && kind !== 4 || existingKind === 4 && kind !== 3) {
+                    throw new Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + name);
+                } else if (!existingKind && kind > 2) {
+                    existingNonFields.set(name, kind);
+                } else {
+                    existingNonFields.set(name, true);
+                }
+            }
+            applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata);
+        }
+        pushInitializers(ret, protoInitializers);
+        pushInitializers(ret, staticInitializers);
+        return ret;
+    }
+    function pushInitializers(ret, initializers) {
+        if (initializers) {
+            ret.push(function(instance) {
+                for(var i = 0; i < initializers.length; i++){
+                    initializers[i].call(instance);
+                }
+                return instance;
+            });
+        }
+    }
+    function applyClassDecs(targetClass, classDecs, metadata) {
+        if (classDecs.length > 0) {
+            var initializers = [];
+            var newClass = targetClass;
+            var name = targetClass.name;
+            for(var i = classDecs.length - 1; i >= 0; i--){
+                var decoratorFinishedRef = {
+                    v: false
+                };
+                try {
+                    var nextNewClass = classDecs[i](newClass, {
+                        kind: "class",
+                        name: name,
+                        addInitializer: createAddInitializerMethod(initializers, decoratorFinishedRef),
+                        metadata
+                    });
+                } finally{
+                    decoratorFinishedRef.v = true;
+                }
+                if (nextNewClass !== undefined) {
+                    assertValidReturnValue(10, nextNewClass);
+                    newClass = nextNewClass;
+                }
+            }
+            return [
+                defineMetadata(newClass, metadata),
+                function() {
+                    for(var i = 0; i < initializers.length; i++){
+                        initializers[i].call(newClass);
+                    }
+                }
+            ];
+        }
+    }
+    function defineMetadata(Class, metadata) {
+        return Object.defineProperty(Class, Symbol.metadata || Symbol.for("Symbol.metadata"), {
+            configurable: true,
+            enumerable: true,
+            value: metadata
+        });
+    }
+    return function applyDecs2203R(targetClass, memberDecs, classDecs, parentClass) {
+        if (parentClass !== void 0) {
+            var parentMetadata = parentClass[Symbol.metadata || Symbol.for("Symbol.metadata")];
+        }
+        var metadata = Object.create(parentMetadata === void 0 ? null : parentMetadata);
+        var e = applyMemberDecs(targetClass, memberDecs, metadata);
+        if (!classDecs.length) defineMetadata(targetClass, metadata);
+        return {
+            e: e,
+            get c () {
+                return applyClassDecs(targetClass, classDecs, metadata);
+            }
+        };
+    };
+}
+function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
+    return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
 }
+var _dec, _dec1, _dec2, _dec3, _dec4, _initProto;
 import { L1_TO_L2_MSG_SUBTREE_HEIGHT, L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH, NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, NUM_BASE_PARITY_PER_ROOT_PARITY } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { padArrayEnd } from '@aztec/foundation/collection';
@@ -22,7 +389,15 @@ import { buildHeaderFromCircuitOutputs, getLastSiblingPath, getPublicChonkVerifi
 import { EpochProvingState } from './epoch-proving-state.js';
 import { ProvingOrchestratorMetrics } from './orchestrator_metrics.js';
 import { TxProvingState } from './tx-proving-state.js';
-const logger = createLogger('prover-client:orchestrator');
+_dec = trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
+        [Attributes.BLOCK_NUMBER]: blockNumber
+    })), _dec1 = trackSpan('ProvingOrchestrator.addTxs', (txs)=>({
+        [Attributes.BLOCK_TXS_COUNT]: txs.length
+    })), _dec2 = trackSpan('ProvingOrchestrator.startChonkVerifierCircuits'), _dec3 = trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber)=>({
+        [Attributes.BLOCK_NUMBER]: blockNumber
+    })), _dec4 = trackSpan('ProvingOrchestrator.prepareBaseRollupInputs', (tx)=>({
+        [Attributes.TX_HASH]: tx.hash.toString()
+    }));
 /**
  * Implements an event driven proving scheduler to build the recursive proof tree. The idea being:
  * 1. Transactions are provided to the scheduler post simulation.
@@ -38,20 +413,53 @@ const logger = createLogger('prover-client:orchestrator');
     dbProvider;
     prover;
     proverId;
+    cancelJobsOnStop;
+    static{
+        ({ e: [_initProto] } = _apply_decs_2203_r(this, [
+            [
+                _dec,
+                2,
+                "startNewBlock"
+            ],
+            [
+                _dec1,
+                2,
+                "addTxs"
+            ],
+            [
+                _dec2,
+                2,
+                "startChonkVerifierCircuits"
+            ],
+            [
+                _dec3,
+                2,
+                "setBlockCompleted"
+            ],
+            [
+                _dec4,
+                2,
+                "prepareBaseRollupInputs"
+            ]
+        ], []));
+    }
     provingState;
     pendingProvingJobs;
     provingPromise;
     metrics;
     // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections
     dbs;
-    constructor(dbProvider, prover, proverId, telemetryClient = getTelemetryClient()){
+    logger;
+    constructor(dbProvider, prover, proverId, cancelJobsOnStop = false, telemetryClient = getTelemetryClient(), bindings){
         this.dbProvider = dbProvider;
         this.prover = prover;
         this.proverId = proverId;
-        this.provingState = undefined;
+        this.cancelJobsOnStop = cancelJobsOnStop;
+        this.provingState = (_initProto(this), undefined);
         this.pendingProvingJobs = [];
         this.provingPromise = undefined;
         this.dbs = new Map();
+        this.logger = createLogger('prover-client:orchestrator', bindings);
         this.metrics = new ProvingOrchestratorMetrics(telemetryClient, 'ProvingOrchestrator');
     }
     get tracer() {
@@ -60,6 +468,9 @@ const logger = createLogger('prover-client:orchestrator');
     getProverId() {
         return this.proverId;
     }
+    getNumActiveForks() {
+        return this.dbs.size;
+    }
     stop() {
         this.cancel();
         return Promise.resolve();
@@ -73,11 +484,18 @@ const logger = createLogger('prover-client:orchestrator');
                 status: 'failure',
                 reason
             }));
-        logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
+        this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`);
         this.provingState = new EpochProvingState(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges, (provingState)=>this.checkAndEnqueueCheckpointRootRollup(provingState), resolve, reject);
         this.provingPromise = promise;
     }
-    async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
+    /**
+   * Starts a new checkpoint.
+   * @param checkpointIndex - The index of the checkpoint in the epoch.
+   * @param constants - The constants for this checkpoint.
+   * @param l1ToL2Messages - The set of L1 to L2 messages to be inserted at the beginning of this checkpoint.
+   * @param totalNumBlocks - The total number of blocks expected in the checkpoint (must be at least one).
+   * @param headerOfLastBlockInPreviousCheckpoint - The header of the last block in the previous checkpoint.
+   */ async startNewCheckpoint(checkpointIndex, constants, l1ToL2Messages, totalNumBlocks, headerOfLastBlockInPreviousCheckpoint) {
         if (!this.provingState) {
             throw new Error('Empty epoch proving state. Call startNewEpoch before starting a checkpoint.');
         }
@@ -88,7 +506,10 @@ const logger = createLogger('prover-client:orchestrator');
         const lastBlockNumber = headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber;
         const db = await this.dbProvider.fork(lastBlockNumber);
         const firstBlockNumber = BlockNumber(lastBlockNumber + 1);
-        this.dbs.set(firstBlockNumber, db);
+        this.dbs.set(firstBlockNumber, {
+            fork: db,
+            cleanupPromise: undefined
+        });
         // Get archive sibling path before any block in this checkpoint lands.
         const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
         // Insert all the l1 to l2 messages into the db. And get the states before and after the insertion.
@@ -113,14 +534,17 @@ const logger = createLogger('prover-client:orchestrator');
             throw new Error(`Checkpoint not accepting further blocks.`);
         }
         const constants = checkpointProvingState.constants;
-        logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
+        this.logger.info(`Starting block ${blockNumber} for slot ${constants.slotNumber}.`);
         // Fork the db only when it's not already set. The db for the first block is set in `startNewCheckpoint`.
         if (!this.dbs.has(blockNumber)) {
             // Fork world state at the end of the immediately previous block
             const db = await this.dbProvider.fork(BlockNumber(blockNumber - 1));
-            this.dbs.set(blockNumber, db);
+            this.dbs.set(blockNumber, {
+                fork: db,
+                cleanupPromise: undefined
+            });
         }
-        const db = this.dbs.get(blockNumber);
+        const db = this.dbs.get(blockNumber).fork;
         // Get archive snapshot and sibling path before any txs in this block lands.
         const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const lastArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
@@ -140,7 +564,8 @@ const logger = createLogger('prover-client:orchestrator');
             const blockEndBlobFields = blockProvingState.getBlockEndBlobFields();
             await endSpongeBlob.absorb(blockEndBlobFields);
             blockProvingState.setEndSpongeBlob(endSpongeBlob);
-            // And also try to accumulate the blobs as far as we can:
+            // Try to accumulate the out hashes and blobs as far as we can:
+            await this.provingState.accumulateCheckpointOutHashes();
             await this.provingState.setBlobAccumulators();
         }
     }
@@ -154,7 +579,7 @@ const logger = createLogger('prover-client:orchestrator');
         if (!txs.length) {
             // To avoid an ugly throw below. If we require an empty block, we can just call setBlockCompleted
             // on a block with no txs. We cannot do that here because we cannot find the blockNumber without any txs.
-            logger.warn(`Provided no txs to orchestrator addTxs.`);
+            this.logger.warn(`Provided no txs to orchestrator addTxs.`);
             return;
         }
         const blockNumber = BlockNumber(txs[0].globalVariables.blockNumber);
@@ -168,8 +593,8 @@ const logger = createLogger('prover-client:orchestrator');
         if (!provingState.isAcceptingTxs()) {
             throw new Error(`Block ${blockNumber} has been initialized with transactions.`);
         }
-        logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
-        const db = this.dbs.get(blockNumber);
+        this.logger.info(`Adding ${txs.length} transactions to block ${blockNumber}`);
+        const db = this.dbs.get(blockNumber).fork;
         const lastArchive = provingState.lastArchiveTreeSnapshot;
         const newL1ToL2MessageTreeSnapshot = provingState.newL1ToL2MessageTreeSnapshot;
         const spongeBlobState = provingState.getStartSpongeBlob().clone();
@@ -179,7 +604,7 @@ const logger = createLogger('prover-client:orchestrator');
                     throw new Error(`Invalid proving state when adding a tx`);
                 }
                 validateTx(tx);
-                logger.debug(`Received transaction: ${tx.hash}`);
+                this.logger.debug(`Received transaction: ${tx.hash}`);
                 const startSpongeBlob = spongeBlobState.clone();
                 const [hints, treeSnapshots] = await this.prepareBaseRollupInputs(tx, lastArchive, newL1ToL2MessageTreeSnapshot, startSpongeBlob, db);
                 if (!provingState.verifyState()) {
@@ -190,10 +615,10 @@ const logger = createLogger('prover-client:orchestrator');
                 const txIndex = provingState.addNewTx(txProvingState);
                 if (txProvingState.requireAvmProof) {
                     this.getOrEnqueueChonkVerifier(provingState, txIndex);
-                    logger.debug(`Enqueueing public VM for tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing public VM for tx ${txIndex}`);
                     this.enqueueVM(provingState, txIndex);
                 } else {
-                    logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
+                    this.logger.debug(`Enqueueing base rollup for private-only tx ${txIndex}`);
                     this.enqueueBaseRollup(provingState, txIndex);
                 }
             } catch (err) {
@@ -207,7 +632,8 @@ const logger = createLogger('prover-client:orchestrator');
         const blockEndBlobFields = provingState.getBlockEndBlobFields();
         await spongeBlobState.absorb(blockEndBlobFields);
         provingState.setEndSpongeBlob(spongeBlobState);
-        // Txs have been added to the block. Now try to accumulate the blobs as far as we can:
+        // Txs have been added to the block. Now try to accumulate the out hashes and blobs as far as we can:
+        await this.provingState.accumulateCheckpointOutHashes();
         await this.provingState.setBlobAccumulators();
     }
     /**
@@ -222,7 +648,7 @@ const logger = createLogger('prover-client:orchestrator');
             const txHash = tx.getTxHash().toString();
             const privateInputs = getPublicChonkVerifierPrivateInputsFromTx(tx, this.proverId.toField());
             const tubeProof = promiseWithResolvers();
-            logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
+            this.logger.debug(`Starting chonk verifier circuit for tx ${txHash}`);
             this.doEnqueueChonkVerifier(txHash, privateInputs, (proof)=>{
                 tubeProof.resolve(proof);
             });
@@ -251,16 +677,16 @@ const logger = createLogger('prover-client:orchestrator');
             throw new Error(`Block ${blockNumber} is still accepting txs. Call setBlockCompleted after all txs have been added.`);
         }
         // Given we've applied every change from this block, now assemble the block header:
-        logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
+        this.logger.verbose(`Block ${blockNumber} completed. Assembling header.`);
         const header = await provingState.buildBlockHeader();
         if (expectedHeader && !header.equals(expectedHeader)) {
-            logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
+            this.logger.error(`Block header mismatch: header=${header} expectedHeader=${expectedHeader}`);
             throw new Error('Block header mismatch');
         }
         // Get db for this block
-        const db = this.dbs.get(provingState.blockNumber);
+        const db = this.dbs.get(provingState.blockNumber).fork;
         // Update the archive tree, so we're ready to start processing the next block:
-        logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
+        this.logger.verbose(`Updating archive tree with block ${provingState.blockNumber} header ${(await header.hash()).toString()}`);
         await db.updateArchive(header);
         await this.verifyBuiltBlockAgainstSyncedState(provingState);
         return header;
@@ -269,33 +695,33 @@ const logger = createLogger('prover-client:orchestrator');
     async verifyBuiltBlockAgainstSyncedState(provingState) {
         const builtBlockHeader = provingState.getBuiltBlockHeader();
         if (!builtBlockHeader) {
-            logger.debug('Block header not built yet, skipping header check.');
+            this.logger.debug('Block header not built yet, skipping header check.');
             return;
         }
         const output = provingState.getBlockRootRollupOutput();
         if (!output) {
-            logger.debug('Block root rollup proof not built yet, skipping header check.');
+            this.logger.debug('Block root rollup proof not built yet, skipping header check.');
             return;
         }
         const header = await buildHeaderFromCircuitOutputs(output);
         if (!(await header.hash()).equals(await builtBlockHeader.hash())) {
-            logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
+            this.logger.error(`Block header mismatch.\nCircuit: ${inspect(header)}\nComputed: ${inspect(builtBlockHeader)}`);
             provingState.reject(`Block header hash mismatch.`);
             return;
         }
         // Get db for this block
         const blockNumber = provingState.blockNumber;
-        const db = this.dbs.get(blockNumber);
+        const db = this.dbs.get(blockNumber).fork;
         const newArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
         const syncedArchive = await getTreeSnapshot(MerkleTreeId.ARCHIVE, this.dbProvider.getSnapshot(blockNumber));
         if (!syncedArchive.equals(newArchive)) {
-            logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
+            this.logger.error(`Archive tree mismatch for block ${blockNumber}: world state synced to ${inspect(syncedArchive)} but built ${inspect(newArchive)}`);
             provingState.reject(`Archive tree mismatch.`);
             return;
         }
         const circuitArchive = output.newArchive;
         if (!newArchive.equals(circuitArchive)) {
-            logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
+            this.logger.error(`New archive mismatch.\nCircuit: ${output.newArchive}\nComputed: ${newArchive}`);
             provingState.reject(`New archive mismatch.`);
             return;
         }
@@ -303,14 +729,17 @@ const logger = createLogger('prover-client:orchestrator');
         // is aborted and never reaches this point, it will leak the fork. We need to add a global cleanup,
         // but have to make sure it only runs once all operations are completed, otherwise some function here
         // will attempt to access the fork after it was closed.
-        logger.debug(`Cleaning up world state fork for ${blockNumber}`);
-        void this.dbs.get(blockNumber)?.close().then(()=>this.dbs.delete(blockNumber)).catch((err)=>logger.error(`Error closing db for block ${blockNumber}`, err));
+        void this.cleanupDBFork(blockNumber);
     }
     /**
-   * Cancel any further proving
+   * Cancel any further proving.
+   * If cancelJobsOnStop is true, aborts all pending jobs with the broker (which marks them as 'Aborted').
+   * If cancelJobsOnStop is false (default), jobs remain in the broker queue and can be reused on restart/reorg.
    */ cancel() {
-        for (const controller of this.pendingProvingJobs){
-            controller.abort();
+        if (this.cancelJobsOnStop) {
+            for (const controller of this.pendingProvingJobs){
+                controller.abort();
+            }
         }
         this.provingState?.cancel();
     }
@@ -332,6 +761,22 @@ const logger = createLogger('prover-client:orchestrator');
         });
         return epochProofResult;
     }
+    async cleanupDBFork(blockNumber) {
+        this.logger.debug(`Cleaning up world state fork for ${blockNumber}`);
+        const fork = this.dbs.get(blockNumber);
+        if (!fork) {
+            return;
+        }
+        try {
+            if (!fork.cleanupPromise) {
+                fork.cleanupPromise = fork.fork.close();
+            }
+            await fork.cleanupPromise;
+            this.dbs.delete(blockNumber);
+        } catch (err) {
+            this.logger.error(`Error closing db for block ${blockNumber}`, err);
+        }
+    }
     /**
    * Enqueue a job to be scheduled
    * @param provingState - The proving state object being operated on
@@ -339,7 +784,7 @@ const logger = createLogger('prover-client:orchestrator');
    * @param job - The actual job, returns a promise notifying of the job's completion
    */ deferredProving(provingState, request, callback) {
         if (!provingState.verifyState()) {
-            logger.debug(`Not enqueuing job, state no longer valid`);
+            this.logger.debug(`Not enqueuing job, state no longer valid`);
             return;
         }
         const controller = new AbortController();
@@ -354,7 +799,7 @@ const logger = createLogger('prover-client:orchestrator');
                 }
                 const result = await request(controller.signal);
                 if (!provingState.verifyState()) {
-                    logger.debug(`State no longer valid, discarding result`);
+                    this.logger.debug(`State no longer valid, discarding result`);
                     return;
                 }
                 // we could have been cancelled whilst waiting for the result
@@ -369,7 +814,7 @@ const logger = createLogger('prover-client:orchestrator');
                     // drop this result
                     return;
                 }
-                logger.error(`Error thrown when proving job`, err);
+                this.logger.error(`Error thrown when proving job`, err);
                 provingState.reject(`${err}`);
             } finally{
                 const index = this.pendingProvingJobs.indexOf(controller);
@@ -425,17 +870,17 @@ const logger = createLogger('prover-client:orchestrator');
     // Executes the next level of merge if all inputs are available
     enqueueBaseRollup(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base rollup, state invalid');
+            this.logger.debug('Not running base rollup, state invalid');
             return;
         }
         if (!provingState.tryStartProvingBase(txIndex)) {
-            logger.debug(`Base rollup for tx ${txIndex} already started.`);
+            this.logger.debug(`Base rollup for tx ${txIndex} already started.`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const { processedTx } = txProvingState;
         const { rollupType, inputs } = txProvingState.getBaseRollupTypeAndInputs();
-        logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
+        this.logger.debug(`Enqueuing deferred proving base rollup for ${processedTx.hash.toString()}`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, `ProvingOrchestrator.prover.${inputs instanceof PrivateTxBaseRollupPrivateInputs ? 'getPrivateTxBaseRollupProof' : 'getPublicTxBaseRollupProof'}`, {
             [Attributes.TX_HASH]: processedTx.hash.toString(),
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
@@ -446,7 +891,7 @@ const logger = createLogger('prover-client:orchestrator');
                 return this.prover.getPublicTxBaseRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), (result)=>{
-            logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
+            this.logger.debug(`Completed proof for ${rollupType} for tx ${processedTx.hash.toString()}`);
             validatePartialState(result.inputs.endTreeSnapshots, txProvingState.treeSnapshots);
             const leafLocation = provingState.setBaseRollupProof(txIndex, result);
             if (provingState.totalNumTxs === 1) {
@@ -460,14 +905,14 @@ const logger = createLogger('prover-client:orchestrator');
     // Once completed, will enqueue the the public tx base rollup.
     getOrEnqueueChonkVerifier(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
         const txHash = txProvingState.processedTx.hash.toString();
         NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH;
         const handleResult = (result)=>{
-            logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
+            this.logger.debug(`Got chonk verifier proof for tx index: ${txIndex}`, {
                 txHash
             });
             txProvingState.setPublicChonkVerifierProof(result);
@@ -475,18 +920,18 @@ const logger = createLogger('prover-client:orchestrator');
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         };
         if (this.provingState?.cachedChonkVerifierProofs.has(txHash)) {
-            logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
+            this.logger.debug(`Chonk verifier proof already enqueued for tx index: ${txIndex}`, {
                 txHash
             });
             void this.provingState.cachedChonkVerifierProofs.get(txHash).then(handleResult);
             return;
         }
-        logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
+        this.logger.debug(`Enqueuing chonk verifier circuit for tx index: ${txIndex}`);
         this.doEnqueueChonkVerifier(txHash, txProvingState.getPublicChonkVerifierPrivateInputs(), handleResult);
     }
     doEnqueueChonkVerifier(txHash, inputs, handler, provingState = this.provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running chonk verifier circuit, state invalid');
+            this.logger.debug('Not running chonk verifier circuit, state invalid');
             return;
         }
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getPublicChonkVerifierProof', {
@@ -498,11 +943,11 @@ const logger = createLogger('prover-client:orchestrator');
     // Enqueues the next level of merge if all inputs are available
     enqueueMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running merge rollup. State no longer valid.');
+            this.logger.debug('Not running merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingMerge(location)) {
-            logger.debug('Merge rollup already started.');
+            this.logger.debug('Merge rollup already started.');
             return;
         }
         const inputs = provingState.getMergeRollupInputs(location);
@@ -516,15 +961,15 @@ const logger = createLogger('prover-client:orchestrator');
     // Executes the block root rollup circuit
     enqueueBlockRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block root rollup, state no longer valid');
+            this.logger.debug('Not running block root rollup, state no longer valid');
             return;
         }
         if (!provingState.tryStartProvingBlockRoot()) {
-            logger.debug('Block root rollup already started.');
+            this.logger.debug('Block root rollup already started.');
             return;
         }
         const { rollupType, inputs } = provingState.getBlockRootRollupTypeAndInputs();
-        logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
+        this.logger.debug(`Enqueuing ${rollupType} for block ${provingState.blockNumber}.`);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getBlockRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
         }, (signal)=>{
@@ -540,27 +985,29 @@ const logger = createLogger('prover-client:orchestrator');
                 return this.prover.getBlockRootRollupProof(inputs, signal, provingState.epochNumber);
             }
         }), async (result)=>{
-            // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
-            await this.verifyBuiltBlockAgainstSyncedState(provingState);
-            logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
+            this.logger.debug(`Completed ${rollupType} proof for block ${provingState.blockNumber}`);
             const leafLocation = provingState.setBlockRootRollupProof(result);
             const checkpointProvingState = provingState.parentCheckpoint;
+            // If the proofs were slower than the block header building, then we need to try validating the block header hashes here.
+            await this.verifyBuiltBlockAgainstSyncedState(provingState);
             if (checkpointProvingState.totalNumBlocks === 1) {
                 this.checkAndEnqueueCheckpointRootRollup(checkpointProvingState);
             } else {
                 this.checkAndEnqueueNextBlockMergeRollup(checkpointProvingState, leafLocation);
             }
+            // We are finished with the block at this point, ensure the fork is cleaned up
+            void this.cleanupDBFork(provingState.blockNumber);
         });
     }
     // Executes the base parity circuit and stores the intermediate state for the root parity circuit
     // Enqueues the root parity circuit if all inputs are available
     enqueueBaseParityCircuit(checkpointProvingState, provingState, baseParityIndex) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running base parity. State no longer valid.');
+            this.logger.debug('Not running base parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBaseParity(baseParityIndex)) {
-            logger.warn(`Base parity ${baseParityIndex} already started.`);
+            this.logger.warn(`Base parity ${baseParityIndex} already started.`);
             return;
         }
         const inputs = checkpointProvingState.getBaseParityInputs(baseParityIndex);
@@ -581,11 +1028,11 @@ const logger = createLogger('prover-client:orchestrator');
     // Enqueues the root rollup proof if all inputs are available
     enqueueRootParityCircuit(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running root parity. State no longer valid.');
+            this.logger.debug('Not running root parity. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingRootParity()) {
-            logger.debug('Root parity already started.');
+            this.logger.debug('Root parity already started.');
             return;
         }
         const inputs = provingState.getParityRootInputs();
@@ -600,11 +1047,11 @@ const logger = createLogger('prover-client:orchestrator');
     // Enqueues the next level of merge if all inputs are available
     enqueueBlockMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running block merge rollup. State no longer valid.');
+            this.logger.debug('Not running block merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingBlockMerge(location)) {
-            logger.debug('Block merge rollup already started.');
+            this.logger.debug('Block merge rollup already started.');
             return;
         }
         const inputs = provingState.getBlockMergeRollupInputs(location);
@@ -617,15 +1064,15 @@ const logger = createLogger('prover-client:orchestrator');
     }
     enqueueCheckpointRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint root rollup. State no longer valid.');
+            this.logger.debug('Not running checkpoint root rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingCheckpointRoot()) {
-            logger.debug('Checkpoint root rollup already started.');
+            this.logger.debug('Checkpoint root rollup already started.');
             return;
         }
         const rollupType = provingState.getCheckpointRootRollupType();
-        logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
+        this.logger.debug(`Enqueuing ${rollupType} for checkpoint ${provingState.index}.`);
         const inputs = provingState.getCheckpointRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: rollupType
@@ -639,11 +1086,11 @@ const logger = createLogger('prover-client:orchestrator');
             const computedEndBlobAccumulatorState = provingState.getEndBlobAccumulator().toBlobAccumulator();
             const circuitEndBlobAccumulatorState = result.inputs.endBlobAccumulator;
             if (!circuitEndBlobAccumulatorState.equals(computedEndBlobAccumulatorState)) {
-                logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
+                this.logger.error(`Blob accumulator state mismatch.\nCircuit: ${inspect(circuitEndBlobAccumulatorState)}\nComputed: ${inspect(computedEndBlobAccumulatorState)}`);
                 provingState.reject(`Blob accumulator state mismatch.`);
                 return;
             }
-            logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
+            this.logger.debug(`Completed ${rollupType} proof for checkpoint ${provingState.index}.`);
             const leafLocation = provingState.setCheckpointRootRollupProof(result);
             const epochProvingState = provingState.parentEpoch;
             if (epochProvingState.totalNumCheckpoints === 1) {
@@ -655,37 +1102,37 @@ const logger = createLogger('prover-client:orchestrator');
     }
     enqueueCheckpointMergeRollup(provingState, location) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running checkpoint merge rollup. State no longer valid.');
+            this.logger.debug('Not running checkpoint merge rollup. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingCheckpointMerge(location)) {
-            logger.debug('Checkpoint merge rollup already started.');
+            this.logger.debug('Checkpoint merge rollup already started.');
             return;
         }
         const inputs = provingState.getCheckpointMergeRollupInputs(location);
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointMergeRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-merge'
         }, (signal)=>this.prover.getCheckpointMergeRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for checkpoint merge rollup.');
+            this.logger.debug('Completed proof for checkpoint merge rollup.');
             provingState.setCheckpointMergeRollupProof(location, result);
             this.checkAndEnqueueNextCheckpointMergeRollup(provingState, location);
         });
     }
     enqueueEpochPadding(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running epoch padding. State no longer valid.');
+            this.logger.debug('Not running epoch padding. State no longer valid.');
             return;
         }
         if (!provingState.tryStartProvingPaddingCheckpoint()) {
-            logger.debug('Padding checkpoint already started.');
+            this.logger.debug('Padding checkpoint already started.');
             return;
         }
-        logger.debug('Padding epoch proof with a padding block root proof.');
+        this.logger.debug('Padding epoch proof with a padding block root proof.');
         const inputs = provingState.getPaddingCheckpointInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getCheckpointPaddingRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-checkpoint-padding'
         }, (signal)=>this.prover.getCheckpointPaddingRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.debug('Completed proof for padding checkpoint.');
+            this.logger.debug('Completed proof for padding checkpoint.');
             provingState.setCheckpointPaddingProof(result);
             this.checkAndEnqueueRootRollup(provingState);
         });
@@ -693,15 +1140,15 @@ const logger = createLogger('prover-client:orchestrator');
     // Executes the root rollup circuit
     enqueueRootRollup(provingState) {
         if (!provingState.verifyState()) {
-            logger.debug('Not running root rollup, state no longer valid');
+            this.logger.debug('Not running root rollup, state no longer valid');
             return;
         }
-        logger.debug(`Preparing root rollup`);
+        this.logger.debug(`Preparing root rollup`);
         const inputs = provingState.getRootRollupInputs();
         this.deferredProving(provingState, wrapCallbackInSpan(this.tracer, 'ProvingOrchestrator.prover.getRootRollupProof', {
             [Attributes.PROTOCOL_CIRCUIT_NAME]: 'rollup-root'
         }, (signal)=>this.prover.getRootRollupProof(inputs, signal, provingState.epochNumber)), (result)=>{
-            logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
+            this.logger.verbose(`Orchestrator completed root rollup for epoch ${provingState.epochNumber}`);
             provingState.setRootRollupProof(result);
             provingState.resolve({
                 status: 'success'
@@ -721,7 +1168,7 @@ const logger = createLogger('prover-client:orchestrator');
     }
     checkAndEnqueueBlockRootRollup(provingState) {
         if (!provingState.isReadyForBlockRootRollup()) {
-            logger.debug('Not ready for block root rollup');
+            this.logger.debug('Not ready for block root rollup');
             return;
         }
         this.enqueueBlockRootRollup(provingState);
@@ -756,7 +1203,7 @@ const logger = createLogger('prover-client:orchestrator');
     }
     checkAndEnqueueRootRollup(provingState) {
         if (!provingState.isReadyForRootRollup()) {
-            logger.debug('Not ready for root rollup');
+            this.logger.debug('Not ready for root rollup');
             return;
         }
         this.enqueueRootRollup(provingState);
@@ -768,7 +1215,7 @@ const logger = createLogger('prover-client:orchestrator');
    * @param txIndex - The index of the transaction being proven
    */ enqueueVM(provingState, txIndex) {
         if (!provingState.verifyState()) {
-            logger.debug(`Not running VM circuit as state is no longer valid`);
+            this.logger.debug(`Not running VM circuit as state is no longer valid`);
             return;
         }
         const txProvingState = provingState.getTxProvingState(txIndex);
@@ -778,9 +1225,9 @@ const logger = createLogger('prover-client:orchestrator');
             const inputs = txProvingState.getAvmInputs();
             return await this.prover.getAvmProof(inputs, signal, provingState.epochNumber);
         });
-        this.deferredProving(provingState, doAvmProving, (proofAndVk)=>{
-            logger.debug(`Proven VM for tx index: ${txIndex}`);
-            txProvingState.setAvmProof(proofAndVk);
+        this.deferredProving(provingState, doAvmProving, (proof)=>{
+            this.logger.debug(`Proven VM for tx index: ${txIndex}`);
+            txProvingState.setAvmProof(proof);
             this.checkAndEnqueueBaseRollup(provingState, txIndex);
         });
     }
@@ -790,30 +1237,7 @@ const logger = createLogger('prover-client:orchestrator');
             return;
         }
         // We must have completed all proving (chonk verifier proof and (if required) vm proof are generated), we now move to the base rollup.
-        logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
+        this.logger.debug(`Public functions completed for tx ${txIndex} enqueueing base rollup`);
         this.enqueueBaseRollup(provingState, txIndex);
     }
 }
-_ts_decorate([
-    trackSpan('ProvingOrchestrator.startNewBlock', (blockNumber)=>({
-            [Attributes.BLOCK_NUMBER]: blockNumber
-        }))
-], ProvingOrchestrator.prototype, "startNewBlock", null);
-_ts_decorate([
-    trackSpan('ProvingOrchestrator.addTxs', (txs)=>({
-            [Attributes.BLOCK_TXS_COUNT]: txs.length
-        }))
-], ProvingOrchestrator.prototype, "addTxs", null);
-_ts_decorate([
-    trackSpan('ProvingOrchestrator.startChonkVerifierCircuits')
-], ProvingOrchestrator.prototype, "startChonkVerifierCircuits", null);
-_ts_decorate([
-    trackSpan('ProvingOrchestrator.setBlockCompleted', (blockNumber)=>({
-            [Attributes.BLOCK_NUMBER]: blockNumber
-        }))
-], ProvingOrchestrator.prototype, "setBlockCompleted", null);
-_ts_decorate([
-    trackSpan('ProvingOrchestrator.prepareBaseRollupInputs', (tx)=>({
-            [Attributes.TX_HASH]: tx.hash.toString()
-        }))
-], ProvingOrchestrator.prototype, "prepareBaseRollupInputs", null);