@aztec/prover-node 0.0.0-test.1 → 0.0.1-commit.017a351

package/dest/job/epoch-proving-job.js

@@ -1,60 +1,512 @@
-function _ts_decorate(decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
+function _ts_add_disposable_resource(env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() {
+            try {
+                inner.call(this);
+            } catch (e) {
+                return Promise.reject(e);
+            }
+        };
+        env.stack.push({
+            value: value,
+            dispose: dispose,
+            async: async
+        });
+    } else if (async) {
+        env.stack.push({
+            async: true
+        });
+    }
+    return value;
 }
+function _ts_dispose_resources(env) {
+    var _SuppressedError = typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed, message) {
+        var e = new Error(message);
+        return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+    };
+    return (_ts_dispose_resources = function _ts_dispose_resources(env) {
+        function fail(e) {
+            env.error = env.hasError ? new _SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while(r = env.stack.pop()){
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) {
+                            fail(e);
+                            return next();
+                        });
+                    } else s |= 1;
+                } catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    })(env);
+}
+function applyDecs2203RFactory() {
+    function createAddInitializerMethod(initializers, decoratorFinishedRef) {
+        return function addInitializer(initializer) {
+            assertNotFinished(decoratorFinishedRef, "addInitializer");
+            assertCallable(initializer, "An initializer");
+            initializers.push(initializer);
+        };
+    }
+    function memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value) {
+        var kindStr;
+        switch(kind){
+            case 1:
+                kindStr = "accessor";
+                break;
+            case 2:
+                kindStr = "method";
+                break;
+            case 3:
+                kindStr = "getter";
+                break;
+            case 4:
+                kindStr = "setter";
+                break;
+            default:
+                kindStr = "field";
+        }
+        var ctx = {
+            kind: kindStr,
+            name: isPrivate ? "#" + name : name,
+            static: isStatic,
+            private: isPrivate,
+            metadata: metadata
+        };
+        var decoratorFinishedRef = {
+            v: false
+        };
+        ctx.addInitializer = createAddInitializerMethod(initializers, decoratorFinishedRef);
+        var get, set;
+        if (kind === 0) {
+            if (isPrivate) {
+                get = desc.get;
+                set = desc.set;
+            } else {
+                get = function() {
+                    return this[name];
+                };
+                set = function(v) {
+                    this[name] = v;
+                };
+            }
+        } else if (kind === 2) {
+            get = function() {
+                return desc.value;
+            };
+        } else {
+            if (kind === 1 || kind === 3) {
+                get = function() {
+                    return desc.get.call(this);
+                };
+            }
+            if (kind === 1 || kind === 4) {
+                set = function(v) {
+                    desc.set.call(this, v);
+                };
+            }
+        }
+        ctx.access = get && set ? {
+            get: get,
+            set: set
+        } : get ? {
+            get: get
+        } : {
+            set: set
+        };
+        try {
+            return dec(value, ctx);
+        } finally{
+            decoratorFinishedRef.v = true;
+        }
+    }
+    function assertNotFinished(decoratorFinishedRef, fnName) {
+        if (decoratorFinishedRef.v) {
+            throw new Error("attempted to call " + fnName + " after decoration was finished");
+        }
+    }
+    function assertCallable(fn, hint) {
+        if (typeof fn !== "function") {
+            throw new TypeError(hint + " must be a function");
+        }
+    }
+    function assertValidReturnValue(kind, value) {
+        var type = typeof value;
+        if (kind === 1) {
+            if (type !== "object" || value === null) {
+                throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0");
+            }
+            if (value.get !== undefined) {
+                assertCallable(value.get, "accessor.get");
+            }
+            if (value.set !== undefined) {
+                assertCallable(value.set, "accessor.set");
+            }
+            if (value.init !== undefined) {
+                assertCallable(value.init, "accessor.init");
+            }
+        } else if (type !== "function") {
+            var hint;
+            if (kind === 0) {
+                hint = "field";
+            } else if (kind === 10) {
+                hint = "class";
+            } else {
+                hint = "method";
+            }
+            throw new TypeError(hint + " decorators must return a function or void 0");
+        }
+    }
+    function applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata) {
+        var decs = decInfo[0];
+        var desc, init, value;
+        if (isPrivate) {
+            if (kind === 0 || kind === 1) {
+                desc = {
+                    get: decInfo[3],
+                    set: decInfo[4]
+                };
+            } else if (kind === 3) {
+                desc = {
+                    get: decInfo[3]
+                };
+            } else if (kind === 4) {
+                desc = {
+                    set: decInfo[3]
+                };
+            } else {
+                desc = {
+                    value: decInfo[3]
+                };
+            }
+        } else if (kind !== 0) {
+            desc = Object.getOwnPropertyDescriptor(base, name);
+        }
+        if (kind === 1) {
+            value = {
+                get: desc.get,
+                set: desc.set
+            };
+        } else if (kind === 2) {
+            value = desc.value;
+        } else if (kind === 3) {
+            value = desc.get;
+        } else if (kind === 4) {
+            value = desc.set;
+        }
+        var newValue, get, set;
+        if (typeof decs === "function") {
+            newValue = memberDec(decs, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
+            if (newValue !== void 0) {
+                assertValidReturnValue(kind, newValue);
+                if (kind === 0) {
+                    init = newValue;
+                } else if (kind === 1) {
+                    init = newValue.init;
+                    get = newValue.get || value.get;
+                    set = newValue.set || value.set;
+                    value = {
+                        get: get,
+                        set: set
+                    };
+                } else {
+                    value = newValue;
+                }
+            }
+        } else {
+            for(var i = decs.length - 1; i >= 0; i--){
+                var dec = decs[i];
+                newValue = memberDec(dec, name, desc, initializers, kind, isStatic, isPrivate, metadata, value);
+                if (newValue !== void 0) {
+                    assertValidReturnValue(kind, newValue);
+                    var newInit;
+                    if (kind === 0) {
+                        newInit = newValue;
+                    } else if (kind === 1) {
+                        newInit = newValue.init;
+                        get = newValue.get || value.get;
+                        set = newValue.set || value.set;
+                        value = {
+                            get: get,
+                            set: set
+                        };
+                    } else {
+                        value = newValue;
+                    }
+                    if (newInit !== void 0) {
+                        if (init === void 0) {
+                            init = newInit;
+                        } else if (typeof init === "function") {
+                            init = [
+                                init,
+                                newInit
+                            ];
+                        } else {
+                            init.push(newInit);
+                        }
+                    }
+                }
+            }
+        }
+        if (kind === 0 || kind === 1) {
+            if (init === void 0) {
+                init = function(instance, init) {
+                    return init;
+                };
+            } else if (typeof init !== "function") {
+                var ownInitializers = init;
+                init = function(instance, init) {
+                    var value = init;
+                    for(var i = 0; i < ownInitializers.length; i++){
+                        value = ownInitializers[i].call(instance, value);
+                    }
+                    return value;
+                };
+            } else {
+                var originalInitializer = init;
+                init = function(instance, init) {
+                    return originalInitializer.call(instance, init);
+                };
+            }
+            ret.push(init);
+        }
+        if (kind !== 0) {
+            if (kind === 1) {
+                desc.get = value.get;
+                desc.set = value.set;
+            } else if (kind === 2) {
+                desc.value = value;
+            } else if (kind === 3) {
+                desc.get = value;
+            } else if (kind === 4) {
+                desc.set = value;
+            }
+            if (isPrivate) {
+                if (kind === 1) {
+                    ret.push(function(instance, args) {
+                        return value.get.call(instance, args);
+                    });
+                    ret.push(function(instance, args) {
+                        return value.set.call(instance, args);
+                    });
+                } else if (kind === 2) {
+                    ret.push(value);
+                } else {
+                    ret.push(function(instance, args) {
+                        return value.call(instance, args);
+                    });
+                }
+            } else {
+                Object.defineProperty(base, name, desc);
+            }
+        }
+    }
+    function applyMemberDecs(Class, decInfos, metadata) {
+        var ret = [];
+        var protoInitializers;
+        var staticInitializers;
+        var existingProtoNonFields = new Map();
+        var existingStaticNonFields = new Map();
+        for(var i = 0; i < decInfos.length; i++){
+            var decInfo = decInfos[i];
+            if (!Array.isArray(decInfo)) continue;
+            var kind = decInfo[1];
+            var name = decInfo[2];
+            var isPrivate = decInfo.length > 3;
+            var isStatic = kind >= 5;
+            var base;
+            var initializers;
+            if (isStatic) {
+                base = Class;
+                kind = kind - 5;
+                staticInitializers = staticInitializers || [];
+                initializers = staticInitializers;
+            } else {
+                base = Class.prototype;
+                protoInitializers = protoInitializers || [];
+                initializers = protoInitializers;
+            }
+            if (kind !== 0 && !isPrivate) {
+                var existingNonFields = isStatic ? existingStaticNonFields : existingProtoNonFields;
+                var existingKind = existingNonFields.get(name) || 0;
+                if (existingKind === true || existingKind === 3 && kind !== 4 || existingKind === 4 && kind !== 3) {
+                    throw new Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + name);
+                } else if (!existingKind && kind > 2) {
+                    existingNonFields.set(name, kind);
+                } else {
+                    existingNonFields.set(name, true);
+                }
+            }
+            applyMemberDec(ret, base, decInfo, name, kind, isStatic, isPrivate, initializers, metadata);
+        }
+        pushInitializers(ret, protoInitializers);
+        pushInitializers(ret, staticInitializers);
+        return ret;
+    }
+    function pushInitializers(ret, initializers) {
+        if (initializers) {
+            ret.push(function(instance) {
+                for(var i = 0; i < initializers.length; i++){
+                    initializers[i].call(instance);
+                }
+                return instance;
+            });
+        }
+    }
+    function applyClassDecs(targetClass, classDecs, metadata) {
+        if (classDecs.length > 0) {
+            var initializers = [];
+            var newClass = targetClass;
+            var name = targetClass.name;
+            for(var i = classDecs.length - 1; i >= 0; i--){
+                var decoratorFinishedRef = {
+                    v: false
+                };
+                try {
+                    var nextNewClass = classDecs[i](newClass, {
+                        kind: "class",
+                        name: name,
+                        addInitializer: createAddInitializerMethod(initializers, decoratorFinishedRef),
+                        metadata
+                    });
+                } finally{
+                    decoratorFinishedRef.v = true;
+                }
+                if (nextNewClass !== undefined) {
+                    assertValidReturnValue(10, nextNewClass);
+                    newClass = nextNewClass;
+                }
+            }
+            return [
+                defineMetadata(newClass, metadata),
+                function() {
+                    for(var i = 0; i < initializers.length; i++){
+                        initializers[i].call(newClass);
+                    }
+                }
+            ];
+        }
+    }
+    function defineMetadata(Class, metadata) {
+        return Object.defineProperty(Class, Symbol.metadata || Symbol.for("Symbol.metadata"), {
+            configurable: true,
+            enumerable: true,
+            value: metadata
+        });
+    }
+    return function applyDecs2203R(targetClass, memberDecs, classDecs, parentClass) {
+        if (parentClass !== void 0) {
+            var parentMetadata = parentClass[Symbol.metadata || Symbol.for("Symbol.metadata")];
+        }
+        var metadata = Object.create(parentMetadata === void 0 ? null : parentMetadata);
+        var e = applyMemberDecs(targetClass, memberDecs, metadata);
+        if (!classDecs.length) defineMetadata(targetClass, metadata);
+        return {
+            e: e,
+            get c () {
+                return applyClassDecs(targetClass, classDecs, metadata);
+            }
+        };
+    };
+}
+function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
+    return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
+}
+var _dec, _initProto;
 import { asyncPool } from '@aztec/foundation/async-pool';
+import { BlockNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
-import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { RunningPromise, promiseWithResolvers } from '@aztec/foundation/promise';
 import { Timer } from '@aztec/foundation/timer';
+import { AVM_MAX_CONCURRENT_SIMULATIONS } from '@aztec/native';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
+import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
 import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
+import { appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging';
+import { CheckpointConstantData } from '@aztec/stdlib/rollup';
 import { Attributes, trackSpan } from '@aztec/telemetry-client';
 import * as crypto from 'node:crypto';
+import { validateEpochProvingJobData } from './epoch-proving-job-data.js';
+_dec = trackSpan('EpochProvingJob.run', function() {
+    return {
+        [Attributes.EPOCH_NUMBER]: this.data.epochNumber
+    };
+});
 /**
- * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
+ * Job that grabs a range of blocks from the unfinalized chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
  * world state as part of public call execution via the public processor.
  */ export class EpochProvingJob {
+    data;
     dbProvider;
-    epochNumber;
-    blocks;
-    txs;
     prover;
     publicProcessorFactory;
     publisher;
     l2BlockSource;
-    l1ToL2MessageSource;
     metrics;
     deadline;
     config;
-    cleanUp;
+    static{
+        ({ e: [_initProto] } = _apply_decs_2203_r(this, [
+            [
+                _dec,
+                2,
+                "run"
+            ]
+        ], []));
+    }
     state;
     log;
     uuid;
     runPromise;
+    abortController;
+    epochCheckPromise;
     deadlineTimeoutHandler;
     tracer;
-    constructor(dbProvider, epochNumber, blocks, txs, prover, publicProcessorFactory, publisher, l2BlockSource, l1ToL2MessageSource, metrics, deadline, config = {
-        parallelBlockLimit: 32
-    }, cleanUp = ()=>Promise.resolve()){
+    constructor(data, dbProvider, prover, publicProcessorFactory, publisher, l2BlockSource, metrics, deadline, config, bindings){
+        this.data = data;
         this.dbProvider = dbProvider;
-        this.epochNumber = epochNumber;
-        this.blocks = blocks;
-        this.txs = txs;
         this.prover = prover;
         this.publicProcessorFactory = publicProcessorFactory;
         this.publisher = publisher;
         this.l2BlockSource = l2BlockSource;
-        this.l1ToL2MessageSource = l1ToL2MessageSource;
         this.metrics = metrics;
         this.deadline = deadline;
         this.config = config;
-        this.cleanUp = cleanUp;
-        this.state = 'initialized';
-        this.log = createLogger('prover-node:epoch-proving-job');
+        this.state = (_initProto(this), 'initialized');
+        this.abortController = new AbortController();
+        validateEpochProvingJobData(data);
         this.uuid = crypto.randomUUID();
-        this.tracer = metrics.client.getTracer('EpochProvingJob');
+        this.log = createLogger('prover-node:epoch-proving-job', {
+            ...bindings,
+            instanceId: `epoch-${data.epochNumber}`
+        });
+        this.tracer = metrics.tracer;
     }
     getId() {
         return this.uuid;
@@ -63,23 +515,46 @@ import * as crypto from 'node:crypto';
         return this.state;
     }
     getEpochNumber() {
-        return this.epochNumber;
+        return this.data.epochNumber;
+    }
+    getDeadline() {
+        return this.deadline;
+    }
+    getProvingData() {
+        return this.data;
+    }
+    get epochNumber() {
+        return this.data.epochNumber;
+    }
+    get checkpoints() {
+        return this.data.checkpoints;
+    }
+    get txs() {
+        return this.data.txs;
+    }
+    get attestations() {
+        return this.data.attestations;
     }
     /**
    * Proves the given epoch and submits the proof to L1.
    */ async run() {
         this.scheduleDeadlineStop();
-        const epochNumber = Number(this.epochNumber);
-        const epochSizeBlocks = this.blocks.length;
-        const epochSizeTxs = this.blocks.reduce((total, current)=>total + current.body.txEffects.length, 0);
-        const [fromBlock, toBlock] = [
-            this.blocks[0].number,
-            this.blocks.at(-1).number
-        ];
-        this.log.info(`Starting epoch ${epochNumber} proving job with blocks ${fromBlock} to ${toBlock}`, {
+        if (!this.config.skipEpochCheck) {
+            await this.scheduleEpochCheck();
+        }
+        const attestations = this.attestations.map((attestation)=>attestation.toViem());
+        const epochNumber = this.epochNumber;
+        const epochSizeCheckpoints = this.checkpoints.length;
+        const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.length, 0);
+        const epochSizeTxs = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.reduce((accumC, block)=>accumC + block.body.txEffects.length, 0), 0);
+        const fromCheckpoint = this.checkpoints[0].number;
+        const toCheckpoint = this.checkpoints.at(-1).number;
+        const fromBlock = this.checkpoints[0].blocks[0].number;
+        const toBlock = this.checkpoints.at(-1).blocks.at(-1).number;
+        this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
             fromBlock,
             toBlock,
-            epochSizeBlocks,
+            epochSizeTxs,
             epochNumber,
             uuid: this.uuid
         });
@@ -88,71 +563,159 @@ import * as crypto from 'node:crypto';
         const { promise, resolve } = promiseWithResolvers();
         this.runPromise = promise;
         try {
-            this.prover.startNewEpoch(epochNumber, fromBlock, epochSizeBlocks);
-            await this.prover.startTubeCircuits(this.txs);
-            await asyncPool(this.config.parallelBlockLimit, this.blocks, async (block)=>{
+            const blobTimer = new Timer();
+            const blobFieldsPerCheckpoint = this.checkpoints.map((checkpoint)=>checkpoint.toBlobFields());
+            const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
+            this.metrics.recordBlobProcessing(blobTimer.ms());
+            this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
+            const chonkTimer = new Timer();
+            await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
+            this.metrics.recordChonkVerifier(chonkTimer.ms());
+            // Everything in the epoch should have the same chainId and version.
+            const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
+            const previousBlockHeaders = this.gatherPreviousBlockHeaders();
+            const allCheckpointsTimer = new Timer();
+            const parallelism = this.config.parallelBlockLimit ? this.config.parallelBlockLimit : AVM_MAX_CONCURRENT_SIMULATIONS > 0 ? AVM_MAX_CONCURRENT_SIMULATIONS : this.checkpoints.length;
+            await this.processCheckpoints(parallelism, async (checkpoint)=>{
                 this.checkState();
-                const globalVariables = block.header.globalVariables;
-                const txs = await this.getTxs(block);
-                const l1ToL2Messages = await this.getL1ToL2Messages(block);
-                const previousHeader = await this.getBlockHeader(block.number - 1);
-                this.log.verbose(`Starting processing block ${block.number}`, {
-                    number: block.number,
-                    blockHash: (await block.hash()).toString(),
-                    lastArchive: block.header.lastArchive.root,
-                    noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
-                    nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
-                    publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
-                    previousHeader: previousHeader.hash(),
-                    uuid: this.uuid,
-                    ...globalVariables
+                const checkpointTimer = new Timer();
+                const checkpointIndex = checkpoint.number - fromCheckpoint;
+                const checkpointConstants = CheckpointConstantData.from({
+                    chainId,
+                    version,
+                    vkTreeRoot: getVKTreeRoot(),
+                    protocolContractsHash: protocolContractsHash,
+                    proverId: this.prover.getProverId().toField(),
+                    slotNumber: checkpoint.header.slotNumber,
+                    coinbase: checkpoint.header.coinbase,
+                    feeRecipient: checkpoint.header.feeRecipient,
+                    gasFees: checkpoint.header.gasFees
                 });
-                // Start block proving
-                await this.prover.startNewBlock(globalVariables, l1ToL2Messages, previousHeader);
-                // Process public fns
-                const db = await this.dbProvider.fork(block.number - 1);
-                const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, true);
-                const processed = await this.processTxs(publicProcessor, txs);
-                await this.prover.addTxs(processed);
-                await db.close();
-                this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
-                    blockNumber: block.number,
-                    blockHash: (await block.hash()).toString(),
+                const previousHeader = previousBlockHeaders[checkpointIndex];
+                const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
+                this.log.debug(`Starting processing checkpoint ${checkpoint.number}`, {
+                    number: checkpoint.number,
+                    checkpointHash: checkpoint.hash().toString(),
+                    headerHash: checkpoint.header.hash().toString(),
+                    numL1ToL2Messages: l1ToL2Messages.length,
+                    previousBlockNumber: previousHeader.globalVariables.blockNumber,
                     uuid: this.uuid
                 });
-                // Mark block as completed to pad it
-                await this.prover.setBlockCompleted(block.number, block.header);
+                await this.prover.startNewCheckpoint(checkpointIndex, checkpointConstants, l1ToL2Messages, checkpoint.blocks.length, previousHeader);
+                for(let blockIndex = 0; blockIndex < checkpoint.blocks.length; blockIndex++){
+                    const blockTimer = new Timer();
+                    const block = checkpoint.blocks[blockIndex];
+                    const globalVariables = block.header.globalVariables;
+                    const txs = this.getTxs(block);
+                    this.log.verbose(`Starting processing block ${block.number}`, {
+                        number: block.number,
+                        blockHash: (await block.hash()).toString(),
+                        lastArchive: block.header.lastArchive.root,
+                        noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
+                        nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
+                        publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
+                        ...globalVariables,
+                        numTxs: txs.length
+                    });
+                    // Start block proving
+                    await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
+                    // Process public fns. L1 to L2 messages are only inserted for the first block of a checkpoint,
+                    // as the fork for subsequent blocks already includes them from the previous block's synced state.
+                    {
+                        const env = {
+                            stack: [],
+                            error: void 0,
+                            hasError: false
+                        };
+                        try {
+                            const db = _ts_add_disposable_resource(env, await this.createFork(BlockNumber(block.number - 1), blockIndex === 0 ? l1ToL2Messages : undefined), true);
+                            this.checkState();
+                            const config = PublicSimulatorConfig.from({
+                                proverId: this.prover.getProverId().toField(),
+                                skipFeeEnforcement: false,
+                                collectDebugLogs: false,
+                                collectHints: true,
+                                collectPublicInputs: true,
+                                collectStatistics: false
+                            });
+                            const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
+                            const processed = await this.processTxs(publicProcessor, txs);
+                            this.checkState();
+                            await this.prover.addTxs(processed);
+                        } catch (e) {
+                            env.error = e;
+                            env.hasError = true;
+                        } finally{
+                            const result = _ts_dispose_resources(env);
+                            if (result) await result;
+                        }
+                    }
+                    this.checkState();
+                    this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
+                        blockNumber: block.number,
+                        blockHash: (await block.hash()).toString(),
+                        uuid: this.uuid
+                    });
+                    // Mark block as completed to pad it
+                    const expectedBlockHeader = block.header;
+                    await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
+                    this.metrics.recordBlockProcessing(blockTimer.ms());
+                }
+                this.metrics.recordCheckpointProcessing(checkpointTimer.ms());
             });
+            this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms());
             const executionTime = timer.ms();
             this.progressState('awaiting-prover');
-            const { publicInputs, proof } = await this.prover.finaliseEpoch();
-            this.log.info(`Finalised proof for epoch ${epochNumber}`, {
+            const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
+            this.log.info(`Finalized proof for epoch ${epochNumber}`, {
                 epochNumber,
                 uuid: this.uuid,
                 duration: timer.ms()
             });
             this.progressState('publishing-proof');
+            if (this.config.skipSubmitProof) {
+                this.log.info(`Proof publishing is disabled. Analyzing estimated L1 fees for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`);
+                try {
+                    await this.publisher.analyzeEpochProofSubmission({
+                        fromCheckpoint,
+                        toCheckpoint,
+                        epochNumber,
+                        publicInputs,
+                        proof,
+                        batchedBlobInputs,
+                        attestations
+                    });
+                } catch (err) {
+                    this.log.warn(`Failed to analyze estimated L1 fees for epoch ${epochNumber}`, err);
+                }
+                this.state = 'completed';
+                this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
+                return;
+            }
             const success = await this.publisher.submitEpochProof({
-                fromBlock,
-                toBlock,
+                fromCheckpoint,
+                toCheckpoint,
                 epochNumber,
                 publicInputs,
-                proof
+                proof,
+                batchedBlobInputs,
+                attestations
             });
             if (!success) {
                 throw new Error('Failed to submit epoch proof to L1');
             }
-            this.log.info(`Submitted proof for epoch ${epochNumber} (blocks ${fromBlock} to ${toBlock})`, {
+            this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
                 epochNumber,
                 uuid: this.uuid
             });
             this.state = 'completed';
-            this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeBlocks, epochSizeTxs);
+            this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
         } catch (err) {
             if (err && err.name === 'HaltExecutionError') {
                 this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
                     uuid: this.uuid,
-                    epochNumber
+                    epochNumber,
+                    details: err.message
                 });
                 return;
             }
@@ -160,31 +723,102 @@ import * as crypto from 'node:crypto';
                 uuid: this.uuid,
                 epochNumber
             });
-            this.state = 'failed';
+            if (this.state === 'processing' || this.state === 'awaiting-prover' || this.state === 'publishing-proof') {
+                this.state = 'failed';
+            }
         } finally{
             clearTimeout(this.deadlineTimeoutHandler);
-            await this.cleanUp(this);
+            await this.epochCheckPromise?.stop();
             await this.prover.stop();
             resolve();
         }
     }
+    /**
+   * Create a new db fork for tx processing, optionally inserting L1 to L2 messages.
+   * L1 to L2 messages should only be inserted for the first block in a checkpoint,
+   * as subsequent blocks' synced state already includes them.
+   * REFACTOR: The prover already spawns a db fork of its own for each block, so we may be able to do away with just one fork.
+   */ async createFork(blockNumber, l1ToL2Messages) {
+        const env = {
+            stack: [],
+            error: void 0,
+            hasError: false
+        };
+        try {
+            this.log.verbose(`Creating fork at ${blockNumber}`, {
+                blockNumber
+            });
+            const cleanup = _ts_add_disposable_resource(env, new AsyncDisposableStack(), true);
+            const db = cleanup.use(await this.dbProvider.fork(blockNumber));
+            if (l1ToL2Messages !== undefined) {
+                this.log.verbose(`Inserting ${l1ToL2Messages.length} L1 to L2 messages in fork`, {
+                    blockNumber,
+                    l1ToL2Messages: l1ToL2Messages.map((m)=>m.toString())
+                });
+                await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
+            }
+            // everything run succesfully so we can release this stack and give control of the fork's lifetime to the caller
+            cleanup.move();
+            return db;
+        } catch (e) {
+            env.error = e;
+            env.hasError = true;
+        } finally{
+            const result = _ts_dispose_resources(env);
+            if (result) await result;
+        }
+    }
+    async processCheckpoints(parallelism, processCheckpoint) {
+        let hasError = false;
+        let firstError;
+        await asyncPool(Math.max(parallelism, 1), this.checkpoints, async (checkpoint)=>{
+            if (hasError || this.abortController.signal.aborted) {
+                return;
+            }
+            try {
+                this.checkState();
+                await processCheckpoint(checkpoint);
+            } catch (err) {
+                if (!hasError) {
+                    hasError = true;
+                    firstError = err;
+                    this.failProcessing();
+                }
+            }
+        });
+        if (hasError) {
+            throw firstError;
+        }
+        if (this.abortController.signal.aborted) {
+            this.checkState();
+        }
+    }
     progressState(state) {
         this.checkState();
         this.state = state;
     }
     checkState() {
-        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed') {
+        if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
             throw new HaltExecutionError(this.state);
         }
     }
     async stop(state = 'stopped') {
         this.state = state;
-        this.prover.cancel();
-        // TODO(palla/prover): Stop the publisher as well
+        this.interruptProcessing();
         if (this.runPromise) {
             await this.runPromise;
         }
     }
+    failProcessing() {
+        if (!EpochProvingJobTerminalState.includes(this.state)) {
+            this.state = 'failed';
+        }
+        this.interruptProcessing();
+    }
+    interruptProcessing() {
+        this.abortController.abort();
+        this.prover.cancel();
+    }
     scheduleDeadlineStop() {
         const deadline = this.deadline;
         if (deadline) {
@@ -209,30 +843,59 @@ import * as crypto from 'node:crypto';
             }, timeout);
         }
     }
-    /* Returns the header for the given block number, or the genesis block for block zero. */ async getBlockHeader(blockNumber) {
-        if (blockNumber === 0) {
-            return (await this.dbProvider.fork()).getInitialHeader();
+    /**
+   * Kicks off a running promise that queries the archiver for the set of L2 blocks of the current epoch.
+   * If those change, stops the proving job with a `rerun` state, so the node re-enqueues it.
+   */ async scheduleEpochCheck() {
+        const l2BlockSource = this.l2BlockSource;
+        if (!l2BlockSource) {
+            this.log.warn(`No L2 block source available, skipping epoch check`);
+            return;
         }
-        return this.l2BlockSource.getBlockHeader(blockNumber);
+        const intervalMs = Math.ceil((await l2BlockSource.getL1Constants()).ethereumSlotDuration / 2) * 1000;
+        this.epochCheckPromise = new RunningPromise(async ()=>{
+            const blockHeaders = (await l2BlockSource.getBlocksData({
+                epoch: this.epochNumber,
+                onlyCheckpointed: true
+            })).map((d)=>d.header);
+            const blockHashes = await Promise.all(blockHeaders.map((header)=>header.hash()));
+            const thisBlocks = this.checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
+            const thisBlockHashes = await Promise.all(thisBlocks.map((block)=>block.hash()));
+            if (blockHeaders.length !== thisBlocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
+                this.log.warn('Epoch blocks changed underfoot', {
+                    uuid: this.uuid,
+                    epochNumber: this.epochNumber,
+                    oldBlockHashes: thisBlockHashes,
+                    newBlockHashes: blockHashes
+                });
+                void this.stop('reorg');
+            }
+        }, this.log, intervalMs).start();
+        this.log.verbose(`Scheduled epoch check for epoch ${this.epochNumber} every ${intervalMs}ms`);
+    }
+    /* Returns the last block header in the previous checkpoint for all checkpoints in the epoch */ gatherPreviousBlockHeaders() {
+        const lastBlocks = this.checkpoints.map((checkpoint)=>checkpoint.blocks.at(-1));
+        return [
+            this.data.previousBlockHeader,
+            ...lastBlocks.map((block)=>block.header).slice(0, -1)
+        ];
     }
-    async getTxs(block) {
-        const txHashes = block.body.txEffects.map((tx)=>tx.txHash.toBigInt());
-        const txsAndHashes = await Promise.all(this.txs.map(async (tx)=>({
-                tx,
-                hash: await tx.getTxHash()
-            })));
-        return txsAndHashes.filter((txAndHash)=>txHashes.includes(txAndHash.hash.toBigInt())).map((txAndHash)=>txAndHash.tx);
+    getTxs(block) {
+        return block.body.txEffects.map((txEffect)=>this.txs.get(txEffect.txHash.toString()));
     }
-    getL1ToL2Messages(block) {
-        return this.l1ToL2MessageSource.getL1ToL2Messages(BigInt(block.number));
+    getL1ToL2Messages(checkpoint) {
+        return this.data.l1ToL2Messages[checkpoint.number];
     }
     async processTxs(publicProcessor, txs) {
         const { deadline } = this;
         const [processedTxs, failedTxs] = await publicProcessor.process(txs, {
-            deadline
+            deadline,
+            signal: this.abortController.signal
         });
+        this.checkState();
         if (failedTxs.length) {
-            throw new Error(`Txs failed processing: ${failedTxs.map(({ tx, error })=>`${tx.getTxHash()} (${error})`).join(', ')}`);
+            const failedTxHashes = await Promise.all(failedTxs.map(({ tx })=>tx.getTxHash()));
+            throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
         }
         if (processedTxs.length !== txs.length) {
             throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
@@ -240,16 +903,10 @@ import * as crypto from 'node:crypto';
         return processedTxs;
     }
 }
-_ts_decorate([
-    trackSpan('EpochProvingJob.run', function() {
-        return {
-            [Attributes.EPOCH_NUMBER]: Number(this.epochNumber)
-        };
-    })
-], EpochProvingJob.prototype, "run", null);
 class HaltExecutionError extends Error {
+    state;
     constructor(state){
-        super(`Halted execution due to state ${state}`);
+        super(`Halted execution due to state ${state}`), this.state = state;
         this.name = 'HaltExecutionError';
     }
 }