@aztec/prover-node 0.0.1-commit.3fd054f6 → 0.0.1-commit.431c48d

package/dest/prover-node.js

@@ -370,28 +370,40 @@ function applyDecs2203RFactory() {
 function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
     return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
 }
-var _dec, _dec1, _initProto;
-import { BlockNumber } from '@aztec/foundation/branded-types';
-import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
+var _initProto;
+import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
+import { assertRequired, compact, pick } from '@aztec/foundation/collection';
 import { memoize } from '@aztec/foundation/decorators';
 import { createLogger } from '@aztec/foundation/log';
-import { DateProvider } from '@aztec/foundation/timer';
+import { RunningPromise } from '@aztec/foundation/running-promise';
+import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
+import { getLastSiblingPath } from '@aztec/prover-client/helpers';
+import { ChonkCache } from '@aztec/prover-client/orchestrator';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
-import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
+import { EventDrivenL2BlockStream, L2TipsMemoryStore } from '@aztec/stdlib/block';
+import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
 import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
-import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
+import { getTelemetryClient } from '@aztec/telemetry-client';
 import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
-import { EpochProvingJob } from './job/epoch-proving-job.js';
+import { CheckpointStore } from './checkpoint-store.js';
 import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
-_dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
-        [Attributes.EPOCH_NUMBER]: epochNumber
-    })), _dec1 = trackSpan('ProverNode.gatherEpochData', (epochNumber)=>({
-        [Attributes.EPOCH_NUMBER]: epochNumber
-    }));
+import { ProofPublishingService } from './proof-publishing-service.js';
+import { SessionManager } from './session-manager.js';
 /**
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
- * proof for the epoch, and submits it to L1.
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
+ * the wait rather than letting `stop()` hang indefinitely.
+ */ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
+/**
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
+ * complete.
+ *
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
  */ export class ProverNode {
     prover;
     publisherFactory;
@@ -400,7 +412,6 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
     contractDataSource;
     worldState;
     p2pClient;
-    epochsMonitor;
     rollupContract;
     l1Metrics;
     telemetryClient;
@@ -408,31 +419,38 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
     dateProvider;
     static{
         ({ e: [_initProto] } = _apply_decs_2203_r(this, [
-            [
-                _dec,
-                2,
-                "createProvingJob"
-            ],
             [
                 memoize,
                 2,
                 "getL1Constants"
-            ],
-            [
-                _dec1,
-                2,
-                "gatherEpochData"
             ]
         ], []));
     }
     log;
-    jobs;
+    checkpointStore;
+    chonkCache;
+    sessionManager;
     config;
     jobMetrics;
     rewardsMetrics;
+    /** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
+    /** Block stream for checkpoint and reorg detection. */ blockStream;
+    /**
+   * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
+   * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
+   * event by comparing the archiver's latest synced L2 slot against each epoch's
+   * submission deadline. Protected so tests can verify the start() seeding.
+   */ lastExpiredEpoch;
+    /**
+   * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
+   * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
+   * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
+   * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
+   */ lastProcessedCheckpoint;
+    /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */ expiryTicker;
     tracer;
-    publisher;
-    constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, epochsMonitor, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
+    publishingService;
+    constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
         this.prover = prover;
         this.publisherFactory = publisherFactory;
         this.l2BlockSource = l2BlockSource;
@@ -440,14 +458,13 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
         this.contractDataSource = contractDataSource;
         this.worldState = worldState;
         this.p2pClient = p2pClient;
-        this.epochsMonitor = epochsMonitor;
         this.rollupContract = rollupContract;
         this.l1Metrics = l1Metrics;
         this.telemetryClient = telemetryClient;
         this.delayer = delayer;
         this.dateProvider = dateProvider;
         this.log = (_initProto(this), createLogger('prover-node'));
-        this.jobs = new Map();
+        this.lastProcessedCheckpoint = CheckpointNumber.ZERO;
         this.config = {
             proverNodePollingIntervalMs: 1_000,
             proverNodeMaxPendingJobs: 100,
@@ -465,6 +482,23 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
         this.tracer = telemetryClient.getTracer('ProverNode');
         this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
         this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
+        this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
+        this.chonkCache = new ChonkCache(this.log.getBindings());
+        this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
+            proverFactory: this.prover,
+            chonkCache: this.chonkCache,
+            publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings()),
+            dbProvider: this.worldState,
+            txProvider: this.p2pClient.getTxProvider(),
+            dateProvider: this.dateProvider,
+            proverId: this.prover.getProverId(),
+            metrics: this.jobMetrics,
+            txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
+            deadline: undefined,
+            // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
+            // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
+            onFailed: (prover)=>void this.tryUploadCheckpointFailure(prover)
+        }, this.log.getBindings());
     }
     getProverId() {
         return this.prover.getProverId();
@@ -472,233 +506,520 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
     getP2P() {
         return this.p2pClient;
     }
-    /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */ getDelayer() {
+    /** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
         return this.delayer;
     }
+    /** Observability summary for the ProverNodeApi. */ getJobs() {
+        return Promise.resolve(this.sessionManager?.getJobs() ?? []);
+    }
+    /** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
+        return this.checkpointStore;
+    }
+    /** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
+        return this.chonkCache;
+    }
+    /** Tests inspect this when looking up live sessions. */ getSessionManager() {
+        if (!this.sessionManager) {
+            throw new Error('SessionManager not yet constructed — start() must be called first.');
+        }
+        return this.sessionManager;
+    }
+    /** Returns the underlying prover instance. */ getProver() {
+        return this.prover;
+    }
+    // ---------------- L2BlockStream handler ----------------
+    async handleBlockStreamEvent(event) {
+        switch(event.type){
+            case 'chain-checkpointed':
+                await this.processCheckpointJump(event.checkpoint.number);
+                break;
+            case 'chain-pruned':
+                await this.handlePruneEvent(event.block);
+                break;
+            case 'chain-proven':
+                this.publishingService?.onChainProven(BlockNumber(event.block.number));
+                break;
+            // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
+            // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
+            // `chain-finalized` carries nothing the prover-node acts on.
+            case 'chain-proposed':
+            case 'chain-finalized':
+            case 'blocks-added':
+                break;
+            default:
+                {
+                    const _ = event;
+                    break;
+                }
+        }
+        // Advance the local tips store only after the proving-side handling (registration / prune) has
+        // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
+        // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
+        await this.tipsStore.handleBlockStreamEvent(event);
+    }
     /**
-   * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
-   * @param epochNumber - The epoch number that was just completed.
-   * @returns false if there is an error, true otherwise
-   */ async handleEpochReadyToProve(epochNumber) {
-        try {
-            this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
-                jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
+   * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
+   * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
+   * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
+   * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
+   * `getCheckpoints` fetch only for checkpoints in provable epochs.
+   *
+   * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
+   * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
+   * behind so the next pass retries from the first checkpoint that did not complete.
+   */ async processCheckpointJump(targetCheckpoint) {
+        if (targetCheckpoint <= this.lastProcessedCheckpoint) {
+            return;
+        }
+        const l1Constants = await this.getL1Constants();
+        // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
+        // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
+        // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
+        // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
+        const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
+        let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
+        if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
+            const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
+            this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
+                from,
+                cappedFrom,
+                targetCheckpoint,
+                maxCheckpoints
             });
-            const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
-            if (activeJobs.length > 0) {
-                this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
-                    activeJobs: activeJobs.map((job)=>job.uuid)
-                });
-                return true;
+            // Advance the cursor past the skipped checkpoints so they are never retried.
+            this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
+            from = cappedFrom;
+        }
+        const limit = Number(targetCheckpoint - from) + 1;
+        const metadatas = await this.l2BlockSource.getCheckpointsData({
+            from,
+            limit
+        });
+        // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
+        // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
+        // never drop an individual checkpoint inside an epoch we will prove.
+        const epochSkippable = new Map();
+        for (const metadata of metadatas){
+            const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
+            let skippable = epochSkippable.get(epochNumber);
+            if (skippable === undefined) {
+                skippable = await this.isEpochFullyProven(epochNumber, l1Constants) || await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants);
+                epochSkippable.set(epochNumber, skippable);
             }
-            await this.startProof(epochNumber);
-            return true;
-        } catch (err) {
-            if (err instanceof EmptyEpochError) {
-                this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
+            if (skippable) {
+                this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
             } else {
-                this.log.error(`Error handling epoch completed`, err);
+                await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
             }
+            // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
+            // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
+            this.lastProcessedCheckpoint = metadata.checkpointNumber;
+        }
+    }
+    /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */ async registerCheckpoint(checkpointNumber, epochNumber) {
+        const published = await this.l2BlockSource.getCheckpoint({
+            number: checkpointNumber
+        });
+        if (!published) {
+            throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
+        }
+        const checkpoint = published.checkpoint;
+        this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
+            checkpointNumber: checkpoint.number,
+            epochNumber,
+            slotNumber: checkpoint.header.slotNumber
+        });
+        const registerData = await this.collectRegisterData(checkpoint, published.attestations);
+        await this.checkpointStore.addOrUpdate(checkpoint, registerData);
+        await this.sessionManager?.onCheckpointAdded(epochNumber);
+        // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
+        // range then prunes at the true divergence instead of the nearest sparse tip anchor.
+        await this.tipsStore.recordBlockHashes(await Promise.all(checkpoint.blocks.map(async (block)=>({
+                number: block.number,
+                hash: (await block.header.hash()).toString()
+            }))));
+    }
+    /**
+   * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
+   * and the archive sibling path.
+   */ async collectRegisterData(checkpoint, attestations) {
+        const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
+        const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
+        const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
+        const lastBlock = checkpoint.blocks.at(-1);
+        const lastBlockHash = await lastBlock.header.hash();
+        await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
+        const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
+        return {
+            attestations,
+            previousBlockHeader,
+            l1ToL2Messages,
+            previousArchiveSiblingPath
+        };
+    }
+    /**
+   * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
+   * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
+   * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
+   * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
+   * fails and the prune is retried next iteration.
+   */ async handlePruneEvent(prunedToBlock) {
+        this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, {
+            prunedToBlock
+        });
+        // Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
+        // so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
+        // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
+        let cursorFloor;
+        if (prunedToBlock.number === 0) {
+            cursorFloor = CheckpointNumber.ZERO;
+        } else {
+            const targetData = await this.l2BlockSource.getBlockData({
+                number: prunedToBlock.number
+            });
+            if (targetData === undefined) {
+                throw new Error(`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`);
+            }
+            // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
+            // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
+            // would permanently skip a rebuilt same-number checkpoint.
+            cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
+        }
+        const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
+        if (this.lastProcessedCheckpoint > cursorFloor) {
+            this.lastProcessedCheckpoint = cursorFloor;
+        }
+        if (affected.length === 0) {
+            return;
+        }
+        const l1Constants = await this.getL1Constants();
+        const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
+        // The session manager cancels every affected session, which in turn calls
+        // publishingService.withdraw(uuid) for each candidate; no separate notification to the
+        // publishing service is needed.
+        await this.sessionManager?.onPrune(affectedEpochs);
+    }
+    /**
+   * Returns true once the chain has advanced past the given epoch's proof-submission window.
+   * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
+   * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
+   * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
+   * be read yet.
+   */ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
+        const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
+        if (latestSlot === undefined) {
             return false;
         }
+        const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
+        return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
+    }
+    /**
+   * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
+   * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
+   * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
+   * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
+   * the last check or the archiver's slot can't be read.
+   */ async checkEpochExpiry() {
+        const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
+        if (latestSlot === undefined) {
+            return;
+        }
+        const l1Constants = await this.getL1Constants();
+        const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
+        const offset = l1Constants.proofSubmissionEpochs + 1;
+        if (latestEpoch < offset) {
+            return;
+        }
+        const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
+        const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
+        if (newlyExpiredUpTo < from) {
+            return;
+        }
+        for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
+            await this.expireEpoch(e);
+        }
+        this.lastExpiredEpoch = newlyExpiredUpTo;
+    }
+    /**
+   * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
+   * CheckpointProver in the store whose epoch is at or below it.
+   */ async expireEpoch(epoch) {
+        try {
+            const blocks = await this.l2BlockSource.getBlocks({
+                epoch,
+                onlyCheckpointed: true
+            });
+            if (blocks.length > 0) {
+                this.chonkCache.releaseForBlocks(blocks);
+            }
+        } catch (err) {
+            this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
+        }
+        this.checkpointStore.reapExpired(epoch);
     }
+    // ---------------- public API ----------------
     /**
-   * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
-   * starts proving jobs for them.
-   */ async start() {
-        this.epochsMonitor.start(this);
+   * Schedules proving for the given epoch and returns the job id without waiting for completion.
+   */ async startProof(epochNumber) {
+        if (!this.sessionManager) {
+            throw new Error('ProverNode not started');
+        }
+        return await this.sessionManager.startProof(epochNumber);
+    }
+    // ---------------- Service lifecycle ----------------
+    async start() {
+        await this.checkpointStore.start();
         await this.publisherFactory.start();
-        this.publisher = await this.publisherFactory.create();
+        this.publishingService = new ProofPublishingService({
+            publisherFactory: this.publisherFactory,
+            l2BlockSource: this.l2BlockSource,
+            dateProvider: this.dateProvider,
+            config: {
+                skipSubmitProof: !!this.config.proverNodeDisableProofPublish
+            },
+            bindings: this.log.getBindings()
+        });
+        this.sessionManager = this.createSessionManager(this.publishingService);
+        // SessionManager owns its own periodic tick; start it here so it begins picking up
+        // epochs that become complete by time (no fresh checkpoint event) and advances once
+        // the previous epoch is proven on L1.
+        this.sessionManager.start();
+        // Now that the store + manager exist, arm the live-state observable gauges.
+        this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
+        const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
+        this.lastExpiredEpoch = lastFullyProvenEpoch;
+        this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
+        this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
+            pollIntervalMS: this.config.proverNodePollingIntervalMs,
+            tipsOnly: true
+        });
+        this.blockStream.start();
+        // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
+        // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
+        // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
+        this.expiryTicker = new RunningPromise(()=>this.checkEpochExpiry(), this.log, this.config.proverNodePollingIntervalMs);
+        this.expiryTicker.start();
         await this.rewardsMetrics.start();
         this.l1Metrics.start();
         this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
     }
-    /**
-   * Stops the prover node and all its dependencies.
-   * Resources not owned by this node (shared with the parent aztec-node) are skipped.
-   */ async stop() {
+    async stop() {
         this.log.info('Stopping ProverNode');
-        await this.epochsMonitor.stop();
+        this.jobMetrics.stopObservingState();
+        await this.blockStream?.stop();
+        await this.expiryTicker?.stop();
+        if (this.sessionManager) {
+            await this.sessionManager.stop();
+        }
+        if (this.publishingService) {
+            // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
+            // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
+            // the tx may still mine, but shutdown must not hang on it.
+            const publishingService = this.publishingService;
+            await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
+        }
+        await this.checkpointStore.stop();
+        this.chonkCache.stop();
         await this.prover.stop();
         await tryStop(this.publisherFactory);
-        this.publisher?.interrupt();
-        await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
         this.rewardsMetrics.stop();
         this.l1Metrics.stop();
         await this.telemetryClient.stop();
         this.log.info('Stopped ProverNode');
     }
-    /** Returns world state status. */ async getWorldStateSyncStatus() {
-        const { syncSummary } = await this.worldState.status();
-        return syncSummary;
+    /**
+   * Constructs the session manager. Extracted so subclasses (test harness) can swap the
+   * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
+   * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
+   * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
+   */ createSessionManager(publishingService) {
+        return new SessionManager({
+            checkpointStore: this.checkpointStore,
+            l2BlockSource: this.l2BlockSource,
+            proverFactory: this.prover,
+            proverId: this.prover.getProverId(),
+            publishingService,
+            metrics: this.jobMetrics,
+            dateProvider: this.dateProvider,
+            config: {
+                maxPendingJobs: this.config.proverNodeMaxPendingJobs,
+                tickIntervalMs: this.config.proverNodePollingIntervalMs,
+                finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
+            },
+            onSessionFailed: async (session)=>{
+                await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
+            },
+            bindings: this.log.getBindings()
+        });
     }
-    /** Returns archiver status. */ getL2Tips() {
-        return this.l2BlockSource.getL2Tips();
+    /**
+   * Installs session hooks for the e2e harness to interpose around top-tree proving
+   * (gate, override, or observe it) without monkey-patching the orchestrator factory.
+   * Applies to every session constructed after this call.
+   */ setSessionHooks(hooks) {
+        if (!this.sessionManager) {
+            throw new Error('ProverNode not started; call start() before setting session hooks.');
+        }
+        this.sessionManager.setSessionHooks(hooks);
     }
     /**
-   * Starts a proving process and returns immediately.
-   */ async startProof(epochNumber) {
-        const job = await this.createProvingJob(epochNumber, {
-            skipEpochCheck: true
-        });
-        void this.runJob(job);
+   * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
+   * CheckpointProver constructed after this call. For the e2e harness only.
+   */ setCheckpointHooks(hooks) {
+        this.checkpointStore.setTestHooks(hooks);
     }
-    async runJob(job) {
-        const epochNumber = job.getEpochNumber();
-        const ctx = {
-            id: job.getId(),
-            epochNumber,
-            state: undefined
-        };
+    /**
+   * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
+   * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
+   * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
+   * store is configured or the checkpoint set is empty.
+   */ async tryUploadEpochFailure(id, checkpoints) {
+        if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
+            return undefined;
+        }
+        const data = await SessionManager.buildProvingData(checkpoints);
+        return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
+        id, data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
+    }
+    /**
+   * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
+   * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
+   * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
+   * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
+   * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
+   */ async tryUploadCheckpointFailure(prover) {
+        if (!this.config.proverNodeFailedEpochStore) {
+            return undefined;
+        }
         try {
-            await job.run();
-            const state = job.getState();
-            ctx.state = state;
-            if (state === 'reorg') {
-                this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
-                await this.createProvingJob(epochNumber);
-            } else if (state === 'failed') {
-                this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
-                await this.tryUploadEpochFailure(job);
-            } else {
-                this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
+            // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
+            // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
+            // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
+            // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
+            // the snapshot (full world-state + archiver) is expensive to produce and store.
+            if (!await this.isCheckpointCanonical(prover.checkpoint)) {
+                this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
+                    checkpointNumber: prover.checkpoint.number
+                });
+                return undefined;
             }
+            const data = await SessionManager.buildProvingData([
+                prover
+            ]);
+            return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, // The prover's content-addressed id; the epoch number is already in the upload path.
+            prover.id, data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
         } catch (err) {
-            this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
-        } finally{
-            this.jobs.delete(job.getId());
+            this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
+            return undefined;
         }
     }
-    async tryUploadEpochFailure(job) {
-        if (this.config.proverNodeFailedEpochStore) {
-            return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, job.getId(), job.getProvingData(), this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
+    // ---------------- helpers ----------------
+    /**
+   * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
+   * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
+   * so this returns false. Protected for direct unit-test access.
+   */ async isCheckpointCanonical(checkpoint) {
+        const lastBlock = checkpoint.blocks.at(-1);
+        if (!lastBlock) {
+            return false;
         }
+        const onChain = await this.l2BlockSource.getBlock({
+            number: lastBlock.number
+        });
+        return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
     }
-    /**
-   * Returns the prover instance.
-   */ getProver() {
-        return this.prover;
+    getL1Constants() {
+        return this.l2BlockSource.getL1Constants();
     }
     /**
-   * Returns an array of jobs being processed.
-   */ getJobs() {
-        return Promise.resolve(Array.from(this.jobs.entries()).map(([uuid, job])=>({
-                uuid,
-                status: job.getState(),
-                epochNumber: job.getEpochNumber()
-            })));
-    }
-    async getActiveJobsForEpoch(epochNumber) {
-        const jobs = await this.getJobs();
-        return jobs.filter((job)=>job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
-    }
-    checkMaximumPendingJobs() {
-        const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
-        if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
-            throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
-        }
-    }
-    async createProvingJob(epochNumber, opts = {}) {
-        this.checkMaximumPendingJobs();
-        this.publisher = await this.publisherFactory.create();
-        // Gather all data for this epoch
-        const epochData = await this.gatherEpochData(epochNumber);
-        const fromCheckpoint = epochData.checkpoints[0].number;
-        const toCheckpoint = epochData.checkpoints.at(-1).number;
-        const fromBlock = epochData.checkpoints[0].blocks[0].number;
-        const lastBlock = epochData.checkpoints.at(-1).blocks.at(-1);
-        const toBlock = lastBlock.number;
-        this.log.verbose(`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`);
-        // Fast forward world state to right before the target block and get a fork
-        const lastBlockHash = await lastBlock.header.hash();
-        await this.worldState.syncImmediate(toBlock, lastBlockHash);
-        // Create a processor factory
-        const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings());
-        // Set deadline for this job to run. It will abort if it takes too long.
-        const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
-        const deadline = new Date(Number(deadlineTs) * 1000);
-        const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
-        this.jobs.set(job.getId(), job);
-        return job;
+   * Returns true if every block in the given epoch is proven on L1. An epoch is only
+   * fully proven when its *last* block is proven. Protected for direct unit-test access.
+   */ async isEpochFullyProven(epochNumber, l1Constants) {
+        const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
+            tag: 'proven'
+        });
+        if (!provenBlockNumber || provenBlockNumber <= 0) {
+            return false;
+        }
+        const provenHeader = (await this.l2BlockSource.getBlockData({
+            number: BlockNumber(provenBlockNumber)
+        }))?.header;
+        if (!provenHeader) {
+            return false;
+        }
+        const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
+        if (epochNumber < provenEpoch) {
+            return true;
+        }
+        if (epochNumber > provenEpoch) {
+            return false;
+        }
+        return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
     }
-    getL1Constants() {
-        return this.l2BlockSource.getL1Constants();
+    /** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
+        const nextHeader = (await this.l2BlockSource.getBlockData({
+            number: BlockNumber(provenBlockNumber + 1)
+        }))?.header;
+        if (nextHeader) {
+            return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
+        }
+        return this.l2BlockSource.isEpochComplete(provenEpoch);
     }
-    async gatherEpochData(epochNumber) {
-        const checkpoints = await this.gatherCheckpoints(epochNumber);
-        const txArray = await this.gatherTxs(epochNumber, checkpoints);
-        const txs = new Map(txArray.map((tx)=>[
-                tx.getTxHash().toString(),
-                tx
-            ]));
-        const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
-        const [firstBlock] = checkpoints[0].blocks;
-        const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
-        const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1).number, 1);
-        const attestations = lastPublishedCheckpoint?.attestations ?? [];
+    /**
+   * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
+   * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
+   * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
+   * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
+   */ async resolveLastFullyProvenEpoch() {
+        const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
+            tag: 'proven'
+        });
+        if (!provenBlockNumber || provenBlockNumber <= 0) {
+            return {
+                lastFullyProvenEpoch: undefined
+            };
+        }
+        const l1Constants = await this.getL1Constants();
+        const provenHeader = (await this.l2BlockSource.getBlockData({
+            number: BlockNumber(provenBlockNumber)
+        }))?.header;
+        if (!provenHeader) {
+            return {
+                lastFullyProvenEpoch: undefined
+            };
+        }
+        const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
+        if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
+            return {
+                lastFullyProvenEpoch: provenEpoch
+            };
+        }
+        const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
         return {
-            checkpoints,
-            txs,
-            l1ToL2Messages,
-            epochNumber,
-            previousBlockHeader,
-            attestations
+            lastFullyProvenEpoch
         };
     }
-    async gatherCheckpoints(epochNumber) {
-        const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
-        if (checkpoints.length === 0) {
-            throw new EmptyEpochError(epochNumber);
-        }
-        return checkpoints;
-    }
-    async gatherTxs(epochNumber, checkpoints) {
-        const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
-        const txProvider = this.p2pClient.getTxProvider();
-        const blocks = checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
-        const txsByBlock = await Promise.all(blocks.map((block)=>txProvider.getTxsForBlock(block, {
-                deadline
-            })));
-        const txs = txsByBlock.map(({ txs })=>txs).flat();
-        const missingTxs = txsByBlock.map(({ missingTxs })=>missingTxs).flat();
-        if (missingTxs.length === 0) {
-            this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, {
-                epochNumber
-            });
-            return txs;
+    /**
+   * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
+   * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
+   * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
+   */ async computeStartingCheckpoint(lastFullyProvenEpoch) {
+        if (lastFullyProvenEpoch === undefined) {
+            return CheckpointNumber.ZERO;
         }
-        throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map((hash)=>hash.toString()).join(', ')}`);
-    }
-    async gatherMessages(epochNumber, checkpoints) {
-        const messages = await Promise.all(checkpoints.map((c)=>this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
-        const messageCount = sum(messages.map((m)=>m.length));
-        this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, {
-            epochNumber
+        const checkpoints = await this.l2BlockSource.getCheckpointsData({
+            epoch: lastFullyProvenEpoch
         });
-        const messagesByCheckpoint = {};
-        for(let i = 0; i < checkpoints.length; i++){
-            messagesByCheckpoint[checkpoints[i].number] = messages[i];
-        }
-        return messagesByCheckpoint;
-    }
-    async gatherPreviousBlockHeader(epochNumber, previousBlockNumber) {
-        const header = await (previousBlockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
-        if (!header) {
-            throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
-        }
-        this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
-        return header;
-    }
-    /** Extracted for testing purposes. */ doCreateEpochProvingJob(data, deadline, publicProcessorFactory, publisher, opts = {}) {
-        const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
-        return new EpochProvingJob(data, this.worldState, this.prover.createEpochProver(), publicProcessorFactory, publisher, this.l2BlockSource, this.jobMetrics, deadline, {
-            parallelBlockLimit,
-            skipSubmitProof: proverNodeDisableProofPublish,
-            ...opts
-        }, this.log.getBindings());
+        return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
     }
-    /** Extracted for testing purposes. */ async triggerMonitors() {
-        await this.epochsMonitor.work();
+    async gatherPreviousBlockHeader(previousBlockNumber) {
+        const data = await this.l2BlockSource.getBlockData({
+            number: BlockNumber(previousBlockNumber)
+        });
+        if (!data?.header) {
+            throw new Error(`Previous block header ${previousBlockNumber} not found`);
+        }
+        return data.header;
     }
     validateConfig() {
         if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
@@ -707,9 +1028,5 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
         }
     }
 }
-class EmptyEpochError extends Error {
-    constructor(epochNumber){
-        super(`No blocks found for epoch ${epochNumber}`);
-        this.name = 'EmptyEpochError';
-    }
-}
+// Re-export so handlers can compare states externally.
+export { EpochProvingJobTerminalState };