@aztec/prover-node 0.0.1-commit.3100065 → 0.0.1-commit.321f6a9

package/dest/session-manager.js

@@ -24,14 +24,6 @@ import { EpochSession, specKey } from './job/epoch-session.js';
    * freshly-constructed session.
    */ reconcileQueue;
     /** Cached L1 constants, populated on first read. */ cachedL1Constants;
-    /**
-   * Highest epoch for which the periodic tick has successfully created a full session.
-   * Monotonic high-water mark: once the tick observes a session for epoch X, it stops
-   * trying to open one — even if that session subsequently fails (only a new checkpoint
-   * event reopens it). Crucially, the mark only advances when a session actually exists
-   * post-open, so transient blockers (atMaxSessionLimit, archiver still indexing) leave
-   * the mark in place and the next tick retries.
-   */ lastTickEpoch;
     /** Test-only hooks applied to every session this manager constructs. */ sessionHooks;
     /** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */ epochTicker;
     constructor(deps){
@@ -150,7 +142,11 @@ import { EpochSession, specKey } from './job/epoch-session.js';
         await this.epochTicker?.stop();
         await this.reconcileQueue.cancel();
         const sessions = this.allSessions();
-        await Promise.allSettled(sessions.map((s)=>s.cancel('prover-node stopping')));
+        // A clean shutdown is just a restart, so preserve the in-flight broker jobs (abortJobs: false)
+        // for the restarted node to reuse rather than re-proving the epoch from scratch.
+        await Promise.allSettled(sessions.map((s)=>s.cancel('prover-node stopping', {
+                abortJobs: false
+            })));
     }
     // ---------------- reconcile ----------------
     scheduleReconcile(trigger) {
@@ -165,31 +161,34 @@ import { EpochSession, specKey } from './job/epoch-session.js';
         for (const epoch of implicatedEpochs){
             await this.openFullSessionIfReady(epoch);
         }
-        // Advance the tick high-water mark only once a session actually exists for the epoch.
-        // `openFullSessionIfReady` can early-return without creating one (atMaxSessionLimit,
-        // archiver still indexing, etc.); in those cases we want the next tick to try again
-        // rather than skip the epoch forever.
-        if (trigger.kind === 'tick' && implicatedEpochs.length === 1) {
-            const epoch = implicatedEpochs[0];
-            if (this.fullSessions.has(epoch)) {
-                this.lastTickEpoch = epoch;
-            }
-        }
         if (trigger.kind === 'start-proof') {
             this.openPartialSession(trigger.spec);
         }
     }
     recreateInvalidSessions() {
         for (const [key, session] of Array.from(this.fullSessions.entries())){
+            const canonical = this.checkpointsForSpec(session.getSpec());
+            const contentChanged = !this.checkpointsMatch(session.getCheckpoints(), canonical);
             if (session.isTerminal()) {
+                // A full session that failed on its own account is retained as a "do not re-prove" marker while
+                // its content is unchanged — this is what stops the tick re-proving a deterministically-failing
+                // epoch. When the content changes (a re-add), it is replaced so the epoch retries over the new
+                // provers. Any other terminal full session is simply dropped.
+                if (session.hasFailed() && !contentChanged) {
+                    continue;
+                }
                 this.fullSessions.delete(key);
+                if (contentChanged && this.canBuildOver(canonical)) {
+                    const newSession = this.constructSession(session.getSpec(), canonical);
+                    this.fullSessions.set(key, newSession);
+                    void this.runSession(newSession);
+                }
                 continue;
             }
-            const canonical = this.checkpointsForSpec(session.getSpec());
-            if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
+            if (contentChanged) {
                 this.fireAndForgetCancel(session, 'canonical content changed');
                 this.fullSessions.delete(key);
-                if (canonical.length > 0) {
+                if (this.canBuildOver(canonical)) {
                     const newSession = this.constructSession(session.getSpec(), canonical);
                     this.fullSessions.set(key, newSession);
                     void this.runSession(newSession);
@@ -205,7 +204,7 @@ import { EpochSession, specKey } from './job/epoch-session.js';
             if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
                 this.fireAndForgetCancel(session, 'canonical content changed');
                 this.partialSessions.delete(key);
-                if (canonical.length > 0) {
+                if (this.canBuildOver(canonical)) {
                     const newSession = this.constructSession(session.getSpec(), canonical);
                     this.partialSessions.set(key, newSession);
                     void this.runSession(newSession);
@@ -213,9 +212,13 @@ import { EpochSession, specKey } from './job/epoch-session.js';
             }
         }
     }
+    /** A session may be built over a checkpoint set only when it is non-empty and contains no failed prover. */ canBuildOver(canonical) {
+        return canonical.length > 0 && !this.hasFailedProver(canonical);
+    }
     async openFullSessionIfReady(epoch) {
-        // `recreateInvalidSessions` runs at the top of every reconcile and deletes terminal sessions
-        // before this is called, so a session present here is live and already covers the epoch.
+        // A session present here already covers the epoch: either live, or a retained genuinely-failed
+        // session kept by `recreateInvalidSessions` as a "do not re-prove" marker. Either way, don't open
+        // another — the retained-failed one is replaced only when its canonical content changes.
         if (this.fullSessions.has(epoch)) {
             return;
         }
@@ -242,6 +245,15 @@ import { EpochSession, specKey } from './job/epoch-session.js';
             });
             return;
         }
+        if (this.hasFailedProver(canonical)) {
+            // A checkpoint prover in the set has failed (a sub-tree fault or a prune-induced fork fault), so a
+            // session over it would fail immediately. Don't re-create it every tick — it recovers when a
+            // prune/re-add replaces the failed prover with a fresh one, or fails for good at expiry.
+            this.log.debug(`Skipping full-session open for epoch ${epoch}: a checkpoint prover has failed`, {
+                epoch
+            });
+            return;
+        }
         const spec = {
             kind: 'full',
             epochNumber: epoch,
@@ -254,7 +266,7 @@ import { EpochSession, specKey } from './job/epoch-session.js';
     }
     openPartialSession(spec) {
         const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
-        if (canonical.length === 0) {
+        if (canonical.length === 0 || this.hasFailedProver(canonical)) {
             return;
         }
         // Reuse a live partial session for this epoch whose checkpoint set already matches the
@@ -312,31 +324,27 @@ import { EpochSession, specKey } from './job/epoch-session.js';
         }
         const state = await session.start();
         this.log.info(`Session ${session.getId()} exited with state ${state}`);
-        if (state === 'failed' && this.deps.onSessionFailed) {
-            // Best-effort suppression of the spurious post-mortem upload a prune produces: if the session's
-            // checkpoints no longer match the store's current set, the failure was caused by the content
-            // changing under it, not a genuine proving fault, so skip the upload. This is inherently racy —
-            // the store lags the world-state unwind, so a fault observed before the prune is reconciled here
-            // still uploads. The epoch is recovered regardless by recreating the session on re-add.
-            if (!this.checkpointsMatch(session.getCheckpoints(), this.checkpointsForSpec(session.getSpec()))) {
-                this.log.info(`Skipping failure upload for session ${session.getId()}: canonical content changed`, {
-                    ...session.getSpec()
-                });
-                return;
-            }
+        // A full session that failed on its own account (top-tree/submit failed with every prover healthy)
+        // is a genuine, race-free failure: upload its post-mortem once. `recreateInvalidSessions` retains
+        // the terminal session so this fires exactly once (it is never re-run over the same content). A
+        // `stopped` session (a prover under it failed) is not uploaded — it may be a prune, and recovers on
+        // re-add.
+        if (session.getKind() === 'full' && session.hasFailed() && this.deps.onSessionFailed) {
             try {
                 await this.deps.onSessionFailed(session);
             } catch (err) {
-                this.log.error(`Error in onSessionFailed callback for ${session.getSpec().epochNumber}`, err);
+                this.log.error(`Error in onSessionFailed callback for epoch ${session.getEpochNumber()}`, err);
             }
         }
     }
     /**
-   * Builds the EpochProvingJobData snapshot for failure upload. Includes every checkpoint
-   * referenced by the session, regardless of whether sub-tree proving completed —
-   * partial state is still useful for post-mortem analysis.
-   */ static buildSessionProvingData(session) {
-        const checkpoints = session.getCheckpoints();
+   * Builds the EpochProvingJobData snapshot for a post-mortem failure upload from a set of
+   * checkpoint provers. Includes every checkpoint regardless of whether sub-tree proving
+   * completed — partial state is still useful for post-mortem analysis.
+   */ static buildProvingData(checkpoints) {
+        if (checkpoints.length === 0) {
+            throw new Error('Cannot build proving data from an empty checkpoint set');
+        }
         const txs = new Map();
         const l1ToL2Messages = {};
         for (const c of checkpoints){
@@ -346,7 +354,7 @@ import { EpochSession, specKey } from './job/epoch-session.js';
             l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
         }
         return {
-            epochNumber: session.getSpec().epochNumber,
+            epochNumber: checkpoints[0].epochNumber,
             checkpoints: checkpoints.map((c)=>c.checkpoint),
             txs,
             l1ToL2Messages,
@@ -366,20 +374,12 @@ import { EpochSession, specKey } from './job/epoch-session.js';
     /**
    * Maps a reconcile trigger to the epochs whose full session should be (re)opened.
    *
-   * This is where the "don't retry a genuinely-failed epoch, but do recover a pruned one" invariant
-   * lives — enforced by which triggers are gated by `lastTickEpoch`:
-   *
-   * - The periodic `tick` IS gated: once a tick has opened a session for an epoch, `lastTickEpoch`
-   *   advances to it and later ticks skip it (`epoch <= lastTickEpoch`). So a failed attempt is never
-   *   resubmitted on a loop by the tick.
-   * - `checkpoint` and `prune` are deliberately NOT gated. They only fire when the epoch's canonical
-   *   content actually changes — a checkpoint arrives, or a reorg prunes/replaces one — which is
-   *   exactly when re-attempting is correct.
-   *
-   * A genuine proving failure produces no content change, hence no checkpoint/prune event, so only
-   * the gated tick could reopen it — and it won't. A prune + re-add fires ungated events, so the
-   * epoch is reopened through this path (and `openFullSessionIfReady` rebuilds over the fresh
-   * provers). See the "onTick does not retry ... but recovers ... re-added" test.
+   * The periodic `tick` returns the next unproven epoch every time; it does not track prior attempts.
+   * `openFullSessionIfReady` is what keeps this from re-proving a doomed epoch: it refuses to build a
+   * session when any checkpoint prover in the set has failed, so a stuck epoch is cheaply skipped each
+   * tick rather than re-proved. `checkpoint` and `prune` fire when an epoch's canonical content changes
+   * (a checkpoint arrives, or a reorg prunes/replaces one) — which is what installs a fresh prover in
+   * place of a failed one, letting the next open succeed and recovering a pruned-then-re-added epoch.
    */ async epochsForTrigger(trigger) {
         switch(trigger.kind){
             case 'checkpoint':
@@ -391,10 +391,7 @@ import { EpochSession, specKey } from './job/epoch-session.js';
             case 'tick':
                 {
                     const epoch = await this.nextUnprovenEpoch();
-                    if (epoch === undefined || this.lastTickEpoch !== undefined && epoch <= this.lastTickEpoch) {
-                        return [];
-                    }
-                    return [
+                    return epoch === undefined ? [] : [
                         epoch
                     ];
                 }
@@ -423,6 +420,13 @@ import { EpochSession, specKey } from './job/epoch-session.js';
     checkpointsForSpec(spec) {
         return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
     }
+    /**
+   * True if any prover in the set has failed. The epoch cannot be proven over a failed prover (it can
+   * never produce its block proofs), so a session must not be built or rebuilt over it until a prune/re-add
+   * has replaced it with a fresh prover.
+   */ hasFailedProver(checkpoints) {
+        return checkpoints.some((c)=>c.isFailed());
+    }
     fireAndForgetCancel(session, reason) {
         void session.cancel(reason).catch((err)=>this.log.warn(`Error cancelling session ${session.getId()}`, err));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/prover-node",
-  "version": "0.0.1-commit.3100065",
+  "version": "0.0.1-commit.321f6a9",
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
@@ -56,28 +56,29 @@
     ]
   },
   "dependencies": {
-    "@aztec/archiver": "0.0.1-commit.3100065",
-    "@aztec/bb-prover": "0.0.1-commit.3100065",
-    "@aztec/blob-client": "0.0.1-commit.3100065",
-    "@aztec/blob-lib": "0.0.1-commit.3100065",
-    "@aztec/constants": "0.0.1-commit.3100065",
-    "@aztec/epoch-cache": "0.0.1-commit.3100065",
-    "@aztec/ethereum": "0.0.1-commit.3100065",
-    "@aztec/foundation": "0.0.1-commit.3100065",
-    "@aztec/kv-store": "0.0.1-commit.3100065",
-    "@aztec/l1-artifacts": "0.0.1-commit.3100065",
-    "@aztec/native": "0.0.1-commit.3100065",
-    "@aztec/node-keystore": "0.0.1-commit.3100065",
-    "@aztec/node-lib": "0.0.1-commit.3100065",
-    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.3100065",
-    "@aztec/p2p": "0.0.1-commit.3100065",
-    "@aztec/protocol-contracts": "0.0.1-commit.3100065",
-    "@aztec/prover-client": "0.0.1-commit.3100065",
-    "@aztec/sequencer-client": "0.0.1-commit.3100065",
-    "@aztec/simulator": "0.0.1-commit.3100065",
-    "@aztec/stdlib": "0.0.1-commit.3100065",
-    "@aztec/telemetry-client": "0.0.1-commit.3100065",
-    "@aztec/world-state": "0.0.1-commit.3100065",
+    "@aztec/archiver": "0.0.1-commit.321f6a9",
+    "@aztec/bb-prover": "0.0.1-commit.321f6a9",
+    "@aztec/bb.js": "0.0.1-commit.321f6a9",
+    "@aztec/blob-client": "0.0.1-commit.321f6a9",
+    "@aztec/blob-lib": "0.0.1-commit.321f6a9",
+    "@aztec/constants": "0.0.1-commit.321f6a9",
+    "@aztec/epoch-cache": "0.0.1-commit.321f6a9",
+    "@aztec/ethereum": "0.0.1-commit.321f6a9",
+    "@aztec/foundation": "0.0.1-commit.321f6a9",
+    "@aztec/kv-store": "0.0.1-commit.321f6a9",
+    "@aztec/l1-artifacts": "0.0.1-commit.321f6a9",
+    "@aztec/native": "0.0.1-commit.321f6a9",
+    "@aztec/node-keystore": "0.0.1-commit.321f6a9",
+    "@aztec/node-lib": "0.0.1-commit.321f6a9",
+    "@aztec/noir-protocol-circuits-types": "0.0.1-commit.321f6a9",
+    "@aztec/p2p": "0.0.1-commit.321f6a9",
+    "@aztec/protocol-contracts": "0.0.1-commit.321f6a9",
+    "@aztec/prover-client": "0.0.1-commit.321f6a9",
+    "@aztec/sequencer-client": "0.0.1-commit.321f6a9",
+    "@aztec/simulator": "0.0.1-commit.321f6a9",
+    "@aztec/stdlib": "0.0.1-commit.321f6a9",
+    "@aztec/telemetry-client": "0.0.1-commit.321f6a9",
+    "@aztec/world-state": "0.0.1-commit.321f6a9",
     "source-map-support": "^0.5.21",
     "tslib": "^2.4.0",
     "viem": "npm:@aztec/viem@2.38.2"

package/src/actions/rerun-epoch-proving-job.ts

@@ -7,7 +7,7 @@ import { type ProverClientConfig, createProverClient } from '@aztec/prover-clien
 import { ProverBrokerConfig, createAndStartProvingBroker } from '@aztec/prover-client/broker';
 import { getLastSiblingPath } from '@aztec/prover-client/helpers';
 import { ChonkCache } from '@aztec/prover-client/orchestrator';
-import { PublicProcessorFactory } from '@aztec/simulator/server';
+import { AvmSimulatorPool, PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2Block } from '@aztec/stdlib/block';
 import { getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
@@ -21,35 +21,30 @@ import { createWorldState } from '@aztec/world-state';
 import { readFileSync } from 'fs';
 
 import { CheckpointProver } from '../job/checkpoint-prover.js';
-import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
+import { type EpochProvingJobData, deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
 import { EpochSession, type SessionSpec } from '../job/epoch-session.js';
 import { ProverNodeJobMetrics } from '../metrics.js';
 
+type RerunConfig = DataStoreConfig &
+  ProverBrokerConfig &
+  ProverClientConfig &
+  Pick<L1ContractsConfig, 'aztecEpochDuration'>;
+
 /**
  * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
  * using the state snapshots, and creates a new epoch proving session to prove the downloaded proving job.
  * Proving is done with a local proving broker and agents as specified by the config.
  */
-export async function rerunEpochProvingJob(
-  localPath: string,
-  log: Logger,
-  config: DataStoreConfig & ProverBrokerConfig & ProverClientConfig & Pick<L1ContractsConfig, 'aztecEpochDuration'>,
-  genesis?: GenesisData,
-) {
-  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
-  log.info(`Loaded proving job data for epoch ${jobData.epochNumber}`);
+export async function rerunEpochProvingJob(localPath: string, log: Logger, config: RerunConfig, genesis?: GenesisData) {
+  await using ctx = await createRerunContext(localPath, log, config, genesis);
+  const { jobData, prover, metrics } = ctx;
 
-  const telemetry = getTelemetryClient();
-  const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
-  const worldState = await createWorldState(config, genesis);
-  const initialBlockHash = await worldState.getInitialHeader().hash();
-  const archiver = await createArchiverStore(config, initialBlockHash);
-  const publicProcessorFactory = new PublicProcessorFactory(
-    createContractDataSource(archiver),
-    undefined,
-    undefined,
-    log.getBindings(),
-  );
+  log.info(`Rerunning epoch proving for epoch ${jobData.epochNumber}`);
+
+  const provers: CheckpointProver[] = [];
+  for (let i = 0; i < jobData.checkpoints.length; i++) {
+    provers.push(await buildCheckpointProver(ctx, i, log));
+  }
 
   // Local rerun never publishes — stub the service so submit() always resolves 'published'
   // and withdraw is a no-op.
@@ -57,51 +52,6 @@ export async function rerunEpochProvingJob(
     submit: () => Promise.resolve('published' as const),
     withdraw: () => {},
   };
-  const broker = await createAndStartProvingBroker(config, telemetry);
-  const prover = await createProverClient(config, worldState, broker, telemetry);
-  const chonkCache = new ChonkCache(log.getBindings());
-
-  const txProvider = makeReplayingTxProvider(jobData.txs);
-
-  log.info(`Rerunning epoch proving for epoch ${jobData.epochNumber}`);
-
-  const provers: CheckpointProver[] = [];
-  for (let i = 0; i < jobData.checkpoints.length; i++) {
-    const checkpoint = jobData.checkpoints[i];
-    const previousBlockHeader =
-      i === 0 ? jobData.previousBlockHeader : jobData.checkpoints[i - 1].blocks.at(-1)!.header;
-    const l1ToL2Messages = jobData.l1ToL2Messages[checkpoint.number] ?? [];
-    const previousArchiveSiblingPath = await getLastSiblingPath(
-      MerkleTreeId.ARCHIVE,
-      worldState.getSnapshot(BlockNumber(checkpoint.blocks[0].number - 1)),
-    );
-    const attestations = checkpoint.number === jobData.checkpoints.at(-1)!.number ? jobData.attestations : [];
-    provers.push(
-      new CheckpointProver(
-        {
-          checkpoint,
-          epochNumber: jobData.epochNumber,
-          attestations,
-          previousBlockHeader,
-          l1ToL2Messages,
-          previousArchiveSiblingPath,
-        },
-        {
-          proverFactory: prover,
-          chonkCache,
-          publicProcessorFactory,
-          dbProvider: worldState,
-          txProvider,
-          dateProvider: new DateProvider(),
-          proverId: prover.getProverId(),
-          metrics,
-          txGatheringTimeoutMs: 120_000,
-          deadline: undefined,
-          log,
-        },
-      ),
-    );
-  }
 
   const l1Constants = { epochDuration: config.aztecEpochDuration };
   const [fromSlot, toSlot] = getSlotRangeForEpoch(jobData.epochNumber, l1Constants);
@@ -125,6 +75,129 @@ export async function rerunEpochProvingJob(
   return finalState;
 }
 
+/**
+ * Re-proves a single downloaded checkpoint proving job (as uploaded by a `CheckpointProver` failure).
+ * Reconstructs just that checkpoint's sub-tree prover from the snapshot and awaits its block proofs — no
+ * epoch top-tree, no L1 submission — so a checkpoint-level failure can be reproduced offline in isolation.
+ * Returns the block-rollup proof outputs on success; throws if the checkpoint fails to prove again.
+ */
+export async function rerunCheckpointProvingJob(
+  localPath: string,
+  log: Logger,
+  config: RerunConfig,
+  genesis?: GenesisData,
+) {
+  await using ctx = await createRerunContext(localPath, log, config, genesis);
+  const { jobData } = ctx;
+  const checkpointNumber = jobData.checkpoints[0].number;
+
+  log.info(`Rerunning checkpoint proving for checkpoint ${checkpointNumber} (epoch ${jobData.epochNumber})`);
+
+  const prover = await buildCheckpointProver(ctx, 0, log);
+  try {
+    const blockProofs = await prover.whenBlockProofsReady();
+    log.info(`Completed proving for checkpoint ${checkpointNumber} with ${blockProofs.length} block proof(s)`);
+    return blockProofs;
+  } finally {
+    prover.cancel({ routine: true });
+    await prover.whenDone();
+  }
+}
+
+/** Everything a rerun needs, reconstructed from the downloaded snapshot + job data. */
+type RerunContext = {
+  jobData: EpochProvingJobData;
+  metrics: ProverNodeJobMetrics;
+  worldState: Awaited<ReturnType<typeof createWorldState>>;
+  publicProcessorFactory: PublicProcessorFactory;
+  prover: Awaited<ReturnType<typeof createProverClient>>;
+  chonkCache: ChonkCache;
+  txProvider: ITxProvider;
+} & AsyncDisposable;
+
+/**
+ * Rebuilds the offline proving environment from a downloaded job: world state + archiver from the
+ * snapshots, a local proving broker + client, a chonk cache, and a tx provider that replays the job's txs.
+ */
+async function createRerunContext(
+  localPath: string,
+  log: Logger,
+  config: RerunConfig,
+  genesis?: GenesisData,
+): Promise<RerunContext> {
+  const jobData = deserializeEpochProvingJobData(readFileSync(localPath));
+  log.info(`Loaded proving job data for epoch ${jobData.epochNumber}`);
+
+  const telemetry = getTelemetryClient();
+  const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
+  const worldState = await createWorldState(config, genesis);
+  const initialBlockHash = await worldState.getInitialHeader().hash();
+  const archiver = await createArchiverStore(config, initialBlockHash);
+  const avmSimulator = await AvmSimulatorPool.spawn({ wsdbIpcPath: worldState.getIpcPath() });
+  const publicProcessorFactory = new PublicProcessorFactory(
+    createContractDataSource(archiver),
+    avmSimulator,
+    undefined,
+    undefined,
+    log.getBindings(),
+  );
+  const broker = await createAndStartProvingBroker(config, telemetry);
+  const prover = await createProverClient(config, worldState, broker, telemetry);
+  const chonkCache = new ChonkCache(log.getBindings());
+  const txProvider = makeReplayingTxProvider(jobData.txs);
+
+  return {
+    jobData,
+    metrics,
+    worldState,
+    publicProcessorFactory,
+    prover,
+    chonkCache,
+    txProvider,
+    async [Symbol.asyncDispose]() {
+      await avmSimulator[Symbol.asyncDispose]();
+      await worldState[Symbol.asyncDispose]();
+    },
+  };
+}
+
+/** Reconstructs the `CheckpointProver` for the checkpoint at `index` in the job, ready to prove. */
+async function buildCheckpointProver(ctx: RerunContext, index: number, log: Logger): Promise<CheckpointProver> {
+  const { jobData, worldState, prover, chonkCache, publicProcessorFactory, txProvider, metrics } = ctx;
+  const checkpoint = jobData.checkpoints[index];
+  const previousBlockHeader =
+    index === 0 ? jobData.previousBlockHeader : jobData.checkpoints[index - 1].blocks.at(-1)!.header;
+  const l1ToL2Messages = jobData.l1ToL2Messages[checkpoint.number] ?? [];
+  const previousArchiveSiblingPath = await getLastSiblingPath(
+    MerkleTreeId.ARCHIVE,
+    worldState.getSnapshot(BlockNumber(checkpoint.blocks[0].number - 1)),
+  );
+  const attestations = checkpoint.number === jobData.checkpoints.at(-1)!.number ? jobData.attestations : [];
+  return new CheckpointProver(
+    {
+      checkpoint,
+      epochNumber: jobData.epochNumber,
+      attestations,
+      previousBlockHeader,
+      l1ToL2Messages,
+      previousArchiveSiblingPath,
+    },
+    {
+      proverFactory: prover,
+      chonkCache,
+      publicProcessorFactory,
+      dbProvider: worldState,
+      txProvider,
+      dateProvider: new DateProvider(),
+      proverId: prover.getProverId(),
+      metrics,
+      txGatheringTimeoutMs: 120_000,
+      deadline: undefined,
+      log,
+    },
+  );
+}
+
 /** Build a synthetic ITxProvider that returns the supplied txs map by lookup. */
 function makeReplayingTxProvider(txs: Map<string, Tx>): ITxProvider {
   const lookup = (hashes: TxHash[]) => {

package/src/checkpoint-store.ts

@@ -4,7 +4,12 @@ import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
 import { type L1RollupConstants, getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
 
-import { CheckpointProver, type CheckpointProverArgs, type CheckpointProverDeps } from './job/checkpoint-prover.js';
+import {
+  CheckpointProver,
+  type CheckpointProverArgs,
+  type CheckpointProverDeps,
+  type CheckpointProverTestHooks,
+} from './job/checkpoint-prover.js';
 
 /** Register-time data needed to construct a `CheckpointProver` (everything except the checkpoint + epoch). */
 export type RegisterCheckpointData = Omit<CheckpointProverArgs, 'checkpoint' | 'epochNumber'>;
@@ -35,6 +40,8 @@ export class CheckpointStore {
    */
   private readonly pendingTeardowns = new Map<number, Promise<void>>();
   private nextTeardownId = 0;
+  /** Test-only hooks injected into every prover this store constructs. */
+  private testHooks: CheckpointProverTestHooks = {};
   private readonly log: Logger;
 
   constructor(
@@ -100,11 +107,22 @@ export class CheckpointStore {
       }
     }
 
-    const prover = this.proverFactoryFn({ ...data, checkpoint, epochNumber }, { ...this.proverDeps, log: this.log });
+    const prover = this.proverFactoryFn(
+      { ...data, checkpoint, epochNumber },
+      { ...this.proverDeps, checkpointProveOverride: this.testHooks.checkpointProveOverride, log: this.log },
+    );
     this.provers.set(id, prover);
     return prover;
   }
 
+  /**
+   * Installs test-only hooks applied to every prover constructed from now on. Used by the e2e harness to
+   * force a checkpoint sub-tree failure without monkey-patching the prover factory.
+   */
+  public setTestHooks(hooks: CheckpointProverTestHooks): void {
+    this.testHooks = hooks;
+  }
+
   /**
    * Cancels and removes every prover that holds a block above the prune target. A checkpoint is orphaned by a prune to
    * block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range straddles

package/src/config.ts

@@ -6,6 +6,7 @@ import {
   numberConfigHelper,
   pickConfigMappings,
 } from '@aztec/foundation/config';
+import { EthAddress } from '@aztec/foundation/eth-address';
 import { type KeyStoreConfig, keyStoreConfigMappings } from '@aztec/node-keystore/config';
 import { ethPrivateKeySchema } from '@aztec/node-keystore/schemas';
 import type { KeyStore } from '@aztec/node-keystore/types';
@@ -44,6 +45,7 @@ export type SpecificProverNodeConfig = {
   txGatheringIntervalMs: number;
   txGatheringBatchSize: number;
   txGatheringMaxParallelRequestsPerNode: number;
+  proofSubmissionTargetAddress?: EthAddress;
 };
 
 export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProverNodeConfig> = {
@@ -97,6 +99,14 @@ export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProver
     description: 'Whether the prover node skips publishing proofs to L1',
     ...booleanConfigHelper(false),
   },
+  proofSubmissionTargetAddress: {
+    env: 'PROVER_NODE_PROOF_SUBMISSION_TARGET_ADDRESS',
+    description:
+      'Optional L1 address the submitEpochRootProof tx is sent to. Must expose the identical submitEpochRootProof ABI ' +
+      'and forward to the rollup. Defaults to the rollup address.',
+    parseEnv: (val: string) => EthAddress.fromString(val),
+    defaultValue: undefined,
+  },
 };
 
 export const proverNodeConfigMappings: ConfigMappingsType<ProverNodeConfig> = {

package/src/factory.ts

@@ -19,6 +19,7 @@ import {
   type ProverTxSenderConfig,
   getPublisherConfigFromProverConfig,
 } from '@aztec/sequencer-client';
+import type { AvmSimulator } from '@aztec/simulator/server';
 import type {
   ITxProvider,
   ProverConfig,
@@ -47,6 +48,8 @@ export type ProverNodeDeps = {
   epochCache: EpochCacheInterface;
   blobClient: BlobClientInterface;
   keyStoreManager?: KeystoreManager;
+  /** AVM execution backend (simulator pool + CDB server) for public simulation. */
+  avmSimulator: AvmSimulator;
 };
 
 /** Creates a new prover node subsystem given a config and dependencies */
@@ -135,6 +138,7 @@ export async function createProverNode(
     deps.publisherFactory ??
     new ProverPublisherFactory(config, {
       rollupContract,
+      proofSubmissionTarget: config.proofSubmissionTargetAddress,
       publisherManager: new PublisherManager(l1TxUtils, getPublisherConfigFromProverConfig(config), {
         bindings: log.getBindings(),
         funder: funderL1TxUtils,
@@ -179,6 +183,7 @@ export async function createProverNode(
     p2pClient,
     rollupContract,
     l1Metrics,
+    deps.avmSimulator,
     proverNodeConfig,
     telemetry,
     delayer,