@aztec/prover-node 5.1.0 → 5.2.0-nightly.20260724

package/src/job/checkpoint-prover.ts

@@ -38,9 +38,25 @@ export type CheckpointProverDeps = {
   txGatheringTimeoutMs: number;
   /** Public processor deadline. */
   deadline: Date | undefined;
+  /**
+   * Fired once when the prover's block proofs reject for a genuine (non-cancel) reason — a sub-tree
+   * fault or a prune-induced fork fault. Useful for performing post mortem on failures.
+   */
+  onFailed?: (prover: CheckpointProver) => void;
+  /**
+   * Test-only hook: if set, invoked at the start of checkpoint execution instead of proving. Lets e2e
+   * tests force a sub-tree failure (it should throw) to exercise the checkpoint failure/upload path.
+   */
+  checkpointProveOverride?: () => Promise<never>;
   log: Logger;
 };
 
+/** Test-only hooks the store injects into every `CheckpointProver` it constructs. */
+export type CheckpointProverTestHooks = {
+  /** If set, invoked at the start of checkpoint execution instead of proving; should throw to fail. */
+  checkpointProveOverride?: () => Promise<never>;
+};
+
 /** Inputs that fully describe a checkpoint at register time. */
 export type CheckpointProverArgs = {
   checkpoint: Checkpoint;
@@ -89,9 +105,18 @@ export class CheckpointProver {
   /** Resolved by the sub-tree on success, rejected on cancel/failure. */
   private readonly blockProofs: PromiseWithResolvers<SubTreeResult['blockProofOutputs']> = promiseWithResolvers();
 
+  // Three independent lifecycle facts — deliberately not collapsed into one status enum, because several
+  // combinations are legal and relied on: a prover can be `completed` and then `cancelled` (routine
+  // teardown of an already-proven checkpoint), or `completed` and then `failed` (block proving was
+  // enqueued, but the sub-tree subsequently faulted). Only `failed` + `cancelled` is excluded — a cancel
+  // is not a failure (enforced in `failBlockProofs`).
+  /** Block-level proving was fully *enqueued* (a progress marker; the sub-tree may still be proving). */
+  private completed = false;
+  /** Block proofs rejected for a genuine (non-cancel) reason — a sub-tree or prune-induced fork fault. */
+  private failed = false;
+  /** The prover was torn down (prune / reap / shutdown). */
   private cancelled = false;
   private subTree?: CheckpointSubTreeOrchestrator;
-  private completed = false;
   private readonly abortController = new AbortController();
 
   /** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */
@@ -139,9 +164,14 @@ export class CheckpointProver {
     return this.cancelled;
   }
 
-  /** True once block-level proving has been fully *enqueued* (sub-tree completion may still be pending). */
-  public isCompleted(): boolean {
-    return this.completed;
+  /**
+   * True once this prover's block proofs have rejected for a genuine (non-cancel) reason — a sub-tree
+   * proving fault or a prune-induced world-state fork fault. A failed prover cannot produce its block
+   * proofs, so the reconciler must not build (or rebuild) an EpochSession over it; it is cleared only by
+   * a prune/re-add replacing it with a fresh prover, or by expiry reaping it.
+   */
+  public isFailed(): boolean {
+    return this.failed;
   }
 
   /** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */
@@ -179,10 +209,29 @@ export class CheckpointProver {
       this.deps.log.error(`Error in CheckpointProver ${this.id}`, err, {
         checkpointNumber: this.checkpoint.number,
       });
-      this.blockProofs.reject(err instanceof Error ? err : new Error(String(err)));
+      this.failBlockProofs(err instanceof Error ? err : new Error(String(err)));
     }
   }
 
+  /**
+   * Rejects the block-proof promise and, unless this is a cancellation, records the prover as failed so
+   * the reconciler won't build an EpochSession over it. First rejection wins, so a later duplicate reject
+   * (e.g. the executeCheckpoint `finally`) is a harmless no-op.
+   */
+  private failBlockProofs(err: Error): void {
+    if (!this.cancelled && !this.failed) {
+      this.failed = true;
+      // Notify the owner so it can upload a post-mortem for this checkpoint. Fire-and-forget: the
+      // callback must not block the prover's teardown, and a throw in it must not mask the rejection.
+      try {
+        this.deps.onFailed?.(this);
+      } catch (err) {
+        this.deps.log.error(`Error in CheckpointProver onFailed callback for ${this.id}`, err);
+      }
+    }
+    this.blockProofs.reject(err);
+  }
+
   private async gatherTxs(): Promise<Map<string, Tx>> {
     const deadline = new Date(this.deps.dateProvider.now() + this.deps.txGatheringTimeoutMs);
     const txsByBlock = await Promise.all(
@@ -205,6 +254,11 @@ export class CheckpointProver {
     let subTreeStarted = false;
 
     try {
+      // Test hook: force a sub-tree failure to exercise the checkpoint failure/upload path.
+      if (this.deps.checkpointProveOverride) {
+        await this.deps.checkpointProveOverride();
+      }
+
       for (const [hash, tx] of txs) {
         this.txs.set(hash, tx);
       }
@@ -248,7 +302,7 @@ export class CheckpointProver {
           this.deps.metrics.recordCheckpointProving(checkpointTimer.ms());
           this.blockProofs.resolve(result.blockProofOutputs);
         },
-        err => this.blockProofs.reject(err),
+        err => this.failBlockProofs(err instanceof Error ? err : new Error(String(err))),
       );
       if (signal.aborted) {
         return;
@@ -328,7 +382,7 @@ export class CheckpointProver {
         if (subTreeStarted) {
           await this.teardownSubTree();
         }
-        this.blockProofs.reject(new Error(`Checkpoint ${this.id} did not complete block processing`));
+        this.failBlockProofs(new Error(`Checkpoint ${this.id} did not complete block processing`));
       }
     }
   }

package/src/job/epoch-session.ts

@@ -95,7 +95,11 @@ export type EpochSessionDeps = {
  *   initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
  *
  * Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
- * `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
+ * `superseded`, `expired` → `timed-out`, `withdrawn` → `cancelled`. A fault ends the attempt in one
+ * of two terminal states depending on its cause: `stopped` if a checkpoint prover under it failed
+ * (possibly a prune — the reconciler will rebuild over a fresh prover on re-add), or `failed` if the
+ * session's own top-tree/submit work failed while every prover was healthy (a genuine, non-prune
+ * failure the reconciler retains and uploads — see `hasFailed()`).
  * Additionally, the session-level deadline fires `cancel('deadline')` and transitions
  * to `timed-out` for the pre-submit window (top-tree proving) — the publishing service
  * handles the post-submit window via the candidate's `deadline`.
@@ -187,6 +191,17 @@ export class EpochSession implements Traceable {
     return EpochProvingJobTerminalState.includes(this.state);
   }
 
+  /**
+   * True if the session ended in its own genuine failure — top-tree proving or L1 submission failed
+   * while every checkpoint prover succeeded. Because healthy provers rule out a prune-induced fault,
+   * this is a race-free "the epoch could not be proven" signal: the reconciler retains such a (full)
+   * session rather than re-proving it, and uploads a post-mortem. A `stopped` session (a checkpoint
+   * prover failed under it) is NOT a session failure in this sense.
+   */
+  public hasFailed(): boolean {
+    return this.state === 'failed';
+  }
+
   /** First block this session proves. */
   public getStartBlockNumber(): BlockNumber {
     return BlockNumber(this.checkpoints[0].checkpoint.blocks[0].number);
@@ -213,8 +228,17 @@ export class EpochSession implements Traceable {
         uuid: this.uuid,
         ...this.spec,
       });
+      // Distinguish the two ways an attempt can fault:
+      //  - a checkpoint prover in the set has failed OR was cancelled (a sub-tree fault, a prune-induced
+      //    fork fault, or a control-plane cancel that reached this catch before the reconcile marked the
+      //    session 'cancelled'): end in the non-declaring terminal 'stopped'. This is not the session's own
+      //    failure and may be a prune, so the reconciler does not upload it; a re-add installs a fresh prover.
+      //  - no prover failed or was cancelled, yet top-tree proving or L1 submission failed: this is the
+      //    session's own, genuine failure — and, because every prover is healthy and un-cancelled, it is
+      //    definitively NOT a prune. End in terminal 'failed' so the reconciler retains it (no pointless
+      //    re-prove) and uploads a race-free post-mortem.
       if (!this.isTerminal()) {
-        this.state = 'failed';
+        this.state = this.checkpoints.some(c => c.isFailed() || c.isCancelled()) ? 'stopped' : 'failed';
       }
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);

package/src/prover-node.ts

@@ -47,7 +47,8 @@ import {
 import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
 import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
 import type { SpecificProverNodeConfig } from './config.js';
-import type { EpochSession, EpochSessionHooks } from './job/epoch-session.js';
+import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
+import type { EpochSessionHooks } from './job/epoch-session.js';
 import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import { ProofPublishingService } from './proof-publishing-service.js';
 import type { ProverPublisherFactory } from './prover-publisher-factory.js';
@@ -168,6 +169,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         metrics: this.jobMetrics,
         txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
         deadline: undefined,
+        // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
+        // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
+        onFailed: prover => void this.tryUploadCheckpointFailure(prover),
       },
       this.log.getBindings(),
     );
@@ -239,11 +243,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         break;
       }
     }
-    // Expiry is driven by the archiver's latest synced L2 slot
-    await this.checkEpochExpiry();
-    // Advance the local tips store only after the proving-side handling has succeeded. Any
-    // failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
-    // skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
+    // Advance the local tips store only after the proving-side handling (registration / prune) has
+    // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
+    // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
     await this.tipsStore.handleBlockStreamEvent(event);
   }
 
@@ -458,8 +460,8 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
   }
 
   /**
-   * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
-   * reaps every CheckpointProver in the store whose epoch number matches.
+   * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
+   * CheckpointProver in the store whose epoch is at or below it.
    */
   private async expireEpoch(epoch: EpochNumber): Promise<void> {
     try {
@@ -515,8 +517,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     });
     this.blockStream.start();
 
-    // With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
-    // from a periodic tick so epochs still expire during idle/no-event periods.
+    // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
+    // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
+    // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
     this.expiryTicker = new RunningPromise(
       () => this.checkEpochExpiry(),
       this.log,
@@ -559,9 +562,10 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
   }
 
   /**
-   * Constructs the session manager. Extracted so subclasses (test harness) can swap
-   * the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
-   * their proving data uploaded.
+   * Constructs the session manager. Extracted so subclasses (test harness) can swap the
+   * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
+   * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
+   * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
    */
   protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
     return new SessionManager({
@@ -578,7 +582,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
       },
       onSessionFailed: async session => {
-        await this.tryUploadSessionFailure(session);
+        await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
       },
       bindings: this.log.getBindings(),
     });
@@ -596,15 +600,32 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     this.sessionManager.setSessionHooks(hooks);
   }
 
-  /** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */
-  public async tryUploadSessionFailure(session: EpochSession): Promise<string | undefined> {
-    if (!this.config.proverNodeFailedEpochStore) {
+  /**
+   * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
+   * CheckpointProver constructed after this call. For the e2e harness only.
+   */
+  public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
+    this.checkpointStore.setTestHooks(hooks);
+  }
+
+  /**
+   * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
+   * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
+   * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
+   * store is configured or the checkpoint set is empty.
+   */
+  public async tryUploadEpochFailure(
+    id: string,
+    checkpoints: readonly CheckpointProver[],
+  ): Promise<string | undefined> {
+    if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
       return undefined;
     }
-    const data = SessionManager.buildSessionProvingData(session);
+    const data = SessionManager.buildProvingData(checkpoints);
     return await uploadEpochProofFailure(
       this.config.proverNodeFailedEpochStore,
-      session.getId(),
+      // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
+      id,
       data,
       this.l2BlockSource as Archiver,
       this.worldState,
@@ -613,8 +634,62 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     );
   }
 
+  /**
+   * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
+   * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
+   * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
+   * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
+   * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
+   */
+  public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
+    if (!this.config.proverNodeFailedEpochStore) {
+      return undefined;
+    }
+    try {
+      // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
+      // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
+      // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
+      // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
+      // the snapshot (full world-state + archiver) is expensive to produce and store.
+      if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
+        this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
+          checkpointNumber: prover.checkpoint.number,
+        });
+        return undefined;
+      }
+      const data = SessionManager.buildProvingData([prover]);
+      return await uploadEpochProofFailure(
+        this.config.proverNodeFailedEpochStore,
+        // The prover's content-addressed id; the epoch number is already in the upload path.
+        prover.id,
+        data,
+        this.l2BlockSource as Archiver,
+        this.worldState,
+        assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
+        this.log,
+      );
+    } catch (err) {
+      this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
+      return undefined;
+    }
+  }
+
   // ---------------- helpers ----------------
 
+  /**
+   * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
+   * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
+   * so this returns false. Protected for direct unit-test access.
+   */
+  protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
+    const lastBlock = checkpoint.blocks.at(-1);
+    if (!lastBlock) {
+      return false;
+    }
+    const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
+    return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
+  }
+
   @memoize
   private getL1Constants(): Promise<L1RollupConstants> {
     return this.l2BlockSource.getL1Constants();

package/src/session-manager.ts

@@ -60,8 +60,9 @@ export type SessionManagerDeps = {
   dateProvider: DateProvider;
   config: SessionManagerConfig;
   /**
-   * Optional callback fired when a session terminates with `failed`. The session manager
-   * doesn't own the failure-upload action; it just notifies the owner.
+   * Fired once when a full session ends in its own genuine failure (`EpochSession.hasFailed()` — top-tree
+   * or submit failed with every prover healthy). The owner uploads a post-mortem here. Not fired for a
+   * `stopped` session (a prover under it failed — possibly a prune), which is recovered on re-add instead.
    */
   onSessionFailed?: (session: EpochSession) => Promise<void>;
   bindings?: LoggerBindings;
@@ -88,15 +89,6 @@ export class SessionManager {
   private readonly reconcileQueue = new SerialQueue();
   /** Cached L1 constants, populated on first read. */
   private cachedL1Constants: L1RollupConstants | undefined;
-  /**
-   * Highest epoch for which the periodic tick has successfully created a full session.
-   * Monotonic high-water mark: once the tick observes a session for epoch X, it stops
-   * trying to open one — even if that session subsequently fails (only a new checkpoint
-   * event reopens it). Crucially, the mark only advances when a session actually exists
-   * post-open, so transient blockers (atMaxSessionLimit, archiver still indexing) leave
-   * the mark in place and the next tick retries.
-   */
-  private lastTickEpoch: EpochNumber | undefined;
   /** Test-only hooks applied to every session this manager constructs. */
   private sessionHooks: EpochSessionHooks | undefined;
   /** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */
@@ -248,17 +240,6 @@ export class SessionManager {
       await this.openFullSessionIfReady(epoch);
     }
 
-    // Advance the tick high-water mark only once a session actually exists for the epoch.
-    // `openFullSessionIfReady` can early-return without creating one (atMaxSessionLimit,
-    // archiver still indexing, etc.); in those cases we want the next tick to try again
-    // rather than skip the epoch forever.
-    if (trigger.kind === 'tick' && implicatedEpochs.length === 1) {
-      const epoch = implicatedEpochs[0];
-      if (this.fullSessions.has(epoch)) {
-        this.lastTickEpoch = epoch;
-      }
-    }
-
     if (trigger.kind === 'start-proof') {
       this.openPartialSession(trigger.spec);
     }
@@ -266,15 +247,30 @@ export class SessionManager {
 
   private recreateInvalidSessions(): void {
     for (const [key, session] of Array.from(this.fullSessions.entries())) {
+      const canonical = this.checkpointsForSpec(session.getSpec());
+      const contentChanged = !this.checkpointsMatch(session.getCheckpoints(), canonical);
+
       if (session.isTerminal()) {
+        // A full session that failed on its own account is retained as a "do not re-prove" marker while
+        // its content is unchanged — this is what stops the tick re-proving a deterministically-failing
+        // epoch. When the content changes (a re-add), it is replaced so the epoch retries over the new
+        // provers. Any other terminal full session is simply dropped.
+        if (session.hasFailed() && !contentChanged) {
+          continue;
+        }
         this.fullSessions.delete(key);
+        if (contentChanged && this.canBuildOver(canonical)) {
+          const newSession = this.constructSession(session.getSpec(), canonical);
+          this.fullSessions.set(key, newSession);
+          void this.runSession(newSession);
+        }
         continue;
       }
-      const canonical = this.checkpointsForSpec(session.getSpec());
-      if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
+
+      if (contentChanged) {
         this.fireAndForgetCancel(session, 'canonical content changed');
         this.fullSessions.delete(key);
-        if (canonical.length > 0) {
+        if (this.canBuildOver(canonical)) {
           const newSession = this.constructSession(session.getSpec(), canonical);
           this.fullSessions.set(key, newSession);
           void this.runSession(newSession);
@@ -290,7 +286,7 @@ export class SessionManager {
       if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
         this.fireAndForgetCancel(session, 'canonical content changed');
         this.partialSessions.delete(key);
-        if (canonical.length > 0) {
+        if (this.canBuildOver(canonical)) {
           const newSession = this.constructSession(session.getSpec(), canonical);
           this.partialSessions.set(key, newSession);
           void this.runSession(newSession);
@@ -299,9 +295,15 @@ export class SessionManager {
     }
   }
 
+  /** A session may be built over a checkpoint set only when it is non-empty and contains no failed prover. */
+  private canBuildOver(canonical: readonly CheckpointProver[]): boolean {
+    return canonical.length > 0 && !this.hasFailedProver(canonical);
+  }
+
   private async openFullSessionIfReady(epoch: EpochNumber): Promise<void> {
-    // `recreateInvalidSessions` runs at the top of every reconcile and deletes terminal sessions
-    // before this is called, so a session present here is live and already covers the epoch.
+    // A session present here already covers the epoch: either live, or a retained genuinely-failed
+    // session kept by `recreateInvalidSessions` as a "do not re-prove" marker. Either way, don't open
+    // another — the retained-failed one is replaced only when its canonical content changes.
     if (this.fullSessions.has(epoch)) {
       return;
     }
@@ -326,6 +328,13 @@ export class SessionManager {
       });
       return;
     }
+    if (this.hasFailedProver(canonical)) {
+      // A checkpoint prover in the set has failed (a sub-tree fault or a prune-induced fork fault), so a
+      // session over it would fail immediately. Don't re-create it every tick — it recovers when a
+      // prune/re-add replaces the failed prover with a fresh one, or fails for good at expiry.
+      this.log.debug(`Skipping full-session open for epoch ${epoch}: a checkpoint prover has failed`, { epoch });
+      return;
+    }
     const spec: SessionSpec = { kind: 'full', epochNumber: epoch, fromSlot, toSlot };
     const session = this.constructSession(spec, canonical);
     this.fullSessions.set(epoch, session);
@@ -334,7 +343,7 @@ export class SessionManager {
 
   private openPartialSession(spec: SessionSpec): void {
     const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
-    if (canonical.length === 0) {
+    if (canonical.length === 0 || this.hasFailedProver(canonical)) {
       return;
     }
     // Reuse a live partial session for this epoch whose checkpoint set already matches the
@@ -406,33 +415,30 @@ export class SessionManager {
     }
     const state = await session.start();
     this.log.info(`Session ${session.getId()} exited with state ${state}`);
-    if (state === 'failed' && this.deps.onSessionFailed) {
-      // Best-effort suppression of the spurious post-mortem upload a prune produces: if the session's
-      // checkpoints no longer match the store's current set, the failure was caused by the content
-      // changing under it, not a genuine proving fault, so skip the upload. This is inherently racy —
-      // the store lags the world-state unwind, so a fault observed before the prune is reconciled here
-      // still uploads. The epoch is recovered regardless by recreating the session on re-add.
-      if (!this.checkpointsMatch(session.getCheckpoints(), this.checkpointsForSpec(session.getSpec()))) {
-        this.log.info(`Skipping failure upload for session ${session.getId()}: canonical content changed`, {
-          ...session.getSpec(),
-        });
-        return;
-      }
+
+    // A full session that failed on its own account (top-tree/submit failed with every prover healthy)
+    // is a genuine, race-free failure: upload its post-mortem once. `recreateInvalidSessions` retains
+    // the terminal session so this fires exactly once (it is never re-run over the same content). A
+    // `stopped` session (a prover under it failed) is not uploaded — it may be a prune, and recovers on
+    // re-add.
+    if (session.getKind() === 'full' && session.hasFailed() && this.deps.onSessionFailed) {
       try {
         await this.deps.onSessionFailed(session);
       } catch (err) {
-        this.log.error(`Error in onSessionFailed callback for ${session.getSpec().epochNumber}`, err);
+        this.log.error(`Error in onSessionFailed callback for epoch ${session.getEpochNumber()}`, err);
       }
     }
   }
 
   /**
-   * Builds the EpochProvingJobData snapshot for failure upload. Includes every checkpoint
-   * referenced by the session, regardless of whether sub-tree proving completed —
-   * partial state is still useful for post-mortem analysis.
+   * Builds the EpochProvingJobData snapshot for a post-mortem failure upload from a set of
+   * checkpoint provers. Includes every checkpoint regardless of whether sub-tree proving
+   * completed — partial state is still useful for post-mortem analysis.
    */
-  public static buildSessionProvingData(session: EpochSession): EpochProvingJobData {
-    const checkpoints = session.getCheckpoints();
+  public static buildProvingData(checkpoints: readonly CheckpointProver[]): EpochProvingJobData {
+    if (checkpoints.length === 0) {
+      throw new Error('Cannot build proving data from an empty checkpoint set');
+    }
     const txs = new Map();
     const l1ToL2Messages: Record<number, Fr[]> = {};
     for (const c of checkpoints) {
@@ -442,7 +448,7 @@ export class SessionManager {
       l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
     }
     return {
-      epochNumber: session.getSpec().epochNumber,
+      epochNumber: checkpoints[0].epochNumber,
       checkpoints: checkpoints.map(c => c.checkpoint),
       txs,
       l1ToL2Messages,
@@ -465,20 +471,12 @@ export class SessionManager {
   /**
    * Maps a reconcile trigger to the epochs whose full session should be (re)opened.
    *
-   * This is where the "don't retry a genuinely-failed epoch, but do recover a pruned one" invariant
-   * lives — enforced by which triggers are gated by `lastTickEpoch`:
-   *
-   * - The periodic `tick` IS gated: once a tick has opened a session for an epoch, `lastTickEpoch`
-   *   advances to it and later ticks skip it (`epoch <= lastTickEpoch`). So a failed attempt is never
-   *   resubmitted on a loop by the tick.
-   * - `checkpoint` and `prune` are deliberately NOT gated. They only fire when the epoch's canonical
-   *   content actually changes — a checkpoint arrives, or a reorg prunes/replaces one — which is
-   *   exactly when re-attempting is correct.
-   *
-   * A genuine proving failure produces no content change, hence no checkpoint/prune event, so only
-   * the gated tick could reopen it — and it won't. A prune + re-add fires ungated events, so the
-   * epoch is reopened through this path (and `openFullSessionIfReady` rebuilds over the fresh
-   * provers). See the "onTick does not retry ... but recovers ... re-added" test.
+   * The periodic `tick` returns the next unproven epoch every time; it does not track prior attempts.
+   * `openFullSessionIfReady` is what keeps this from re-proving a doomed epoch: it refuses to build a
+   * session when any checkpoint prover in the set has failed, so a stuck epoch is cheaply skipped each
+   * tick rather than re-proved. `checkpoint` and `prune` fire when an epoch's canonical content changes
+   * (a checkpoint arrives, or a reorg prunes/replaces one) — which is what installs a fresh prover in
+   * place of a failed one, letting the next open succeed and recovering a pruned-then-re-added epoch.
    */
   private async epochsForTrigger(trigger: ReconcileTrigger): Promise<EpochNumber[]> {
     switch (trigger.kind) {
@@ -488,10 +486,7 @@ export class SessionManager {
         return trigger.affectedEpochs;
       case 'tick': {
         const epoch = await this.nextUnprovenEpoch();
-        if (epoch === undefined || (this.lastTickEpoch !== undefined && epoch <= this.lastTickEpoch)) {
-          return [];
-        }
-        return [epoch];
+        return epoch === undefined ? [] : [epoch];
       }
       case 'start-proof':
         return [];
@@ -518,6 +513,15 @@ export class SessionManager {
     return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
   }
 
+  /**
+   * True if any prover in the set has failed. The epoch cannot be proven over a failed prover (it can
+   * never produce its block proofs), so a session must not be built or rebuilt over it until a prune/re-add
+   * has replaced it with a fresh prover.
+   */
+  private hasFailedProver(checkpoints: readonly CheckpointProver[]): boolean {
+    return checkpoints.some(c => c.isFailed());
+  }
+
   private fireAndForgetCancel(session: EpochSession, reason: string): void {
     void session.cancel(reason).catch(err => this.log.warn(`Error cancelling session ${session.getId()}`, err));
   }