@aztec/prover-node 0.0.1-commit.3100065 → 0.0.1-commit.330febf

package/src/session-manager.ts

@@ -60,8 +60,9 @@ export type SessionManagerDeps = {
   dateProvider: DateProvider;
   config: SessionManagerConfig;
   /**
-   * Optional callback fired when a session terminates with `failed`. The session manager
-   * doesn't own the failure-upload action; it just notifies the owner.
+   * Fired once when a full session ends in its own genuine failure (`EpochSession.hasFailed()` — top-tree
+   * or submit failed with every prover healthy). The owner uploads a post-mortem here. Not fired for a
+   * `stopped` session (a prover under it failed — possibly a prune), which is recovered on re-add instead.
    */
   onSessionFailed?: (session: EpochSession) => Promise<void>;
   bindings?: LoggerBindings;
@@ -88,15 +89,6 @@ export class SessionManager {
   private readonly reconcileQueue = new SerialQueue();
   /** Cached L1 constants, populated on first read. */
   private cachedL1Constants: L1RollupConstants | undefined;
-  /**
-   * Highest epoch for which the periodic tick has successfully created a full session.
-   * Monotonic high-water mark: once the tick observes a session for epoch X, it stops
-   * trying to open one — even if that session subsequently fails (only a new checkpoint
-   * event reopens it). Crucially, the mark only advances when a session actually exists
-   * post-open, so transient blockers (atMaxSessionLimit, archiver still indexing) leave
-   * the mark in place and the next tick retries.
-   */
-  private lastTickEpoch: EpochNumber | undefined;
   /** Test-only hooks applied to every session this manager constructs. */
   private sessionHooks: EpochSessionHooks | undefined;
   /** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */
@@ -227,7 +219,9 @@ export class SessionManager {
     await this.epochTicker?.stop();
     await this.reconcileQueue.cancel();
     const sessions = this.allSessions();
-    await Promise.allSettled(sessions.map(s => s.cancel('prover-node stopping')));
+    // A clean shutdown is just a restart, so preserve the in-flight broker jobs (abortJobs: false)
+    // for the restarted node to reuse rather than re-proving the epoch from scratch.
+    await Promise.allSettled(sessions.map(s => s.cancel('prover-node stopping', { abortJobs: false })));
   }
 
   // ---------------- reconcile ----------------
@@ -246,17 +240,6 @@ export class SessionManager {
       await this.openFullSessionIfReady(epoch);
     }
 
-    // Advance the tick high-water mark only once a session actually exists for the epoch.
-    // `openFullSessionIfReady` can early-return without creating one (atMaxSessionLimit,
-    // archiver still indexing, etc.); in those cases we want the next tick to try again
-    // rather than skip the epoch forever.
-    if (trigger.kind === 'tick' && implicatedEpochs.length === 1) {
-      const epoch = implicatedEpochs[0];
-      if (this.fullSessions.has(epoch)) {
-        this.lastTickEpoch = epoch;
-      }
-    }
-
     if (trigger.kind === 'start-proof') {
       this.openPartialSession(trigger.spec);
     }
@@ -264,15 +247,30 @@ export class SessionManager {
 
   private recreateInvalidSessions(): void {
     for (const [key, session] of Array.from(this.fullSessions.entries())) {
+      const canonical = this.checkpointsForSpec(session.getSpec());
+      const contentChanged = !this.checkpointsMatch(session.getCheckpoints(), canonical);
+
       if (session.isTerminal()) {
+        // A full session that failed on its own account is retained as a "do not re-prove" marker while
+        // its content is unchanged — this is what stops the tick re-proving a deterministically-failing
+        // epoch. When the content changes (a re-add), it is replaced so the epoch retries over the new
+        // provers. Any other terminal full session is simply dropped.
+        if (session.hasFailed() && !contentChanged) {
+          continue;
+        }
         this.fullSessions.delete(key);
+        if (contentChanged && this.canBuildOver(canonical)) {
+          const newSession = this.constructSession(session.getSpec(), canonical);
+          this.fullSessions.set(key, newSession);
+          void this.runSession(newSession);
+        }
         continue;
       }
-      const canonical = this.checkpointsForSpec(session.getSpec());
-      if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
+
+      if (contentChanged) {
         this.fireAndForgetCancel(session, 'canonical content changed');
         this.fullSessions.delete(key);
-        if (canonical.length > 0) {
+        if (this.canBuildOver(canonical)) {
           const newSession = this.constructSession(session.getSpec(), canonical);
           this.fullSessions.set(key, newSession);
           void this.runSession(newSession);
@@ -288,7 +286,7 @@ export class SessionManager {
       if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
         this.fireAndForgetCancel(session, 'canonical content changed');
         this.partialSessions.delete(key);
-        if (canonical.length > 0) {
+        if (this.canBuildOver(canonical)) {
           const newSession = this.constructSession(session.getSpec(), canonical);
           this.partialSessions.set(key, newSession);
           void this.runSession(newSession);
@@ -297,9 +295,15 @@ export class SessionManager {
     }
   }
 
+  /** A session may be built over a checkpoint set only when it is non-empty and contains no failed prover. */
+  private canBuildOver(canonical: readonly CheckpointProver[]): boolean {
+    return canonical.length > 0 && !this.hasFailedProver(canonical);
+  }
+
   private async openFullSessionIfReady(epoch: EpochNumber): Promise<void> {
-    // `recreateInvalidSessions` runs at the top of every reconcile and deletes terminal sessions
-    // before this is called, so a session present here is live and already covers the epoch.
+    // A session present here already covers the epoch: either live, or a retained genuinely-failed
+    // session kept by `recreateInvalidSessions` as a "do not re-prove" marker. Either way, don't open
+    // another — the retained-failed one is replaced only when its canonical content changes.
     if (this.fullSessions.has(epoch)) {
       return;
     }
@@ -324,6 +328,13 @@ export class SessionManager {
       });
       return;
     }
+    if (this.hasFailedProver(canonical)) {
+      // A checkpoint prover in the set has failed (a sub-tree fault or a prune-induced fork fault), so a
+      // session over it would fail immediately. Don't re-create it every tick — it recovers when a
+      // prune/re-add replaces the failed prover with a fresh one, or fails for good at expiry.
+      this.log.debug(`Skipping full-session open for epoch ${epoch}: a checkpoint prover has failed`, { epoch });
+      return;
+    }
     const spec: SessionSpec = { kind: 'full', epochNumber: epoch, fromSlot, toSlot };
     const session = this.constructSession(spec, canonical);
     this.fullSessions.set(epoch, session);
@@ -332,7 +343,7 @@ export class SessionManager {
 
   private openPartialSession(spec: SessionSpec): void {
     const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
-    if (canonical.length === 0) {
+    if (canonical.length === 0 || this.hasFailedProver(canonical)) {
       return;
     }
     // Reuse a live partial session for this epoch whose checkpoint set already matches the
@@ -404,43 +415,43 @@ export class SessionManager {
     }
     const state = await session.start();
     this.log.info(`Session ${session.getId()} exited with state ${state}`);
-    if (state === 'failed' && this.deps.onSessionFailed) {
-      // Best-effort suppression of the spurious post-mortem upload a prune produces: if the session's
-      // checkpoints no longer match the store's current set, the failure was caused by the content
-      // changing under it, not a genuine proving fault, so skip the upload. This is inherently racy —
-      // the store lags the world-state unwind, so a fault observed before the prune is reconciled here
-      // still uploads. The epoch is recovered regardless by recreating the session on re-add.
-      if (!this.checkpointsMatch(session.getCheckpoints(), this.checkpointsForSpec(session.getSpec()))) {
-        this.log.info(`Skipping failure upload for session ${session.getId()}: canonical content changed`, {
-          ...session.getSpec(),
-        });
-        return;
-      }
+
+    // A full session that failed on its own account (top-tree/submit failed with every prover healthy)
+    // is a genuine, race-free failure: upload its post-mortem once. `recreateInvalidSessions` retains
+    // the terminal session so this fires exactly once (it is never re-run over the same content). A
+    // `stopped` session (a prover under it failed) is not uploaded — it may be a prune, and recovers on
+    // re-add.
+    if (session.getKind() === 'full' && session.hasFailed() && this.deps.onSessionFailed) {
       try {
         await this.deps.onSessionFailed(session);
       } catch (err) {
-        this.log.error(`Error in onSessionFailed callback for ${session.getSpec().epochNumber}`, err);
+        this.log.error(`Error in onSessionFailed callback for epoch ${session.getEpochNumber()}`, err);
       }
     }
   }
 
   /**
-   * Builds the EpochProvingJobData snapshot for failure upload. Includes every checkpoint
-   * referenced by the session, regardless of whether sub-tree proving completed —
-   * partial state is still useful for post-mortem analysis.
+   * Builds the EpochProvingJobData snapshot for a post-mortem failure upload from a set of
+   * checkpoint provers. Includes every checkpoint regardless of whether sub-tree proving
+   * completed — partial state is still useful for post-mortem analysis.
    */
-  public static buildSessionProvingData(session: EpochSession): EpochProvingJobData {
-    const checkpoints = session.getCheckpoints();
+  public static async buildProvingData(checkpoints: readonly CheckpointProver[]): Promise<EpochProvingJobData> {
+    if (checkpoints.length === 0) {
+      throw new Error('Cannot build proving data from an empty checkpoint set');
+    }
+    // Provers no longer cache their txs; re-fetch each checkpoint's txs from the tx pool (concurrently)
+    // for the snapshot. The pool retains them past the proving window (A-1274), so this is durable.
+    const perCheckpoint = await Promise.all(checkpoints.map(async c => ({ c, txs: await c.getTxsForUpload() })));
     const txs = new Map();
     const l1ToL2Messages: Record<number, Fr[]> = {};
-    for (const c of checkpoints) {
-      for (const [hash, tx] of c.txs) {
+    for (const { c, txs: checkpointTxs } of perCheckpoint) {
+      for (const [hash, tx] of checkpointTxs) {
         txs.set(hash, tx);
       }
       l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
     }
     return {
-      epochNumber: session.getSpec().epochNumber,
+      epochNumber: checkpoints[0].epochNumber,
       checkpoints: checkpoints.map(c => c.checkpoint),
       txs,
       l1ToL2Messages,
@@ -463,20 +474,12 @@ export class SessionManager {
   /**
    * Maps a reconcile trigger to the epochs whose full session should be (re)opened.
    *
-   * This is where the "don't retry a genuinely-failed epoch, but do recover a pruned one" invariant
-   * lives — enforced by which triggers are gated by `lastTickEpoch`:
-   *
-   * - The periodic `tick` IS gated: once a tick has opened a session for an epoch, `lastTickEpoch`
-   *   advances to it and later ticks skip it (`epoch <= lastTickEpoch`). So a failed attempt is never
-   *   resubmitted on a loop by the tick.
-   * - `checkpoint` and `prune` are deliberately NOT gated. They only fire when the epoch's canonical
-   *   content actually changes — a checkpoint arrives, or a reorg prunes/replaces one — which is
-   *   exactly when re-attempting is correct.
-   *
-   * A genuine proving failure produces no content change, hence no checkpoint/prune event, so only
-   * the gated tick could reopen it — and it won't. A prune + re-add fires ungated events, so the
-   * epoch is reopened through this path (and `openFullSessionIfReady` rebuilds over the fresh
-   * provers). See the "onTick does not retry ... but recovers ... re-added" test.
+   * The periodic `tick` returns the next unproven epoch every time; it does not track prior attempts.
+   * `openFullSessionIfReady` is what keeps this from re-proving a doomed epoch: it refuses to build a
+   * session when any checkpoint prover in the set has failed, so a stuck epoch is cheaply skipped each
+   * tick rather than re-proved. `checkpoint` and `prune` fire when an epoch's canonical content changes
+   * (a checkpoint arrives, or a reorg prunes/replaces one) — which is what installs a fresh prover in
+   * place of a failed one, letting the next open succeed and recovering a pruned-then-re-added epoch.
    */
   private async epochsForTrigger(trigger: ReconcileTrigger): Promise<EpochNumber[]> {
     switch (trigger.kind) {
@@ -486,10 +489,7 @@ export class SessionManager {
         return trigger.affectedEpochs;
       case 'tick': {
         const epoch = await this.nextUnprovenEpoch();
-        if (epoch === undefined || (this.lastTickEpoch !== undefined && epoch <= this.lastTickEpoch)) {
-          return [];
-        }
-        return [epoch];
+        return epoch === undefined ? [] : [epoch];
       }
       case 'start-proof':
         return [];
@@ -516,6 +516,15 @@ export class SessionManager {
     return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
   }
 
+  /**
+   * True if any prover in the set has failed. The epoch cannot be proven over a failed prover (it can
+   * never produce its block proofs), so a session must not be built or rebuilt over it until a prune/re-add
+   * has replaced it with a fresh prover.
+   */
+  private hasFailedProver(checkpoints: readonly CheckpointProver[]): boolean {
+    return checkpoints.some(c => c.isFailed());
+  }
+
   private fireAndForgetCancel(session: EpochSession, reason: string): void {
     void session.cancel(reason).catch(err => this.log.warn(`Error cancelling session ${session.getId()}`, err));
   }