@aztec/prover-node 0.0.1-commit.a5db02d → 0.0.1-commit.aa0c64f

package/src/job/checkpoint-prover.ts

@@ -38,9 +38,25 @@ export type CheckpointProverDeps = {
   txGatheringTimeoutMs: number;
   /** Public processor deadline. */
   deadline: Date | undefined;
+  /**
+   * Fired once when the prover's block proofs reject for a genuine (non-cancel) reason — a sub-tree
+   * fault or a prune-induced fork fault. Useful for performing post mortem on failures.
+   */
+  onFailed?: (prover: CheckpointProver) => void;
+  /**
+   * Test-only hook: if set, invoked at the start of checkpoint execution instead of proving. Lets e2e
+   * tests force a sub-tree failure (it should throw) to exercise the checkpoint failure/upload path.
+   */
+  checkpointProveOverride?: () => Promise<never>;
   log: Logger;
 };
 
+/** Test-only hooks the store injects into every `CheckpointProver` it constructs. */
+export type CheckpointProverTestHooks = {
+  /** If set, invoked at the start of checkpoint execution instead of proving; should throw to fail. */
+  checkpointProveOverride?: () => Promise<never>;
+};
+
 /** Inputs that fully describe a checkpoint at register time. */
 export type CheckpointProverArgs = {
   checkpoint: Checkpoint;
@@ -89,9 +105,18 @@ export class CheckpointProver {
   /** Resolved by the sub-tree on success, rejected on cancel/failure. */
   private readonly blockProofs: PromiseWithResolvers<SubTreeResult['blockProofOutputs']> = promiseWithResolvers();
 
+  // Three independent lifecycle facts — deliberately not collapsed into one status enum, because several
+  // combinations are legal and relied on: a prover can be `completed` and then `cancelled` (routine
+  // teardown of an already-proven checkpoint), or `completed` and then `failed` (block proving was
+  // enqueued, but the sub-tree subsequently faulted). Only `failed` + `cancelled` is excluded — a cancel
+  // is not a failure (enforced in `failBlockProofs`).
+  /** Block-level proving was fully *enqueued* (a progress marker; the sub-tree may still be proving). */
+  private completed = false;
+  /** Block proofs rejected for a genuine (non-cancel) reason — a sub-tree or prune-induced fork fault. */
+  private failed = false;
+  /** The prover was torn down (prune / reap / shutdown). */
   private cancelled = false;
   private subTree?: CheckpointSubTreeOrchestrator;
-  private completed = false;
   private readonly abortController = new AbortController();
 
   /** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */
@@ -139,9 +164,14 @@ export class CheckpointProver {
     return this.cancelled;
   }
 
-  /** True once block-level proving has been fully *enqueued* (sub-tree completion may still be pending). */
-  public isCompleted(): boolean {
-    return this.completed;
+  /**
+   * True once this prover's block proofs have rejected for a genuine (non-cancel) reason — a sub-tree
+   * proving fault or a prune-induced world-state fork fault. A failed prover cannot produce its block
+   * proofs, so the reconciler must not build (or rebuild) an EpochSession over it; it is cleared only by
+   * a prune/re-add replacing it with a fresh prover, or by expiry reaping it.
+   */
+  public isFailed(): boolean {
+    return this.failed;
   }
 
   /** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */
@@ -179,10 +209,29 @@ export class CheckpointProver {
       this.deps.log.error(`Error in CheckpointProver ${this.id}`, err, {
         checkpointNumber: this.checkpoint.number,
       });
-      this.blockProofs.reject(err instanceof Error ? err : new Error(String(err)));
+      this.failBlockProofs(err instanceof Error ? err : new Error(String(err)));
     }
   }
 
+  /**
+   * Rejects the block-proof promise and, unless this is a cancellation, records the prover as failed so
+   * the reconciler won't build an EpochSession over it. First rejection wins, so a later duplicate reject
+   * (e.g. the executeCheckpoint `finally`) is a harmless no-op.
+   */
+  private failBlockProofs(err: Error): void {
+    if (!this.cancelled && !this.failed) {
+      this.failed = true;
+      // Notify the owner so it can upload a post-mortem for this checkpoint. Fire-and-forget: the
+      // callback must not block the prover's teardown, and a throw in it must not mask the rejection.
+      try {
+        this.deps.onFailed?.(this);
+      } catch (err) {
+        this.deps.log.error(`Error in CheckpointProver onFailed callback for ${this.id}`, err);
+      }
+    }
+    this.blockProofs.reject(err);
+  }
+
   private async gatherTxs(): Promise<Map<string, Tx>> {
     const deadline = new Date(this.deps.dateProvider.now() + this.deps.txGatheringTimeoutMs);
     const txsByBlock = await Promise.all(
@@ -205,6 +254,11 @@ export class CheckpointProver {
     let subTreeStarted = false;
 
     try {
+      // Test hook: force a sub-tree failure to exercise the checkpoint failure/upload path.
+      if (this.deps.checkpointProveOverride) {
+        await this.deps.checkpointProveOverride();
+      }
+
       for (const [hash, tx] of txs) {
         this.txs.set(hash, tx);
       }
@@ -248,7 +302,7 @@ export class CheckpointProver {
           this.deps.metrics.recordCheckpointProving(checkpointTimer.ms());
           this.blockProofs.resolve(result.blockProofOutputs);
         },
-        err => this.blockProofs.reject(err),
+        err => this.failBlockProofs(err instanceof Error ? err : new Error(String(err))),
       );
       if (signal.aborted) {
         return;
@@ -328,7 +382,7 @@ export class CheckpointProver {
         if (subTreeStarted) {
           await this.teardownSubTree();
         }
-        this.blockProofs.reject(new Error(`Checkpoint ${this.id} did not complete block processing`));
+        this.failBlockProofs(new Error(`Checkpoint ${this.id} did not complete block processing`));
       }
     }
   }

package/src/job/epoch-session.ts

@@ -95,7 +95,11 @@ export type EpochSessionDeps = {
  *   initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
  *
  * Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
- * `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
+ * `superseded`, `expired` → `timed-out`, `withdrawn` → `cancelled`. A fault ends the attempt in one
+ * of two terminal states depending on its cause: `stopped` if a checkpoint prover under it failed
+ * (possibly a prune — the reconciler will rebuild over a fresh prover on re-add), or `failed` if the
+ * session's own top-tree/submit work failed while every prover was healthy (a genuine, non-prune
+ * failure the reconciler retains and uploads — see `hasFailed()`).
  * Additionally, the session-level deadline fires `cancel('deadline')` and transitions
  * to `timed-out` for the pre-submit window (top-tree proving) — the publishing service
  * handles the post-submit window via the candidate's `deadline`.
@@ -187,6 +191,17 @@ export class EpochSession implements Traceable {
     return EpochProvingJobTerminalState.includes(this.state);
   }
 
+  /**
+   * True if the session ended in its own genuine failure — top-tree proving or L1 submission failed
+   * while every checkpoint prover succeeded. Because healthy provers rule out a prune-induced fault,
+   * this is a race-free "the epoch could not be proven" signal: the reconciler retains such a (full)
+   * session rather than re-proving it, and uploads a post-mortem. A `stopped` session (a checkpoint
+   * prover failed under it) is NOT a session failure in this sense.
+   */
+  public hasFailed(): boolean {
+    return this.state === 'failed';
+  }
+
   /** First block this session proves. */
   public getStartBlockNumber(): BlockNumber {
     return BlockNumber(this.checkpoints[0].checkpoint.blocks[0].number);
@@ -213,8 +228,17 @@ export class EpochSession implements Traceable {
         uuid: this.uuid,
         ...this.spec,
       });
+      // Distinguish the two ways an attempt can fault:
+      //  - a checkpoint prover in the set has failed OR was cancelled (a sub-tree fault, a prune-induced
+      //    fork fault, or a control-plane cancel that reached this catch before the reconcile marked the
+      //    session 'cancelled'): end in the non-declaring terminal 'stopped'. This is not the session's own
+      //    failure and may be a prune, so the reconciler does not upload it; a re-add installs a fresh prover.
+      //  - no prover failed or was cancelled, yet top-tree proving or L1 submission failed: this is the
+      //    session's own, genuine failure — and, because every prover is healthy and un-cancelled, it is
+      //    definitively NOT a prune. End in terminal 'failed' so the reconciler retains it (no pointless
+      //    re-prove) and uploads a race-free post-mortem.
       if (!this.isTerminal()) {
-        this.state = 'failed';
+        this.state = this.checkpoints.some(c => c.isFailed() || c.isCancelled()) ? 'stopped' : 'failed';
       }
     } finally {
       clearTimeout(this.deadlineTimeoutHandler);

package/src/prover-node-publisher.ts

@@ -40,6 +40,8 @@ export class ProverNodePublisher {
 
   protected rollupContract: RollupContract;
 
+  protected proofSubmissionTarget: Hex;
+
   public readonly l1TxUtils: L1TxUtils;
 
   constructor(
@@ -47,6 +49,7 @@ export class ProverNodePublisher {
     deps: {
       rollupContract: RollupContract;
       l1TxUtils: L1TxUtils;
+      proofSubmissionTarget?: EthAddress;
       telemetry?: TelemetryClient;
     },
     bindings?: LoggerBindings,
@@ -57,6 +60,7 @@ export class ProverNodePublisher {
     this.log = createLogger('prover-node:l1-tx-publisher', bindings);
 
     this.rollupContract = deps.rollupContract;
+    this.proofSubmissionTarget = deps.proofSubmissionTarget?.toString() ?? deps.rollupContract.address;
     this.l1TxUtils = deps.l1TxUtils;
   }
 
@@ -217,7 +221,7 @@ export class ProverNodePublisher {
     const senderAddress = this.l1TxUtils.getSenderAddress();
 
     const [gasLimit, gasPrice, latestBlock] = await Promise.all([
-      this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.rollupContract.address, data }),
+      this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.proofSubmissionTarget, data }),
       this.l1TxUtils.getGasPrice(),
       this.l1TxUtils.client.getBlock({ blockTag: 'latest' }),
     ]);
@@ -288,7 +292,7 @@ export class ProverNodePublisher {
     });
     try {
       const { receipt } = await this.l1TxUtils.sendAndMonitorTransaction(
-        { to: this.rollupContract.address, data },
+        { to: this.proofSubmissionTarget, data },
         { txTimeoutAt: args.deadline },
       );
       if (receipt.status !== 'success') {
@@ -298,7 +302,7 @@ export class ProverNodePublisher {
             args: [...txArgs],
             functionName: 'submitEpochRootProof',
             abi: RollupAbi,
-            address: this.rollupContract.address,
+            address: this.proofSubmissionTarget,
           },
           /*blobInputs*/ undefined,
           /*stateOverride*/ [],

package/src/prover-node.ts

@@ -10,7 +10,7 @@ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
 import type { EpochProverFactory } from '@aztec/prover-client';
 import { getLastSiblingPath } from '@aztec/prover-client/helpers';
 import { ChonkCache } from '@aztec/prover-client/orchestrator';
-import { PublicProcessorFactory } from '@aztec/simulator/server';
+import { type AvmSimulator, PublicProcessorFactory } from '@aztec/simulator/server';
 import {
   EventDrivenL2BlockStream,
   type L2BlockId,
@@ -47,7 +47,8 @@ import {
 import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
 import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
 import type { SpecificProverNodeConfig } from './config.js';
-import type { EpochSession, EpochSessionHooks } from './job/epoch-session.js';
+import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
+import type { EpochSessionHooks } from './job/epoch-session.js';
 import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import { ProofPublishingService } from './proof-publishing-service.js';
 import type { ProverPublisherFactory } from './prover-publisher-factory.js';
@@ -121,6 +122,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
     protected readonly rollupContract: RollupContract,
     protected readonly l1Metrics: L1Metrics,
+    private readonly avmSimulator: AvmSimulator,
     config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
     private delayer?: Delayer,
@@ -157,6 +159,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         chonkCache: this.chonkCache,
         publicProcessorFactory: new PublicProcessorFactory(
           this.contractDataSource,
+          this.avmSimulator,
           this.dateProvider,
           this.telemetryClient,
           this.log.getBindings(),
@@ -168,6 +171,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         metrics: this.jobMetrics,
         txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
         deadline: undefined,
+        // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
+        // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
+        onFailed: prover => void this.tryUploadCheckpointFailure(prover),
       },
       this.log.getBindings(),
     );
@@ -239,11 +245,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         break;
       }
     }
-    // Expiry is driven by the archiver's latest synced L2 slot
-    await this.checkEpochExpiry();
-    // Advance the local tips store only after the proving-side handling has succeeded. Any
-    // failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
-    // skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
+    // Advance the local tips store only after the proving-side handling (registration / prune) has
+    // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
+    // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
     await this.tipsStore.handleBlockStreamEvent(event);
   }
 
@@ -458,8 +462,8 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
   }
 
   /**
-   * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
-   * reaps every CheckpointProver in the store whose epoch number matches.
+   * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
+   * CheckpointProver in the store whose epoch is at or below it.
    */
   private async expireEpoch(epoch: EpochNumber): Promise<void> {
     try {
@@ -515,8 +519,9 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     });
     this.blockStream.start();
 
-    // With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
-    // from a periodic tick so epochs still expire during idle/no-event periods.
+    // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
+    // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
+    // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
     this.expiryTicker = new RunningPromise(
       () => this.checkEpochExpiry(),
       this.log,
@@ -559,9 +564,10 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
   }
 
   /**
-   * Constructs the session manager. Extracted so subclasses (test harness) can swap
-   * the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
-   * their proving data uploaded.
+   * Constructs the session manager. Extracted so subclasses (test harness) can swap the
+   * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
+   * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
+   * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
    */
   protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
     return new SessionManager({
@@ -578,7 +584,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
         finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
       },
       onSessionFailed: async session => {
-        await this.tryUploadSessionFailure(session);
+        await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
       },
       bindings: this.log.getBindings(),
     });
@@ -596,15 +602,32 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     this.sessionManager.setSessionHooks(hooks);
   }
 
-  /** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */
-  public async tryUploadSessionFailure(session: EpochSession): Promise<string | undefined> {
-    if (!this.config.proverNodeFailedEpochStore) {
+  /**
+   * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
+   * CheckpointProver constructed after this call. For the e2e harness only.
+   */
+  public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
+    this.checkpointStore.setTestHooks(hooks);
+  }
+
+  /**
+   * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
+   * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
+   * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
+   * store is configured or the checkpoint set is empty.
+   */
+  public async tryUploadEpochFailure(
+    id: string,
+    checkpoints: readonly CheckpointProver[],
+  ): Promise<string | undefined> {
+    if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
       return undefined;
     }
-    const data = SessionManager.buildSessionProvingData(session);
+    const data = SessionManager.buildProvingData(checkpoints);
     return await uploadEpochProofFailure(
       this.config.proverNodeFailedEpochStore,
-      session.getId(),
+      // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
+      id,
       data,
       this.l2BlockSource as Archiver,
       this.worldState,
@@ -613,8 +636,62 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
     );
   }
 
+  /**
+   * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
+   * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
+   * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
+   * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
+   * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
+   */
+  public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
+    if (!this.config.proverNodeFailedEpochStore) {
+      return undefined;
+    }
+    try {
+      // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
+      // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
+      // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
+      // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
+      // the snapshot (full world-state + archiver) is expensive to produce and store.
+      if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
+        this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
+          checkpointNumber: prover.checkpoint.number,
+        });
+        return undefined;
+      }
+      const data = SessionManager.buildProvingData([prover]);
+      return await uploadEpochProofFailure(
+        this.config.proverNodeFailedEpochStore,
+        // The prover's content-addressed id; the epoch number is already in the upload path.
+        prover.id,
+        data,
+        this.l2BlockSource as Archiver,
+        this.worldState,
+        assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
+        this.log,
+      );
+    } catch (err) {
+      this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
+      return undefined;
+    }
+  }
+
   // ---------------- helpers ----------------
 
+  /**
+   * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
+   * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
+   * so this returns false. Protected for direct unit-test access.
+   */
+  protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
+    const lastBlock = checkpoint.blocks.at(-1);
+    if (!lastBlock) {
+      return false;
+    }
+    const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
+    return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
+  }
+
   @memoize
   private getL1Constants(): Promise<L1RollupConstants> {
     return this.l2BlockSource.getL1Constants();

package/src/prover-publisher-factory.ts

@@ -1,6 +1,7 @@
 import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { LoggerBindings } from '@aztec/foundation/log';
 import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
 import type { TelemetryClient } from '@aztec/telemetry-client';
@@ -13,6 +14,7 @@ export class ProverPublisherFactory {
     private deps: {
       rollupContract: RollupContract;
       publisherManager: PublisherManager<L1TxUtils>;
+      proofSubmissionTarget?: EthAddress;
       telemetry?: TelemetryClient;
     },
     private bindings?: LoggerBindings,
@@ -37,6 +39,7 @@ export class ProverPublisherFactory {
       {
         rollupContract: this.deps.rollupContract,
         l1TxUtils: l1Publisher,
+        proofSubmissionTarget: this.deps.proofSubmissionTarget,
         telemetry: this.deps.telemetry,
       },
       this.bindings,