@aztec/prover-node 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7

package/src/prover-publisher-factory.ts

@@ -1,29 +1,31 @@
 import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { LoggerBindings } from '@aztec/foundation/log';
-import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
+import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import { ProverNodePublisher } from './prover-node-publisher.js';
 
 export class ProverPublisherFactory {
   constructor(
-    private config: TxSenderConfig & PublisherConfig,
+    private config: ProverTxSenderConfig & ProverPublisherConfig,
     private deps: {
       rollupContract: RollupContract;
       publisherManager: PublisherManager<L1TxUtils>;
+      proofSubmissionTarget?: EthAddress;
       telemetry?: TelemetryClient;
     },
     private bindings?: LoggerBindings,
   ) {}
 
   public async start() {
-    await this.deps.publisherManager.loadState();
+    await this.deps.publisherManager.start();
   }
 
-  public stop() {
-    this.deps.publisherManager.interrupt();
+  public async stop() {
+    await this.deps.publisherManager.stop();
   }
 
   /**
@@ -37,6 +39,7 @@ export class ProverPublisherFactory {
       {
         rollupContract: this.deps.rollupContract,
         l1TxUtils: l1Publisher,
+        proofSubmissionTarget: this.deps.proofSubmissionTarget,
         telemetry: this.deps.telemetry,
       },
       this.bindings,

package/src/session-manager.ts

@@ -0,0 +1,592 @@
+import { BlockNumber, type EpochNumber } from '@aztec/foundation/branded-types';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+import { RunningPromise } from '@aztec/foundation/running-promise';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { EpochProverFactory } from '@aztec/prover-client';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
+import {
+  type L1RollupConstants,
+  getEpochAtSlot,
+  getProofSubmissionDeadlineTimestamp,
+  getSlotRangeForEpoch,
+} from '@aztec/stdlib/epoch-helpers';
+import type { EpochProvingJobState } from '@aztec/stdlib/interfaces/server';
+
+import type { CheckpointStore } from './checkpoint-store.js';
+import { CheckpointProver } from './job/checkpoint-prover.js';
+import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
+import {
+  EpochSession,
+  type EpochSessionDeps,
+  type EpochSessionHooks,
+  type EpochSessionOptions,
+  type SessionSpec,
+  specKey,
+} from './job/epoch-session.js';
+import type { ProverNodeJobMetrics } from './metrics.js';
+import type { ProofPublishingService } from './proof-publishing-service.js';
+
+/** Trigger payload for `reconcile`. */
+export type ReconcileTrigger =
+  | { kind: 'checkpoint'; epoch: EpochNumber }
+  | { kind: 'prune'; affectedEpochs: EpochNumber[] }
+  | { kind: 'tick' }
+  | { kind: 'start-proof'; spec: SessionSpec };
+
+/** Config bag for session lifecycle decisions. */
+export type SessionManagerConfig = {
+  /** Cap on the number of non-terminal sessions (full + partial). 0 disables. */
+  maxPendingJobs: number;
+  /** Interval at which the internal periodic tick fires `reconcile({ kind: 'tick' })`. */
+  tickIntervalMs: number;
+  /** Forwarded to every session: delay before top-tree proving, letting late reorgs settle. */
+  finalizationDelayMs: number | undefined;
+};
+
+export type SessionManagerDeps = {
+  checkpointStore: CheckpointStore;
+  l2BlockSource: Pick<
+    L2BlockSource,
+    'isEpochComplete' | 'getCheckpoints' | 'getL1Constants' | 'getBlockNumber' | 'getBlockData'
+  >;
+  proverFactory: EpochProverFactory;
+  proverId: EthAddress;
+  publishingService: ProofPublishingService;
+  metrics: ProverNodeJobMetrics;
+  dateProvider: DateProvider;
+  config: SessionManagerConfig;
+  /**
+   * Fired once when a full session ends in its own genuine failure (`EpochSession.hasFailed()` — top-tree
+   * or submit failed with every prover healthy). The owner uploads a post-mortem here. Not fired for a
+   * `stopped` session (a prover under it failed — possibly a prune), which is recovered on re-add instead.
+   */
+  onSessionFailed?: (session: EpochSession) => Promise<void>;
+  bindings?: LoggerBindings;
+};
+
+/**
+ * Owns the lifecycle of every `EpochSession`. Each L2BlockStream event and periodic tick
+ * arrives via a dedicated entry point (`onCheckpointAdded`, `onPrune`, `onTick`, etc.) which
+ * schedules a `reconcile(trigger)` on a serial queue. Reconcile walks both session
+ * maps, cancels any session whose canonical content has shifted, re-creates it with
+ * the same spec but new content, and opens fresh full sessions for any epoch implicated
+ * by the trigger.
+ */
+export class SessionManager {
+  private readonly log: Logger;
+  private readonly fullSessions: Map<EpochNumber, EpochSession> = new Map();
+  private readonly partialSessions: Map<string, EpochSession> = new Map();
+  /**
+   * Serialises every reconcile call. The trigger sources (L2BlockStream events, the
+   * periodic tick, JSON-RPC `startProof`) run independently, so without this queue two
+   * reconciles could interleave on the `await session.cancel(...)` step and orphan a
+   * freshly-constructed session.
+   */
+  private readonly reconcileQueue = new SerialQueue();
+  /** Cached L1 constants, populated on first read. */
+  private cachedL1Constants: L1RollupConstants | undefined;
+  /** Test-only hooks applied to every session this manager constructs. */
+  private sessionHooks: EpochSessionHooks | undefined;
+  /** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */
+  private epochTicker: RunningPromise | undefined;
+
+  constructor(private readonly deps: SessionManagerDeps) {
+    this.log = createLogger('prover-node:session-manager', deps.bindings);
+    this.reconcileQueue.start();
+  }
+
+  /**
+   * Starts the periodic tick. Separated from the constructor so tests can drive `onTick()`
+   * manually without the background ticker interleaving. Idempotent.
+   */
+  public start(): void {
+    if (this.epochTicker) {
+      return;
+    }
+    this.epochTicker = new RunningPromise(() => this.onTick(), this.log, this.deps.config.tickIntervalMs);
+    this.epochTicker.start();
+  }
+
+  /**
+   * Installs hooks applied to every session constructed from now on. Used by the e2e
+   * harness to interpose around top-tree proving (gate it, override it, observe it)
+   * without monkey-patching the orchestrator factory.
+   */
+  public setSessionHooks(hooks: EpochSessionHooks): void {
+    this.sessionHooks = hooks;
+  }
+
+  // ---------------- read-only views ----------------
+
+  /** Every live (non-terminal) session. */
+  public allSessions(): EpochSession[] {
+    return [...this.fullSessions.values(), ...this.partialSessions.values()];
+  }
+
+  /** Returns the full session for `epoch`, if any. */
+  public getFullSession(epoch: EpochNumber): EpochSession | undefined {
+    return this.fullSessions.get(epoch);
+  }
+
+  /** Returns the partial session for `spec`, if any. */
+  public getPartialSession(spec: SessionSpec): EpochSession | undefined {
+    return this.partialSessions.get(specKey(spec));
+  }
+
+  /** Observability summary used by the prover-node API. */
+  public getJobs(): { uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[] {
+    return this.allSessions().map(s => ({
+      uuid: s.getId(),
+      status: s.getState(),
+      epochNumber: s.getEpochNumber(),
+    }));
+  }
+
+  // ---------------- event entry points ----------------
+
+  /** Called by ProverNode after a chain-checkpointed event has been added to the store. */
+  public onCheckpointAdded(epoch: EpochNumber): Promise<void> {
+    return this.scheduleReconcile({ kind: 'checkpoint', epoch });
+  }
+
+  /** Called by ProverNode after a chain-pruned event has flipped store provers to pruned. */
+  public onPrune(affectedEpochs: EpochNumber[]): Promise<void> {
+    return this.scheduleReconcile({ kind: 'prune', affectedEpochs });
+  }
+
+  /**
+   * Called periodically by ProverNode's ticker. Picks up epochs that have become complete
+   * by time without a fresh checkpoint event (e.g. the epoch's last slots are empty), and
+   * advances to the next epoch once the previous one is proven on L1.
+   */
+  public onTick(): Promise<void> {
+    return this.scheduleReconcile({ kind: 'tick' });
+  }
+
+  // ---------------- public API ----------------
+
+  /**
+   * Schedules a proof attempt for the supplied epoch and returns the job id without waiting for
+   * the proof to complete — proving can far outlast an HTTP request, so callers poll `getJobs()`
+   * for the outcome. Every session — full or partial — begins at the epoch's first slot; the
+   * partial's spec stops at the last canonical slot, while the full's stops at the epoch's last
+   * slot. Dedupes against any existing session covering the same range, returning its id.
+   */
+  public async startProof(epoch: EpochNumber): Promise<string> {
+    const canonical = await this.deps.checkpointStore.listForEpoch(epoch);
+    if (canonical.length === 0) {
+      throw new EmptyEpochError(epoch);
+    }
+    // Don't re-prove an epoch the L1 proven chain already encompasses — it was already proven
+    // (possibly by another prover node), so a fresh proof would be wasted work.
+    if (await this.isProvenChainEncompassing(canonical)) {
+      throw new EpochAlreadyProvenError(epoch);
+    }
+    const l1Constants = await this.getL1Constants();
+    const [fromSlot] = getSlotRangeForEpoch(epoch, l1Constants);
+    const toSlot = canonical[canonical.length - 1].slotNumber;
+    const spec: SessionSpec = { kind: 'partial', epochNumber: epoch, fromSlot, toSlot };
+
+    // Reuse a session already covering this exact range rather than scheduling a duplicate.
+    const existingFull = this.getFullSession(epoch);
+    if (
+      existingFull &&
+      !existingFull.isTerminal() &&
+      existingFull.getSpec().fromSlot === fromSlot &&
+      existingFull.getSpec().toSlot === toSlot
+    ) {
+      return existingFull.getId();
+    }
+    const existingPartial = this.getPartialSession(spec);
+    if (existingPartial && !existingPartial.isTerminal()) {
+      return existingPartial.getId();
+    }
+
+    await this.scheduleReconcile({ kind: 'start-proof', spec });
+    const created = this.getPartialSession(spec);
+    if (!created) {
+      throw new Error(`Failed to schedule partial proof for epoch ${epoch}`);
+    }
+    return created.getId();
+  }
+
+  /** Stops the tick, drains the reconcile queue, and cancels every live session. */
+  public async stop(): Promise<void> {
+    await this.epochTicker?.stop();
+    await this.reconcileQueue.cancel();
+    const sessions = this.allSessions();
+    // A clean shutdown is just a restart, so preserve the in-flight broker jobs (abortJobs: false)
+    // for the restarted node to reuse rather than re-proving the epoch from scratch.
+    await Promise.allSettled(sessions.map(s => s.cancel('prover-node stopping', { abortJobs: false })));
+  }
+
+  // ---------------- reconcile ----------------
+
+  private scheduleReconcile(trigger: ReconcileTrigger): Promise<void> {
+    return this.reconcileQueue.put(() => this.reconcile(trigger));
+  }
+
+  private async reconcile(trigger: ReconcileTrigger): Promise<void> {
+    this.log.debug(`Reconciling`, { trigger });
+
+    this.recreateInvalidSessions();
+
+    const implicatedEpochs = await this.epochsForTrigger(trigger);
+    for (const epoch of implicatedEpochs) {
+      await this.openFullSessionIfReady(epoch);
+    }
+
+    if (trigger.kind === 'start-proof') {
+      this.openPartialSession(trigger.spec);
+    }
+  }
+
+  private recreateInvalidSessions(): void {
+    for (const [key, session] of Array.from(this.fullSessions.entries())) {
+      const canonical = this.checkpointsForSpec(session.getSpec());
+      const contentChanged = !this.checkpointsMatch(session.getCheckpoints(), canonical);
+
+      if (session.isTerminal()) {
+        // A full session that failed on its own account is retained as a "do not re-prove" marker while
+        // its content is unchanged — this is what stops the tick re-proving a deterministically-failing
+        // epoch. When the content changes (a re-add), it is replaced so the epoch retries over the new
+        // provers. Any other terminal full session is simply dropped.
+        if (session.hasFailed() && !contentChanged) {
+          continue;
+        }
+        this.fullSessions.delete(key);
+        if (contentChanged && this.canBuildOver(canonical)) {
+          const newSession = this.constructSession(session.getSpec(), canonical);
+          this.fullSessions.set(key, newSession);
+          void this.runSession(newSession);
+        }
+        continue;
+      }
+
+      if (contentChanged) {
+        this.fireAndForgetCancel(session, 'canonical content changed');
+        this.fullSessions.delete(key);
+        if (this.canBuildOver(canonical)) {
+          const newSession = this.constructSession(session.getSpec(), canonical);
+          this.fullSessions.set(key, newSession);
+          void this.runSession(newSession);
+        }
+      }
+    }
+    for (const [key, session] of Array.from(this.partialSessions.entries())) {
+      if (session.isTerminal()) {
+        this.partialSessions.delete(key);
+        continue;
+      }
+      const canonical = this.checkpointsForSpec(session.getSpec());
+      if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
+        this.fireAndForgetCancel(session, 'canonical content changed');
+        this.partialSessions.delete(key);
+        if (this.canBuildOver(canonical)) {
+          const newSession = this.constructSession(session.getSpec(), canonical);
+          this.partialSessions.set(key, newSession);
+          void this.runSession(newSession);
+        }
+      }
+    }
+  }
+
+  /** A session may be built over a checkpoint set only when it is non-empty and contains no failed prover. */
+  private canBuildOver(canonical: readonly CheckpointProver[]): boolean {
+    return canonical.length > 0 && !this.hasFailedProver(canonical);
+  }
+
+  private async openFullSessionIfReady(epoch: EpochNumber): Promise<void> {
+    // A session present here already covers the epoch: either live, or a retained genuinely-failed
+    // session kept by `recreateInvalidSessions` as a "do not re-prove" marker. Either way, don't open
+    // another — the retained-failed one is replaced only when its canonical content changes.
+    if (this.fullSessions.has(epoch)) {
+      return;
+    }
+    if (this.atMaxSessionLimit()) {
+      this.log.debug(`Skipping full-session open for epoch ${epoch}: max pending jobs reached`);
+      return;
+    }
+    if (!(await this.deps.l2BlockSource.isEpochComplete(epoch))) {
+      return;
+    }
+    const l1Constants = await this.getL1Constants();
+    const archiverCps = await this.deps.l2BlockSource.getCheckpoints({ epoch });
+    if (archiverCps.length === 0) {
+      return;
+    }
+    const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants);
+    const canonical = this.deps.checkpointStore.listInSlotRange(fromSlot, toSlot);
+    if (!this.archiverFullyCovered(archiverCps, canonical)) {
+      this.log.debug(`Skipping full-session open for epoch ${epoch}: archiver checkpoints not all in store`, {
+        archiverCount: archiverCps.length,
+        storeCount: canonical.length,
+      });
+      return;
+    }
+    if (this.hasFailedProver(canonical)) {
+      // A checkpoint prover in the set has failed (a sub-tree fault or a prune-induced fork fault), so a
+      // session over it would fail immediately. Don't re-create it every tick — it recovers when a
+      // prune/re-add replaces the failed prover with a fresh one, or fails for good at expiry.
+      this.log.debug(`Skipping full-session open for epoch ${epoch}: a checkpoint prover has failed`, { epoch });
+      return;
+    }
+    const spec: SessionSpec = { kind: 'full', epochNumber: epoch, fromSlot, toSlot };
+    const session = this.constructSession(spec, canonical);
+    this.fullSessions.set(epoch, session);
+    void this.runSession(session);
+  }
+
+  private openPartialSession(spec: SessionSpec): void {
+    const canonical = this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
+    if (canonical.length === 0 || this.hasFailedProver(canonical)) {
+      return;
+    }
+    // Reuse a live partial session for this epoch whose checkpoint set already matches the
+    // canonical content — e.g. a repeated `startProof` with no new checkpoints mined since the
+    // last one. Reconstructing would re-prove identical content and burn a pending-job slot.
+    const existing = Array.from(this.partialSessions.values()).find(
+      s =>
+        s.getSpec().epochNumber === spec.epochNumber &&
+        !s.isTerminal() &&
+        this.checkpointsMatch(s.getCheckpoints(), canonical),
+    );
+    if (existing) {
+      return;
+    }
+    if (this.atMaxSessionLimit()) {
+      throw new Error(`Maximum pending proving jobs ${this.deps.config.maxPendingJobs} reached.`);
+    }
+    const session = this.constructSession(spec, canonical);
+    this.partialSessions.set(specKey(spec), session);
+    void this.runSession(session);
+  }
+
+  // ---------------- session construction ----------------
+
+  protected constructSession(spec: SessionSpec, checkpoints: readonly CheckpointProver[]): EpochSession {
+    return this.doConstructSession(spec, checkpoints, this.buildSessionDeps(spec.epochNumber), this.sessionHooks);
+  }
+
+  /** Extracted for test override. */
+  protected doConstructSession(
+    spec: SessionSpec,
+    checkpoints: readonly CheckpointProver[],
+    sessionDeps: EpochSessionDeps,
+    hooks?: EpochSessionHooks,
+  ): EpochSession {
+    return new EpochSession(spec, checkpoints, { ...sessionDeps, hooks });
+  }
+
+  private buildSessionDeps(epochNumber: EpochNumber): EpochSessionDeps {
+    const config: EpochSessionOptions = {
+      finalizationDelayMs: this.deps.config.finalizationDelayMs,
+    };
+    return {
+      proverFactory: this.deps.proverFactory,
+      proverId: this.deps.proverId,
+      publishingService: this.deps.publishingService,
+      metrics: this.deps.metrics,
+      dateProvider: this.deps.dateProvider,
+      deadline: this.computeDeadline(epochNumber),
+      config,
+      bindings: this.deps.bindings,
+    };
+  }
+
+  private computeDeadline(epochNumber: EpochNumber): Date | undefined {
+    if (!this.cachedL1Constants) {
+      return undefined;
+    }
+    const ts = getProofSubmissionDeadlineTimestamp(epochNumber, this.cachedL1Constants);
+    return new Date(Number(ts) * 1000);
+  }
+
+  private async runSession(session: EpochSession): Promise<void> {
+    // A reconcile may have cancelled this session before it starts (content-change
+    // recreation). Don't proceed — start() would build a TopTreeJob that should never run.
+    if (session.isTerminal()) {
+      this.log.debug(`Skipping start for ${session.getId()}: already terminal (${session.getState()})`);
+      return;
+    }
+    const state = await session.start();
+    this.log.info(`Session ${session.getId()} exited with state ${state}`);
+
+    // A full session that failed on its own account (top-tree/submit failed with every prover healthy)
+    // is a genuine, race-free failure: upload its post-mortem once. `recreateInvalidSessions` retains
+    // the terminal session so this fires exactly once (it is never re-run over the same content). A
+    // `stopped` session (a prover under it failed) is not uploaded — it may be a prune, and recovers on
+    // re-add.
+    if (session.getKind() === 'full' && session.hasFailed() && this.deps.onSessionFailed) {
+      try {
+        await this.deps.onSessionFailed(session);
+      } catch (err) {
+        this.log.error(`Error in onSessionFailed callback for epoch ${session.getEpochNumber()}`, err);
+      }
+    }
+  }
+
+  /**
+   * Builds the EpochProvingJobData snapshot for a post-mortem failure upload from a set of
+   * checkpoint provers. Includes every checkpoint regardless of whether sub-tree proving
+   * completed — partial state is still useful for post-mortem analysis.
+   */
+  public static async buildProvingData(checkpoints: readonly CheckpointProver[]): Promise<EpochProvingJobData> {
+    if (checkpoints.length === 0) {
+      throw new Error('Cannot build proving data from an empty checkpoint set');
+    }
+    // Provers no longer cache their txs; re-fetch each checkpoint's txs from the tx pool (concurrently)
+    // for the snapshot. The pool retains them past the proving window (A-1274), so this is durable.
+    const perCheckpoint = await Promise.all(checkpoints.map(async c => ({ c, txs: await c.getTxsForUpload() })));
+    const txs = new Map();
+    const l1ToL2Messages: Record<number, Fr[]> = {};
+    for (const { c, txs: checkpointTxs } of perCheckpoint) {
+      for (const [hash, tx] of checkpointTxs) {
+        txs.set(hash, tx);
+      }
+      l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
+    }
+    return {
+      epochNumber: checkpoints[0].epochNumber,
+      checkpoints: checkpoints.map(c => c.checkpoint),
+      txs,
+      l1ToL2Messages,
+      previousBlockHeader: checkpoints[0].previousBlockHeader,
+      attestations: [],
+    };
+  }
+
+  // ---------------- reconcile helpers ----------------
+
+  private atMaxSessionLimit(): boolean {
+    const { maxPendingJobs: max } = this.deps.config;
+    if (!max || max <= 0) {
+      return false;
+    }
+    const live = this.allSessions().filter(s => !s.isTerminal()).length;
+    return live >= max;
+  }
+
+  /**
+   * Maps a reconcile trigger to the epochs whose full session should be (re)opened.
+   *
+   * The periodic `tick` returns the next unproven epoch every time; it does not track prior attempts.
+   * `openFullSessionIfReady` is what keeps this from re-proving a doomed epoch: it refuses to build a
+   * session when any checkpoint prover in the set has failed, so a stuck epoch is cheaply skipped each
+   * tick rather than re-proved. `checkpoint` and `prune` fire when an epoch's canonical content changes
+   * (a checkpoint arrives, or a reorg prunes/replaces one) — which is what installs a fresh prover in
+   * place of a failed one, letting the next open succeed and recovering a pruned-then-re-added epoch.
+   */
+  private async epochsForTrigger(trigger: ReconcileTrigger): Promise<EpochNumber[]> {
+    switch (trigger.kind) {
+      case 'checkpoint':
+        return [trigger.epoch];
+      case 'prune':
+        return trigger.affectedEpochs;
+      case 'tick': {
+        const epoch = await this.nextUnprovenEpoch();
+        return epoch === undefined ? [] : [epoch];
+      }
+      case 'start-proof':
+        return [];
+    }
+  }
+
+  /**
+   * The next epoch to prove: the epoch containing the first block after the proven tip.
+   * Returns undefined when that block has not been mined yet (e.g. nothing new to prove).
+   * Subsequent ticks advance only once the chain's proven height moves forward, so epochs
+   * are proven in order rather than all at once.
+   */
+  private async nextUnprovenEpoch(): Promise<EpochNumber | undefined> {
+    const lastProven = (await this.deps.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
+    const firstToProve = BlockNumber(lastProven + 1);
+    const header = (await this.deps.l2BlockSource.getBlockData({ number: firstToProve }))?.header;
+    if (!header) {
+      return undefined;
+    }
+    return getEpochAtSlot(header.getSlot(), await this.getL1Constants());
+  }
+
+  private checkpointsForSpec(spec: SessionSpec): CheckpointProver[] {
+    return this.deps.checkpointStore.listInSlotRange(spec.fromSlot, spec.toSlot);
+  }
+
+  /**
+   * True if any prover in the set has failed. The epoch cannot be proven over a failed prover (it can
+   * never produce its block proofs), so a session must not be built or rebuilt over it until a prune/re-add
+   * has replaced it with a fresh prover.
+   */
+  private hasFailedProver(checkpoints: readonly CheckpointProver[]): boolean {
+    return checkpoints.some(c => c.isFailed());
+  }
+
+  private fireAndForgetCancel(session: EpochSession, reason: string): void {
+    void session.cancel(reason).catch(err => this.log.warn(`Error cancelling session ${session.getId()}`, err));
+  }
+
+  private checkpointsMatch(a: readonly CheckpointProver[], b: readonly CheckpointProver[]): boolean {
+    if (a.length !== b.length) {
+      return false;
+    }
+    for (let i = 0; i < a.length; i++) {
+      if (a[i].id !== b[i].id || a[i].isCancelled()) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private archiverFullyCovered(
+    archiverCps: readonly PublishedCheckpoint[],
+    storeCps: readonly CheckpointProver[],
+  ): boolean {
+    if (storeCps.length < archiverCps.length) {
+      return false;
+    }
+    // Compare by content-addressed id (number, slot, archive root) rather than checkpoint number:
+    // a reorg can keep the number while changing the checkpoint's post-state archive root.
+    const storeIds = new Set(storeCps.map(p => p.id));
+    return archiverCps.every(cp => storeIds.has(CheckpointProver.idFor(cp.checkpoint)));
+  }
+
+  /**
+   * Returns true if the L1 proven tip already covers every canonical checkpoint in the set — i.e.
+   * the epoch has already been fully proven, so there is no point starting a new proof for it.
+   * Conservatively returns false when nothing is proven yet.
+   */
+  private async isProvenChainEncompassing(canonical: readonly CheckpointProver[]): Promise<boolean> {
+    const provenBlock = await this.deps.l2BlockSource.getBlockNumber({ tag: 'proven' });
+    if (!provenBlock || provenBlock <= 0) {
+      return false;
+    }
+    const lastCheckpoint = canonical[canonical.length - 1].checkpoint;
+    const lastBlock = lastCheckpoint.blocks[lastCheckpoint.blocks.length - 1].number;
+    return provenBlock >= lastBlock;
+  }
+
+  private async getL1Constants(): Promise<L1RollupConstants> {
+    if (!this.cachedL1Constants) {
+      this.cachedL1Constants = await this.deps.l2BlockSource.getL1Constants();
+    }
+    return this.cachedL1Constants;
+  }
+}
+
+class EmptyEpochError extends Error {
+  constructor(epochNumber: EpochNumber) {
+    super(`No blocks found for epoch ${epochNumber}`);
+    this.name = 'EmptyEpochError';
+  }
+}
+
+class EpochAlreadyProvenError extends Error {
+  constructor(epochNumber: EpochNumber) {
+    super(`Epoch ${epochNumber} is already proven on L1`);
+    this.name = 'EpochAlreadyProvenError';
+  }
+}

package/src/test/index.ts

@@ -1,14 +1,14 @@
+import type { EpochProverFactory } from '@aztec/prover-client';
 import type { EpochProverManager } from '@aztec/stdlib/interfaces/server';
 
-import type { EpochProvingJob } from '../job/epoch-proving-job.js';
-import type { ProverNodePublisher } from '../prover-node-publisher.js';
+import type { ProofPublishingService } from '../proof-publishing-service.js';
 import { ProverNode } from '../prover-node.js';
+import type { SessionManager } from '../session-manager.js';
 
 abstract class TestProverNodeClass extends ProverNode {
-  declare public prover: EpochProverManager;
-  declare public publisher: ProverNodePublisher;
-
-  public abstract override tryUploadEpochFailure(job: EpochProvingJob): Promise<string | undefined>;
+  declare public prover: EpochProverManager & EpochProverFactory;
+  declare public publishingService: ProofPublishingService;
+  declare public sessionManager: SessionManager;
 }
 
 export type TestProverNode = TestProverNodeClass;