@aztec/prover-node 5.0.0-private.20260319 → 5.0.0-rc.1

package/src/job/checkpoint-prover.ts

@@ -0,0 +1,465 @@
+import { type ARCHIVE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
+import { BlockNumber, type EpochNumber, type SlotNumber } from '@aztec/foundation/branded-types';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import type { Logger } from '@aztec/foundation/log';
+import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
+import type { Tuple } from '@aztec/foundation/serialize';
+import { type DateProvider, Timer } from '@aztec/foundation/timer';
+import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
+import { protocolContractsHash } from '@aztec/protocol-contracts';
+import type { EpochProverFactory } from '@aztec/prover-client';
+import type { CheckpointSubTreeOrchestrator, ChonkCache, SubTreeResult } from '@aztec/prover-client/orchestrator';
+import type { PublicProcessor, PublicProcessorFactory } from '@aztec/simulator/server';
+import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
+import type { CommitteeAttestation, L2Block } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
+import type { ForkMerkleTreeOperations, ITxProvider } from '@aztec/stdlib/interfaces/server';
+import { CheckpointConstantData } from '@aztec/stdlib/rollup';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
+import type { BlockHeader, ProcessedTx, Tx } from '@aztec/stdlib/tx';
+
+import type { ProverNodeJobMetrics } from '../metrics.js';
+
+/** Dependencies a `CheckpointProver` needs at construction. */
+export type CheckpointProverDeps = {
+  proverFactory: EpochProverFactory;
+  /** Shared chonk-verifier cache. Survives across all sessions / epochs. */
+  chonkCache: ChonkCache;
+  publicProcessorFactory: PublicProcessorFactory;
+  dbProvider: Pick<ForkMerkleTreeOperations, 'fork'>;
+  txProvider: ITxProvider;
+  /** Clock the prover-node operates against — e2e fixtures inject a cheat-controlled one. */
+  dateProvider: DateProvider;
+  proverId: EthAddress;
+  metrics: ProverNodeJobMetrics;
+  /** Tx gathering deadline. */
+  txGatheringTimeoutMs: number;
+  /** Public processor deadline. */
+  deadline: Date | undefined;
+  log: Logger;
+};
+
+/** Inputs that fully describe a checkpoint at register time. */
+export type CheckpointProverArgs = {
+  checkpoint: Checkpoint;
+  /** Epoch the checkpoint belongs to (derivable from slot + L1 constants; cached at register time). */
+  epochNumber: EpochNumber;
+  attestations: CommitteeAttestation[];
+  previousBlockHeader: BlockHeader;
+  l1ToL2Messages: Fr[];
+  previousArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>;
+};
+
+/**
+ * Self-contained per-checkpoint prover, content-addressed by
+ * `(checkpoint number, slot number, checkpoint archive root)`.
+ *
+ * The store creates a CheckpointProver once per content-key. Keying on the checkpoint's
+ * own archive root (its post-state) means two checkpoints are "the same" iff they
+ * produce the same archive — so a reorg branch, or a replacement built on the same
+ * predecessor but with different content, keys to a distinct prover; an identical
+ * re-add keys to the same one and reuses its in-flight sub-tree work.
+ *
+ * The prover eagerly starts its own tx gather and sub-tree work in the constructor, so
+ * callers only need to call `whenBlockProofsReady()` to obtain the resulting block-rollup
+ * proofs.
+ *
+ * The prover survives prune/re-add cycles via `markPruned()` / `markCanonical()` —
+ * sub-tree proving keeps running underneath, so a checkpoint that is re-added after
+ * a brief reorg can be re-consumed with no re-proving.
+ *
+ * `cancel()` is idempotent. It aborts the gather + sub-tree, rejects the block-proof
+ * promise, and exposes a `whenDone()` that resolves once teardown has unwound.
+ */
+export class CheckpointProver {
+  readonly id: string;
+  readonly checkpoint: Checkpoint;
+  readonly epochNumber: EpochNumber;
+  readonly slotNumber: SlotNumber;
+  readonly attestations: CommitteeAttestation[];
+  readonly previousBlockHeader: BlockHeader;
+  readonly l1ToL2Messages: Fr[];
+  readonly previousArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>;
+
+  /** Per-prover tx map — populated by the internal gather. Empty until then. */
+  readonly txs: Map<string, Tx> = new Map();
+
+  /** Resolved by the sub-tree on success, rejected on cancel/failure. */
+  private readonly blockProofs: PromiseWithResolvers<SubTreeResult['blockProofOutputs']> = promiseWithResolvers();
+
+  private cancelled = false;
+  private subTree?: CheckpointSubTreeOrchestrator;
+  private completed = false;
+  /** Pruned in the canonical chain but not yet reaped — sub-tree continues running. */
+  private pruned = false;
+  private readonly abortController = new AbortController();
+
+  /** Tracks the eager gather+execute task so `cancel()` and `whenDone()` can await its unwind. */
+  private readonly runPromise: Promise<void>;
+  /** Tracks the cancel-driven teardown so `whenDone()` can await it. */
+  private cancelPromise?: Promise<void>;
+
+  constructor(
+    args: CheckpointProverArgs,
+    private readonly deps: CheckpointProverDeps,
+  ) {
+    this.checkpoint = args.checkpoint;
+    this.epochNumber = args.epochNumber;
+    this.slotNumber = args.checkpoint.header.slotNumber;
+    this.attestations = args.attestations;
+    this.previousBlockHeader = args.previousBlockHeader;
+    this.l1ToL2Messages = args.l1ToL2Messages;
+    this.previousArchiveSiblingPath = args.previousArchiveSiblingPath;
+    this.id = CheckpointProver.idFor(args.checkpoint);
+    // Mark blockProofs as observed so a cancel that lands before any consumer awaits
+    // does not surface as an unhandled rejection.
+    this.blockProofs.promise.catch(() => {});
+    deps.log.info(`Created CheckpointProver ${this.id}`, {
+      checkpointNumber: this.checkpoint.number,
+      epochNumber: this.epochNumber,
+      slotNumber: this.slotNumber,
+      blockCount: this.checkpoint.blocks.length,
+      l1ToL2MessageCount: this.l1ToL2Messages.length,
+      archiveRoot: this.checkpoint.archive.root.toString(),
+    });
+    // Kick off the eager gather + sub-tree pipeline.
+    this.runPromise = this.gatherAndExecute();
+  }
+
+  /**
+   * Stable content-addressed identifier: `${checkpoint number}:${slot}:${archive root}`.
+   * The archive root is the checkpoint's post-state, so it distinguishes any two
+   * checkpoints that differ in history or content while collapsing identical re-adds.
+   */
+  public static idFor(checkpoint: Checkpoint): string {
+    return `${checkpoint.number}:${checkpoint.header.slotNumber}:${checkpoint.archive.root.toString()}`;
+  }
+
+  public isCancelled(): boolean {
+    return this.cancelled;
+  }
+
+  /** True once block-level proving has been fully *enqueued* (sub-tree completion may still be pending). */
+  public isCompleted(): boolean {
+    return this.completed;
+  }
+
+  public isPruned(): boolean {
+    return this.pruned;
+  }
+
+  /**
+   * Mark this prover as no longer present in the canonical chain. Sub-tree proving keeps
+   * running so the work survives if the checkpoint is re-added. Idempotent.
+   */
+  public markPruned(): void {
+    if (this.pruned) {
+      return;
+    }
+    this.pruned = true;
+    this.deps.log.info(`Marking CheckpointProver ${this.id} as pruned`, {
+      checkpointNumber: this.checkpoint.number,
+      slotNumber: this.slotNumber,
+    });
+  }
+
+  /** Mark this prover as part of the canonical chain again after a re-add. Idempotent. */
+  public markCanonical(): void {
+    if (!this.pruned) {
+      return;
+    }
+    this.pruned = false;
+    this.deps.log.info(`Marking CheckpointProver ${this.id} as canonical`, {
+      checkpointNumber: this.checkpoint.number,
+      slotNumber: this.slotNumber,
+    });
+  }
+
+  /** AbortSignal that fires on cancel — for callers that want to wire their own tasks. */
+  public getAbortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
+  /** Promise that resolves with the block-rollup proofs for this checkpoint (or rejects on cancel/failure). */
+  public whenBlockProofsReady(): Promise<SubTreeResult['blockProofOutputs']> {
+    return this.blockProofs.promise;
+  }
+
+  /** Resolves when all in-flight work for this prover has fully unwound. */
+  public async whenDone(): Promise<void> {
+    await this.runPromise.catch(() => {});
+    if (this.cancelPromise) {
+      await this.cancelPromise;
+    }
+  }
+
+  private async gatherAndExecute(): Promise<void> {
+    try {
+      const txs = await this.gatherTxs();
+      if (this.cancelled) {
+        return;
+      }
+      await this.executeCheckpoint(txs);
+    } catch (err) {
+      if (this.cancelled) {
+        this.deps.log.debug(`CheckpointProver ${this.id} cancelled during gather/execute`, {
+          checkpointNumber: this.checkpoint.number,
+        });
+        return;
+      }
+      this.deps.log.error(`Error in CheckpointProver ${this.id}`, err, {
+        checkpointNumber: this.checkpoint.number,
+      });
+      this.blockProofs.reject(err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  private async gatherTxs(): Promise<Map<string, Tx>> {
+    const deadline = new Date(this.deps.dateProvider.now() + this.deps.txGatheringTimeoutMs);
+    const txsByBlock = await Promise.all(
+      this.checkpoint.blocks.map(block => this.deps.txProvider.getTxsForBlock(block, { deadline })),
+    );
+    const txs = txsByBlock.map(({ txs }) => txs).flat();
+    const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
+
+    if (missingTxs.length > 0) {
+      throw new Error(
+        `Txs not found for checkpoint ${this.checkpoint.number}: ${missingTxs.map(hash => hash.toString()).join(', ')}`,
+      );
+    }
+    return new Map<string, Tx>(txs.map(tx => [tx.getTxHash().toString(), tx]));
+  }
+
+  private async executeCheckpoint(txs: Map<string, Tx>): Promise<void> {
+    const signal = this.abortController.signal;
+    const checkpointTimer = new Timer();
+    let subTreeStarted = false;
+
+    try {
+      for (const [hash, tx] of txs) {
+        this.txs.set(hash, tx);
+      }
+
+      const { chainId, version } = this.checkpoint.blocks[0].header.globalVariables;
+      const checkpointConstants = CheckpointConstantData.from({
+        chainId,
+        version,
+        vkTreeRoot: getVKTreeRoot(),
+        protocolContractsHash: protocolContractsHash,
+        proverId: this.deps.proverId.toField(),
+        slotNumber: this.checkpoint.header.slotNumber,
+        coinbase: this.checkpoint.header.coinbase,
+        feeRecipient: this.checkpoint.header.feeRecipient,
+        gasFees: this.checkpoint.header.gasFees,
+      });
+
+      this.deps.log.info(`Starting processing checkpoint ${this.checkpoint.number}`, {
+        checkpointNumber: this.checkpoint.number,
+        checkpointHash: this.checkpoint.hash().toString(),
+        blockCount: this.checkpoint.blocks.length,
+      });
+
+      this.subTree = await this.deps.proverFactory.createCheckpointSubTreeOrchestrator(
+        this.deps.chonkCache,
+        this.epochNumber,
+        checkpointConstants,
+        this.l1ToL2Messages,
+        this.checkpoint.blocks.length,
+        this.previousBlockHeader,
+      );
+      subTreeStarted = true;
+      // Bridge the sub-tree's result onto blockProofs.
+      void this.subTree.getSubTreeResult().then(
+        result => {
+          this.deps.log.info(`Sub-tree block proofs ready for checkpoint ${this.checkpoint.number}`, {
+            checkpointNumber: this.checkpoint.number,
+            blockProofCount: result.blockProofOutputs.length,
+          });
+          this.blockProofs.resolve(result.blockProofOutputs);
+        },
+        err => this.blockProofs.reject(err),
+      );
+      if (signal.aborted) {
+        return;
+      }
+
+      const allTxs = this.checkpoint.blocks.flatMap(block =>
+        block.body.txEffects.map(txEffect => txs.get(txEffect.txHash.toString())!),
+      );
+      const publicTxs = allTxs.filter(tx => tx?.data.forPublic);
+      if (publicTxs.length > 0) {
+        await this.subTree.startChonkVerifierCircuits(publicTxs);
+        if (signal.aborted) {
+          return;
+        }
+      }
+
+      for (let blockIndex = 0; blockIndex < this.checkpoint.blocks.length; blockIndex++) {
+        const blockTimer = new Timer();
+        const block = this.checkpoint.blocks[blockIndex];
+        const globalVariables = block.header.globalVariables;
+        const blockTxs = this.getTxsForBlock(block, txs);
+
+        await this.subTree.startNewBlock(block.number, globalVariables.timestamp, blockTxs.length);
+        if (signal.aborted) {
+          return;
+        }
+
+        const db = await this.createFork(
+          BlockNumber(block.number - 1),
+          blockIndex === 0 ? this.l1ToL2Messages : undefined,
+        );
+        try {
+          if (signal.aborted) {
+            return;
+          }
+          const config = PublicSimulatorConfig.from({
+            proverId: this.deps.proverId.toField(),
+            skipFeeEnforcement: false,
+            collectDebugLogs: false,
+            collectHints: true,
+            collectPublicInputs: true,
+            collectStatistics: false,
+          });
+          const publicProcessor = this.deps.publicProcessorFactory.create(db, globalVariables, config);
+          const processed = await this.processTxs(publicProcessor, blockTxs);
+          if (signal.aborted) {
+            return;
+          }
+          await this.subTree.addTxs(processed);
+        } finally {
+          await db.close();
+        }
+        if (signal.aborted) {
+          return;
+        }
+
+        await this.subTree.setBlockCompleted(block.number, block.header);
+        this.deps.metrics.recordBlockProcessing(blockTimer.ms());
+        if (signal.aborted) {
+          return;
+        }
+      }
+
+      this.completed = true;
+      this.deps.metrics.recordCheckpointProcessing(checkpointTimer.ms());
+      this.deps.log.info(
+        `Finished enqueueing block-level proving for checkpoint ${this.checkpoint.number} in ${checkpointTimer.ms()}ms`,
+        {
+          checkpointNumber: this.checkpoint.number,
+          blockCount: this.checkpoint.blocks.length,
+          durationMs: checkpointTimer.ms(),
+        },
+      );
+    } finally {
+      if (!this.completed) {
+        if (subTreeStarted) {
+          await this.teardownSubTree();
+        }
+        this.blockProofs.reject(new Error(`Checkpoint ${this.id} did not complete block processing`));
+      }
+    }
+  }
+
+  /**
+   * Mark cancelled. Idempotent. Aborts in-flight work, rejects the block-proof promise,
+   * and kicks off a background teardown of the sub-tree. The teardown promise is exposed
+   * via `whenDone()`.
+   *
+   * `routine` distinguishes a post-finalize teardown (sub-tree already proven, fires
+   * once at prover exit) from a real abort (reorg, prune, deadline). Behaviour is
+   * identical either way; the flag only adjusts log verbosity.
+   */
+  public cancel({ routine = false }: { routine?: boolean } = {}): void {
+    if (this.cancelled) {
+      return;
+    }
+    this.cancelled = true;
+    // A teardown of a completed prover is routine regardless of the caller's flag —
+    // we logged the work as done already, so don't relabel it as a mid-flight cancel.
+    if (routine || this.completed) {
+      this.deps.log.verbose(`Tearing down CheckpointProver ${this.id}`, {
+        checkpointNumber: this.checkpoint.number,
+        wasCompleted: this.completed,
+      });
+    } else {
+      this.deps.log.info(`Cancelling in-flight CheckpointProver ${this.id}`, {
+        checkpointNumber: this.checkpoint.number,
+        wasCompleted: this.completed,
+      });
+    }
+    this.abortController.abort();
+    this.blockProofs.reject(new Error(`Checkpoint ${this.id} cancelled`));
+    this.cancelPromise = this.runCancel().catch(() => {});
+  }
+
+  private async runCancel(): Promise<void> {
+    if (this.subTree) {
+      try {
+        this.subTree.cancel();
+      } catch (err) {
+        this.deps.log.error('Error cancelling sub-tree', err);
+      }
+    }
+    await this.runPromise.catch(() => {});
+    if (this.subTree) {
+      await this.teardownSubTree();
+    }
+  }
+
+  private async teardownSubTree(): Promise<void> {
+    const { subTree } = this;
+    this.subTree = undefined;
+    if (subTree) {
+      this.deps.log.debug(`Tearing down sub-tree for checkpoint ${this.checkpoint.number}`, {
+        checkpointNumber: this.checkpoint.number,
+      });
+      try {
+        await subTree.stop();
+      } catch (err) {
+        this.deps.log.error('Error stopping sub-tree', err);
+      }
+    }
+  }
+
+  private getTxsForBlock(block: L2Block, txs: Map<string, Tx>): Tx[] {
+    return block.body.txEffects.map(txEffect => txs.get(txEffect.txHash.toString())!);
+  }
+
+  private async processTxs(publicProcessor: PublicProcessor, txs: Tx[]): Promise<ProcessedTx[]> {
+    const [processedTxs, failedTxs] = await publicProcessor.process(txs, { deadline: this.deps.deadline });
+
+    if (failedTxs.length) {
+      const failedTxHashes = await Promise.all(failedTxs.map(({ tx }) => tx.getTxHash()));
+      throw new Error(
+        `Txs failed processing: ${failedTxs
+          .map(({ error }, index) => `${failedTxHashes[index]} (${error})`)
+          .join(', ')}`,
+      );
+    }
+
+    if (processedTxs.length !== txs.length) {
+      throw new Error(`Failed to process all txs: processed ${processedTxs.length} out of ${txs.length}`);
+    }
+
+    return processedTxs;
+  }
+
+  private async createFork(blockNumber: BlockNumber, l1ToL2Messages: Fr[] | undefined) {
+    const db = await this.deps.dbProvider.fork(blockNumber);
+
+    if (l1ToL2Messages !== undefined) {
+      const l1ToL2MessagesPadded = padArrayEnd<Fr, number>(
+        l1ToL2Messages,
+        Fr.ZERO,
+        NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP,
+        'Too many L1 to L2 messages',
+      );
+      await db.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2MessagesPadded);
+    }
+
+    return db;
+  }
+}