@aztec/prover-node 0.0.1-commit.2f68f620 → 0.0.1-commit.3100065

package/src/actions/rerun-epoch-proving-job.ts

@@ -1,23 +1,33 @@
 import { createArchiverStore, createContractDataSource } from '@aztec/archiver';
 import type { L1ContractsConfig } from '@aztec/ethereum/config';
+import { BlockNumber } from '@aztec/foundation/branded-types';
 import type { Logger } from '@aztec/foundation/log';
+import { DateProvider } from '@aztec/foundation/timer';
 import { type ProverClientConfig, createProverClient } from '@aztec/prover-client';
 import { ProverBrokerConfig, createAndStartProvingBroker } from '@aztec/prover-client/broker';
+import { getLastSiblingPath } from '@aztec/prover-client/helpers';
+import { ChonkCache } from '@aztec/prover-client/orchestrator';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
+import type { L2Block } from '@aztec/stdlib/block';
+import { getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+import type { ITxProvider } from '@aztec/stdlib/interfaces/server';
 import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
+import type { Tx, TxHash } from '@aztec/stdlib/tx';
 import type { GenesisData } from '@aztec/stdlib/world-state';
 import { getTelemetryClient } from '@aztec/telemetry-client';
 import { createWorldState } from '@aztec/world-state';
 
 import { readFileSync } from 'fs';
 
+import { CheckpointProver } from '../job/checkpoint-prover.js';
 import { deserializeEpochProvingJobData } from '../job/epoch-proving-job-data.js';
-import { EpochProvingJob } from '../job/epoch-proving-job.js';
+import { EpochSession, type SessionSpec } from '../job/epoch-session.js';
 import { ProverNodeJobMetrics } from '../metrics.js';
 
 /**
  * Given a local folder where `downloadEpochProvingJob` was called, creates a new archiver and world state
- * using the state snapshots, and creates a new epoch proving job to prove the downloaded proving job.
+ * using the state snapshots, and creates a new epoch proving session to prove the downloaded proving job.
  * Proving is done with a local proving broker and agents as specified by the config.
  */
 export async function rerunEpochProvingJob(
@@ -31,7 +41,7 @@ export async function rerunEpochProvingJob(
 
   const telemetry = getTelemetryClient();
   const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job'));
-  await using worldState = await createWorldState(config, genesis);
+  const worldState = await createWorldState(config, genesis);
   const initialBlockHash = await worldState.getInitialHeader().hash();
   const archiver = await createArchiverStore(config, initialBlockHash);
   const publicProcessorFactory = new PublicProcessorFactory(
@@ -41,34 +51,99 @@ export async function rerunEpochProvingJob(
     log.getBindings(),
   );
 
-  const publisher = {
-    submitEpochProof: () => Promise.resolve(true),
-    analyzeEpochProofSubmission: () => Promise.resolve(),
+  // Local rerun never publishes — stub the service so submit() always resolves 'published'
+  // and withdraw is a no-op.
+  const publishingService = {
+    submit: () => Promise.resolve('published' as const),
+    withdraw: () => {},
   };
-  const l2BlockSourceForReorgDetection = undefined;
-  const deadline = undefined;
-
-  // This starts a local proving broker that does not get exposed as a service. This should be good enough for
-  // smallish epochs to be proven if we run on a large machine, but as epochs grow larger, we may want to switch
-  // this out for a live proving broker with multiple agents that we can connect to.
   const broker = await createAndStartProvingBroker(config, telemetry);
   const prover = await createProverClient(config, worldState, broker, telemetry);
+  const chonkCache = new ChonkCache(log.getBindings());
+
+  const txProvider = makeReplayingTxProvider(jobData.txs);
+
+  log.info(`Rerunning epoch proving for epoch ${jobData.epochNumber}`);
 
-  const provingJob = new EpochProvingJob(
-    jobData,
-    worldState,
-    prover.createEpochProver(),
-    publicProcessorFactory,
-    publisher,
-    l2BlockSourceForReorgDetection,
+  const provers: CheckpointProver[] = [];
+  for (let i = 0; i < jobData.checkpoints.length; i++) {
+    const checkpoint = jobData.checkpoints[i];
+    const previousBlockHeader =
+      i === 0 ? jobData.previousBlockHeader : jobData.checkpoints[i - 1].blocks.at(-1)!.header;
+    const l1ToL2Messages = jobData.l1ToL2Messages[checkpoint.number] ?? [];
+    const previousArchiveSiblingPath = await getLastSiblingPath(
+      MerkleTreeId.ARCHIVE,
+      worldState.getSnapshot(BlockNumber(checkpoint.blocks[0].number - 1)),
+    );
+    const attestations = checkpoint.number === jobData.checkpoints.at(-1)!.number ? jobData.attestations : [];
+    provers.push(
+      new CheckpointProver(
+        {
+          checkpoint,
+          epochNumber: jobData.epochNumber,
+          attestations,
+          previousBlockHeader,
+          l1ToL2Messages,
+          previousArchiveSiblingPath,
+        },
+        {
+          proverFactory: prover,
+          chonkCache,
+          publicProcessorFactory,
+          dbProvider: worldState,
+          txProvider,
+          dateProvider: new DateProvider(),
+          proverId: prover.getProverId(),
+          metrics,
+          txGatheringTimeoutMs: 120_000,
+          deadline: undefined,
+          log,
+        },
+      ),
+    );
+  }
+
+  const l1Constants = { epochDuration: config.aztecEpochDuration };
+  const [fromSlot, toSlot] = getSlotRangeForEpoch(jobData.epochNumber, l1Constants);
+  const spec: SessionSpec = { kind: 'full', epochNumber: jobData.epochNumber, fromSlot, toSlot };
+
+  const session = new EpochSession(spec, provers, {
+    proverFactory: prover,
+    proverId: prover.getProverId(),
+    publishingService,
     metrics,
-    deadline,
-    { skipEpochCheck: true },
-    log.getBindings(),
-  );
+    dateProvider: new DateProvider(),
+    deadline: undefined,
+    config: {},
+    bindings: log.getBindings(),
+  });
+
+  const finalState = await session.start();
+  log.info(`Completed proving for epoch ${jobData.epochNumber} with status ${finalState}`, {
+    derivedEpoch: getEpochAtSlot(provers[0].slotNumber, l1Constants),
+  });
+  return finalState;
+}
 
-  log.info(`Rerunning epoch proving job for epoch ${jobData.epochNumber}`);
-  await provingJob.run();
-  log.info(`Completed job for epoch ${jobData.epochNumber} with status ${provingJob.getState()}`);
-  return provingJob.getState();
+/** Build a synthetic ITxProvider that returns the supplied txs map by lookup. */
+function makeReplayingTxProvider(txs: Map<string, Tx>): ITxProvider {
+  const lookup = (hashes: TxHash[]) => {
+    const found: Tx[] = [];
+    const missing: TxHash[] = [];
+    for (const hash of hashes) {
+      const tx = txs.get(hash.toString());
+      if (tx) {
+        found.push(tx);
+      } else {
+        missing.push(hash);
+      }
+    }
+    return { txs: found, missingTxs: missing };
+  };
+  return {
+    getAvailableTxs: hashes => Promise.resolve(lookup(hashes)),
+    hasTxs: hashes => Promise.resolve(hashes.map(h => txs.has(h.toString()))),
+    getTxsForBlockProposal: () => Promise.resolve({ txs: [], missingTxs: [] }),
+    getTxsForBlock: (block: L2Block) => Promise.resolve(lookup(block.body.txEffects.map(e => e.txHash))),
+  };
 }

package/src/checkpoint-store.ts

@@ -0,0 +1,194 @@
+import type { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { Checkpoint } from '@aztec/stdlib/checkpoint';
+import { type L1RollupConstants, getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
+
+import { CheckpointProver, type CheckpointProverArgs, type CheckpointProverDeps } from './job/checkpoint-prover.js';
+
+/** Register-time data needed to construct a `CheckpointProver` (everything except the checkpoint + epoch). */
+export type RegisterCheckpointData = Omit<CheckpointProverArgs, 'checkpoint' | 'epochNumber'>;
+
+/** Factory used by the store to construct new provers. Tests can inject a stub. */
+export type CheckpointProverFactory = (args: CheckpointProverArgs, deps: CheckpointProverDeps) => CheckpointProver;
+
+/**
+ * Prover-node-wide registry of `CheckpointProver` instances, content-addressed by
+ * `(checkpoint number, slot, checkpoint archive root)`.
+ *
+ * The store survives every epoch / session boundary. A prover lives from its first
+ * `addOrUpdate` call until either:
+ *  - its checkpoint is pruned by an L1 reorg (`cancelAndRemoveAboveBlock`), or
+ *  - its epoch's proof-submission window has closed (`reapExpired`), so the proof could no
+ *    longer be accepted on L1 even if produced.
+ *
+ * A prover's sub-tree work forks world-state per block and does not survive a prune of a base
+ * block, so there is nothing to preserve across a reorg: a pruned prover is cancelled and
+ * dropped, and a re-add (even of identical content) constructs a fresh prover.
+ */
+export class CheckpointStore {
+  private readonly provers = new Map<string, CheckpointProver>();
+  /**
+   * Teardowns of provers already removed from `provers` (by prune or reap), awaited on `stop()`.
+   * Keyed by a monotonic id rather than the prover's content id: a prune-then-re-add can leave two
+   * teardowns for the same content id in flight at once, which a content-id key would clobber.
+   */
+  private readonly pendingTeardowns = new Map<number, Promise<void>>();
+  private nextTeardownId = 0;
+  private readonly log: Logger;
+
+  constructor(
+    private readonly l2BlockSource: Pick<L2BlockSource, 'getL1Constants'>,
+    private readonly proverDeps: Omit<CheckpointProverDeps, 'log'>,
+    bindings?: LoggerBindings,
+    private readonly proverFactoryFn: CheckpointProverFactory = (args, deps) => new CheckpointProver(args, deps),
+  ) {
+    this.log = createLogger('prover-node:checkpoint-store', bindings);
+  }
+
+  public start(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  public async stop(): Promise<void> {
+    // Cancel every live prover, then await both their teardown and any still in flight for provers
+    // already removed by a prune or reap.
+    const provers = Array.from(this.provers.values());
+    this.provers.clear();
+    for (const prover of provers) {
+      prover.cancel();
+    }
+    await Promise.allSettled([...provers.map(p => p.whenDone()), ...this.pendingTeardowns.values()]);
+  }
+
+  /**
+   * Tracks the teardown of a prover just removed from the store so `stop()` can await it. The entry
+   * removes itself once teardown settles, so the map stays bounded by the number in flight.
+   */
+  private trackTeardown(prover: CheckpointProver): void {
+    const id = this.nextTeardownId++;
+    const done = prover.whenDone();
+    this.pendingTeardowns.set(id, done);
+    void done.finally(() => this.pendingTeardowns.delete(id));
+  }
+
+  /**
+   * Registers a checkpoint with the store. If a prover already exists for the
+   * `(number, slot, archive root)` content key it is reused (an at-least-once re-registration of
+   * still-canonical content); otherwise a new prover is constructed.
+   */
+  public async addOrUpdate(checkpoint: Checkpoint, data: RegisterCheckpointData): Promise<CheckpointProver> {
+    const l1Constants = await this.l2BlockSource.getL1Constants();
+    const epochNumber = getEpochAtSlot(checkpoint.header.slotNumber, l1Constants);
+    const id = CheckpointProver.idFor(checkpoint);
+
+    const existing = this.provers.get(id);
+    if (existing) {
+      return existing;
+    }
+
+    // At most one canonical checkpoint per slot. A different checkpoint at the same slot means the
+    // caller forgot to prune the old chain before adding the replacement — surface it rather than
+    // silently creating a parallel canonical chain. A pruned checkpoint has already been removed,
+    // so every prover still in the store is canonical.
+    for (const prover of this.provers.values()) {
+      if (prover.slotNumber === checkpoint.header.slotNumber) {
+        throw new Error(
+          `Cannot add checkpoint ${checkpoint.number} (archive ${checkpoint.archive.root}) at slot ${checkpoint.header.slotNumber}: ` +
+            `a different checkpoint already occupies this slot. Prune it first.`,
+        );
+      }
+    }
+
+    const prover = this.proverFactoryFn({ ...data, checkpoint, epochNumber }, { ...this.proverDeps, log: this.log });
+    this.provers.set(id, prover);
+    return prover;
+  }
+
+  /**
+   * Cancels and removes every prover that holds a block above the prune target. A checkpoint is orphaned by a prune to
+   * block `targetBlockNumber` iff its last block sits above the target — including a checkpoint whose range straddles
+   * the target (partially orphaned), which block-range marking catches without boundary ambiguity. Keying off the
+   * surviving block number (rather than a checkpoint number) is correct even when the source has already
+   * re-checkpointed past the divergence: the prune event reports the highest surviving block, which by construction
+   * survives on the source, whereas the source's current checkpointed tip can sit above the prune target.
+   *
+   * The prover's in-flight sub-tree work forks world-state per block and faults once its base block is pruned, so it
+   * cannot be reused; it is cancelled (aborting the fork reads) and dropped. A subsequent re-add constructs a fresh
+   * prover. Returns the removed provers.
+   */
+  public cancelAndRemoveAboveBlock(targetBlockNumber: BlockNumber): CheckpointProver[] {
+    const affected: CheckpointProver[] = [];
+    for (const [id, prover] of Array.from(this.provers.entries())) {
+      const lastBlockNumber = prover.checkpoint.blocks.at(-1)!.number;
+      if (lastBlockNumber > targetBlockNumber) {
+        prover.cancel();
+        this.trackTeardown(prover);
+        this.provers.delete(id);
+        affected.push(prover);
+      }
+    }
+    return affected;
+  }
+
+  /**
+   * Drops provers whose epoch is at or below the supplied expired epoch. Once an epoch's
+   * proof-submission window has closed, its proof can no longer be accepted on L1, so the
+   * prover is no longer needed.
+   */
+  public reapExpired(expiredEpoch: EpochNumber): void {
+    const reaped: { id: string; checkpointNumber: CheckpointNumber; epochNumber: EpochNumber }[] = [];
+    for (const [id, prover] of Array.from(this.provers.entries())) {
+      if (prover.epochNumber <= expiredEpoch) {
+        reaped.push({ id, checkpointNumber: prover.checkpoint.number, epochNumber: prover.epochNumber });
+        prover.cancel({ routine: true });
+        this.trackTeardown(prover);
+        this.provers.delete(id);
+      }
+    }
+    if (reaped.length > 0) {
+      this.log.info(`Reaped ${reaped.length} expired CheckpointProver(s) for expiredEpoch ${expiredEpoch}`, {
+        expiredEpoch,
+        reapedCount: reaped.length,
+        reaped,
+      });
+    }
+  }
+
+  /** Returns the prover with the supplied id, or undefined. */
+  public get(id: string): CheckpointProver | undefined {
+    return this.provers.get(id);
+  }
+
+  /** Returns the prover for the supplied checkpoint (by its content-addressed id), or undefined. */
+  public getByCheckpoint(checkpoint: Checkpoint): CheckpointProver | undefined {
+    return this.provers.get(CheckpointProver.idFor(checkpoint));
+  }
+
+  /** Every prover currently in the store, in insertion order. */
+  public listAll(): CheckpointProver[] {
+    return Array.from(this.provers.values());
+  }
+
+  /** Provers in the store, sorted by checkpoint number. */
+  public list(): CheckpointProver[] {
+    return Array.from(this.provers.values()).sort((a, b) => a.checkpoint.number - b.checkpoint.number);
+  }
+
+  /**
+   * Provers whose slot is in the supplied epoch's slot range, sorted by checkpoint number.
+   */
+  public async listForEpoch(epoch: EpochNumber): Promise<CheckpointProver[]> {
+    const l1Constants = await this.l2BlockSource.getL1Constants();
+    const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants);
+    return this.listInSlotRange(fromSlot, toSlot);
+  }
+
+  /** Provers whose slot falls within `[fromSlot, toSlot]`, sorted by checkpoint number. */
+  public listInSlotRange(fromSlot: SlotNumber, toSlot: SlotNumber): CheckpointProver[] {
+    return this.list().filter(p => p.slotNumber >= fromSlot && p.slotNumber <= toSlot);
+  }
+}
+
+/** Sub-set of `L1RollupConstants` actually consumed by the store's slot helpers. */
+export type CheckpointStoreL1Constants = Pick<L1RollupConstants, 'epochDuration'>;

package/src/config.ts

@@ -6,7 +6,6 @@ import {
   numberConfigHelper,
   pickConfigMappings,
 } from '@aztec/foundation/config';
-import { EthAddress } from '@aztec/foundation/eth-address';
 import { type KeyStoreConfig, keyStoreConfigMappings } from '@aztec/node-keystore/config';
 import { ethPrivateKeySchema } from '@aztec/node-keystore/schemas';
 import type { KeyStore } from '@aztec/node-keystore/types';
@@ -45,7 +44,6 @@ export type SpecificProverNodeConfig = {
   txGatheringIntervalMs: number;
   txGatheringBatchSize: number;
   txGatheringMaxParallelRequestsPerNode: number;
-  proofSubmissionTargetAddress?: EthAddress;
 };
 
 export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProverNodeConfig> = {
@@ -70,7 +68,8 @@ export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProver
     defaultValue: undefined,
   },
   proverNodeEpochProvingDelayMs: {
-    description: 'Optional delay in milliseconds to wait before proving a new epoch',
+    description:
+      'Optional delay in milliseconds to wait for late-arriving events (e.g. reorgs) to settle before starting top-tree proving for an epoch',
     defaultValue: undefined,
   },
   txGatheringIntervalMs: {
@@ -98,14 +97,6 @@ export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProver
     description: 'Whether the prover node skips publishing proofs to L1',
     ...booleanConfigHelper(false),
   },
-  proofSubmissionTargetAddress: {
-    env: 'PROVER_NODE_PROOF_SUBMISSION_TARGET_ADDRESS',
-    description:
-      'Optional L1 address the submitEpochRootProof tx is sent to. Must expose the identical submitEpochRootProof ABI ' +
-      'and forward to the rollup. Defaults to the rollup address.',
-    parseEnv: (val: string) => EthAddress.fromString(val),
-    defaultValue: undefined,
-  },
 };
 
 export const proverNodeConfigMappings: ConfigMappingsType<ProverNodeConfig> = {

package/src/factory.ts

@@ -31,7 +31,6 @@ import { L1Metrics, type TelemetryClient, getTelemetryClient } from '@aztec/tele
 import { createPublicClient } from 'viem';
 
 import type { SpecificProverNodeConfig } from './config.js';
-import { EpochMonitor } from './monitors/epoch-monitor.js';
 import { ProverNode } from './prover-node.js';
 import { ProverPublisherFactory } from './prover-publisher-factory.js';
 
@@ -136,7 +135,6 @@ export async function createProverNode(
     deps.publisherFactory ??
     new ProverPublisherFactory(config, {
       rollupContract,
-      proofSubmissionTarget: config.proofSubmissionTargetAddress,
       publisherManager: new PublisherManager(l1TxUtils, getPublisherConfigFromProverConfig(config), {
         bindings: log.getBindings(),
         funder: funderL1TxUtils,
@@ -162,12 +160,6 @@ export async function createProverNode(
     ),
   };
 
-  const epochMonitor = await EpochMonitor.create(
-    archiver,
-    { pollingIntervalMs: config.proverNodePollingIntervalMs, provingDelayMs: config.proverNodeEpochProvingDelayMs },
-    telemetry,
-  );
-
   const l1Metrics = new L1Metrics(
     telemetry.getMeter('ProverNodeL1Metrics'),
     publicClient,
@@ -185,7 +177,6 @@ export async function createProverNode(
     archiver,
     worldStateSynchronizer,
     p2pClient,
-    epochMonitor,
     rollupContract,
     l1Metrics,
     proverNodeConfig,

package/src/index.ts

@@ -2,5 +2,6 @@ export * from './actions/index.js';
 export * from './config.js';
 export * from './factory.js';
 export * from './monitors/index.js';
+export * from './proof-publishing-service.js';
 export * from './prover-node-publisher.js';
 export * from './prover-node.js';