@aztec/prover-node 0.0.1-commit.2eb6648a → 0.0.1-commit.2f68f620

package/src/prover-node.ts

@@ -1,13 +1,12 @@
 import type { Archiver } from '@aztec/archiver';
 import type { RollupContract } from '@aztec/ethereum/contracts';
+import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
 import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
 import type { Fr } from '@aztec/foundation/curves/bn254';
 import { memoize } from '@aztec/foundation/decorators';
 import { createLogger } from '@aztec/foundation/log';
 import { DateProvider } from '@aztec/foundation/timer';
-import type { DataStoreConfig } from '@aztec/kv-store/config';
-import type { P2PClient } from '@aztec/p2p';
 import { PublicProcessorFactory } from '@aztec/simulator/server';
 import type { L2BlockSource } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
@@ -16,15 +15,15 @@ import type { ContractDataSource } from '@aztec/stdlib/contract';
 import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import {
   type EpochProverManager,
-  EpochProvingJobTerminalState,
+  type ITxProvider,
   type ProverNodeApi,
   type Service,
   type WorldStateSyncStatus,
   type WorldStateSynchronizer,
   tryStop,
 } from '@aztec/stdlib/interfaces/server';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
-import type { P2PClientType } from '@aztec/stdlib/p2p';
 import type { Tx } from '@aztec/stdlib/tx';
 import {
   Attributes,
@@ -39,7 +38,7 @@ import {
 import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
 import type { SpecificProverNodeConfig } from './config.js';
 import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
-import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
+import { EpochProvingJob, type EpochProvingJobOptions, type EpochProvingJobState } from './job/epoch-proving-job.js';
 import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
 import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
 import type { ProverNodePublisher } from './prover-node-publisher.js';
@@ -55,12 +54,12 @@ type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfi
  */
 export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
   private log = createLogger('prover-node');
-  private dateProvider = new DateProvider();
 
   private jobs: Map<string, EpochProvingJob> = new Map();
   private config: ProverNodeOptions;
   private jobMetrics: ProverNodeJobMetrics;
   private rewardsMetrics: ProverNodeRewardsMetrics;
+  private startingProofEpochs: Set<EpochNumber> = new Set();
 
   public readonly tracer: Tracer;
 
@@ -73,17 +72,19 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly contractDataSource: ContractDataSource,
     protected readonly worldState: WorldStateSynchronizer,
-    protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
+    protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
     protected readonly epochsMonitor: EpochMonitor,
     protected readonly rollupContract: RollupContract,
     protected readonly l1Metrics: L1Metrics,
     config: Partial<ProverNodeOptions> = {},
     protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
+    private delayer?: Delayer,
+    private readonly dateProvider: DateProvider = new DateProvider(),
   ) {
     this.config = {
       proverNodePollingIntervalMs: 1_000,
       proverNodeMaxPendingJobs: 100,
-      proverNodeMaxParallelBlocksPerEpoch: 32,
+      proverNodeMaxParallelBlocksPerEpoch: 0,
       txGatheringIntervalMs: 1_000,
       txGatheringBatchSize: 10,
       txGatheringMaxParallelRequestsPerNode: 100,
@@ -111,6 +112,11 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     return this.p2pClient;
   }
 
+  /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */
+  public getDelayer(): Delayer | undefined {
+    return this.delayer;
+  }
+
   /**
    * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
    * @param epochNumber - The epoch number that was just completed.
@@ -121,15 +127,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
       this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
         jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
       });
-      const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
-      if (activeJobs.length > 0) {
-        this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
-          activeJobs: activeJobs.map(job => job.uuid),
-        });
-        return true;
-      }
-      await this.startProof(epochNumber);
-      return true;
+      return await this.startProofIfNeeded(epochNumber);
     } catch (err) {
       if (err instanceof EmptyEpochError) {
         this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
@@ -155,17 +153,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
 
   /**
    * Stops the prover node and all its dependencies.
+   * Resources not owned by this node (shared with the parent aztec-node) are skipped.
    */
   async stop() {
     this.log.info('Stopping ProverNode');
     await this.epochsMonitor.stop();
-    await this.prover.stop();
-    await tryStop(this.p2pClient);
-    await tryStop(this.l2BlockSource);
-    await tryStop(this.publisherFactory);
     this.publisher?.interrupt();
     await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
-    await this.worldState.stop();
+    await this.prover.stop();
+    await tryStop(this.publisherFactory);
     this.rewardsMetrics.stop();
     this.l1Metrics.stop();
     await this.telemetryClient.stop();
@@ -187,7 +183,38 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    * Starts a proving process and returns immediately.
    */
   public async startProof(epochNumber: EpochNumber) {
-    const job = await this.createProvingJob(epochNumber, { skipEpochCheck: true });
+    await this.startProofIfNeeded(epochNumber, { skipEpochCheck: true });
+  }
+
+  private async startProofIfNeeded(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}): Promise<boolean> {
+    if (this.startingProofEpochs.has(epochNumber)) {
+      this.log.warn(`Not starting proof for ${epochNumber} since a proof is already being started for the epoch`, {
+        epochNumber,
+      });
+      return false;
+    }
+
+    this.startingProofEpochs.add(epochNumber);
+
+    try {
+      const activeJobs = await this.activeJobsCoverEpoch(epochNumber);
+      if (activeJobs.length > 0) {
+        this.log.warn(`Not starting proof for ${epochNumber} since an active job already covers the epoch`, {
+          epochNumber,
+          activeJobs,
+        });
+        return true;
+      }
+
+      await this.startProofInternal(epochNumber, opts);
+      return true;
+    } finally {
+      this.startingProofEpochs.delete(epochNumber);
+    }
+  }
+
+  private async startProofInternal(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}) {
+    const job = await this.createProvingJob(epochNumber, opts);
     void this.runJob(job);
   }
 
@@ -241,20 +268,42 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
    * Returns an array of jobs being processed.
    */
   public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
-    return Promise.resolve(
-      Array.from(this.jobs.entries()).map(([uuid, job]) => ({
-        uuid,
-        status: job.getState(),
-        epochNumber: job.getEpochNumber(),
-      })),
-    );
+    return Promise.resolve(this.getJobsInternal());
   }
 
-  protected async getActiveJobsForEpoch(
-    epochNumber: EpochNumber,
-  ): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
-    const jobs = await this.getJobs();
-    return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
+  private getJobsInternal(): { uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[] {
+    return Array.from(this.jobs.entries()).map(([uuid, job]) => ({
+      uuid,
+      status: job.getState(),
+      epochNumber: job.getEpochNumber(),
+    }));
+  }
+
+  private async activeJobsCoverEpoch(epochNumber: EpochNumber): Promise<string[]> {
+    const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: epochNumber });
+    if (checkpoints.length === 0) {
+      return [];
+    }
+
+    const firstCheckpoint = checkpoints.at(0)!.checkpointNumber;
+    const latestCheckpoint = checkpoints.at(-1)!.checkpointNumber;
+
+    const jobs: string[] = [];
+    for (const job of this.jobs.values()) {
+      if (job.getEpochNumber() !== epochNumber) {
+        continue;
+      }
+
+      const jobCheckpoints = job.getProvingData().checkpoints;
+      const checkpointOverlap =
+        jobCheckpoints.at(0)!.number <= firstCheckpoint && jobCheckpoints.at(-1)!.number >= latestCheckpoint;
+
+      if (checkpointOverlap && !['failed', 'stopped', 'timed-out'].includes(job.getState())) {
+        jobs.push(job.getId());
+      }
+    }
+
+    return jobs;
   }
 
   private checkMaximumPendingJobs() {
@@ -275,13 +324,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
     const fromCheckpoint = epochData.checkpoints[0].number;
     const toCheckpoint = epochData.checkpoints.at(-1)!.number;
     const fromBlock = epochData.checkpoints[0].blocks[0].number;
-    const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
+    const lastBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!;
+    const toBlock = lastBlock.number;
     this.log.verbose(
       `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
     );
 
     // Fast forward world state to right before the target block and get a fork
-    await this.worldState.syncImmediate(toBlock);
+    const lastBlockHash = await lastBlock.header.hash();
+    await this.worldState.syncImmediate(toBlock, lastBlockHash);
 
     // Create a processor factory
     const publicProcessorFactory = new PublicProcessorFactory(
@@ -306,26 +357,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
 
   @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
   private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
-    const checkpoints = await this.gatherCheckpoints(epochNumber);
+    const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({ epoch: epochNumber });
+    if (publishedCheckpoints.length === 0) {
+      throw new EmptyEpochError(epochNumber);
+    }
+    const checkpoints = publishedCheckpoints.map(p => p.checkpoint);
+    const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
     const txArray = await this.gatherTxs(epochNumber, checkpoints);
     const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
     const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
     const [firstBlock] = checkpoints[0].blocks;
     const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
-    const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
-    const attestations = lastPublishedCheckpoint?.attestations ?? [];
 
     return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
   }
 
-  private async gatherCheckpoints(epochNumber: EpochNumber) {
-    const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
-    if (checkpoints.length === 0) {
-      throw new EmptyEpochError(epochNumber);
-    }
-    return checkpoints;
-  }
-
   private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
     const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
     const txProvider = this.p2pClient.getTxProvider();
@@ -354,16 +400,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   }
 
   private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
-    const header = await (previousBlockNumber === 0
-      ? this.worldState.getCommitted().getInitialHeader()
-      : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
-
-    if (!header) {
+    const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
+    if (!data?.header) {
       throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
     }
 
-    this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
-    return header;
+    this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
+    return data.header;
   }
 
   /** Extracted for testing purposes. */

package/src/prover-publisher-factory.ts

@@ -1,29 +1,31 @@
 import type { RollupContract } from '@aztec/ethereum/contracts';
 import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import type { LoggerBindings } from '@aztec/foundation/log';
-import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
+import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
 import type { TelemetryClient } from '@aztec/telemetry-client';
 
 import { ProverNodePublisher } from './prover-node-publisher.js';
 
 export class ProverPublisherFactory {
   constructor(
-    private config: TxSenderConfig & PublisherConfig,
+    private config: ProverTxSenderConfig & ProverPublisherConfig,
     private deps: {
       rollupContract: RollupContract;
       publisherManager: PublisherManager<L1TxUtils>;
+      proofSubmissionTarget?: EthAddress;
       telemetry?: TelemetryClient;
     },
     private bindings?: LoggerBindings,
   ) {}
 
   public async start() {
-    await this.deps.publisherManager.loadState();
+    await this.deps.publisherManager.start();
   }
 
-  public stop() {
-    this.deps.publisherManager.interrupt();
+  public async stop() {
+    await this.deps.publisherManager.stop();
   }
 
   /**
@@ -37,6 +39,7 @@ export class ProverPublisherFactory {
       {
         rollupContract: this.deps.rollupContract,
         l1TxUtils: l1Publisher,
+        proofSubmissionTarget: this.deps.proofSubmissionTarget,
         telemetry: this.deps.telemetry,
       },
       this.bindings,