@aztec/prover-node 0.0.1-commit.179ed20 → 0.0.1-commit.189eedb3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dest/actions/download-epoch-proving-job.js +1 -1
  2. package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
  3. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  4. package/dest/actions/rerun-epoch-proving-job.js +105 -27
  5. package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
  6. package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
  7. package/dest/bin/run-failed-epoch.js +6 -5
  8. package/dest/config.d.ts +5 -8
  9. package/dest/config.d.ts.map +1 -1
  10. package/dest/config.js +16 -19
  11. package/dest/factory.d.ts +19 -13
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +40 -60
  14. package/dest/job/epoch-proving-job.d.ts +6 -2
  15. package/dest/job/epoch-proving-job.d.ts.map +1 -1
  16. package/dest/job/epoch-proving-job.js +206 -35
  17. package/dest/metrics.d.ts +21 -1
  18. package/dest/metrics.d.ts.map +1 -1
  19. package/dest/metrics.js +47 -0
  20. package/dest/monitors/epoch-monitor.d.ts +1 -1
  21. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  22. package/dest/monitors/epoch-monitor.js +11 -9
  23. package/dest/prover-node-publisher.d.ts +22 -3
  24. package/dest/prover-node-publisher.d.ts.map +1 -1
  25. package/dest/prover-node-publisher.js +202 -7
  26. package/dest/prover-node.d.ts +24 -14
  27. package/dest/prover-node.d.ts.map +1 -1
  28. package/dest/prover-node.js +91 -41
  29. package/dest/prover-publisher-factory.d.ts +4 -4
  30. package/dest/prover-publisher-factory.d.ts.map +1 -1
  31. package/dest/prover-publisher-factory.js +3 -3
  32. package/package.json +23 -22
  33. package/src/actions/download-epoch-proving-job.ts +1 -1
  34. package/src/actions/rerun-epoch-proving-job.ts +17 -6
  35. package/src/actions/upload-epoch-proof-failure.ts +1 -1
  36. package/src/bin/run-failed-epoch.ts +5 -3
  37. package/src/config.ts +23 -31
  38. package/src/factory.ts +68 -103
  39. package/src/job/epoch-proving-job.ts +137 -41
  40. package/src/metrics.ts +71 -0
  41. package/src/monitors/epoch-monitor.ts +5 -6
  42. package/src/prover-node-publisher.ts +235 -12
  43. package/src/prover-node.ts +98 -55
  44. package/src/prover-publisher-factory.ts +5 -5
@@ -69,9 +69,8 @@ export class EpochMonitor implements Traceable {
69
69
 
70
70
  public async work() {
71
71
  const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
72
- this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
73
72
  if (epochToProve === undefined) {
74
- this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
73
+ this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
75
74
  return;
76
75
  }
77
76
  if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -86,20 +85,20 @@ export class EpochMonitor implements Traceable {
86
85
  }
87
86
 
88
87
  if (this.options.provingDelayMs) {
89
- this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
88
+ this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
90
89
  await sleep(this.options.provingDelayMs);
91
90
  }
92
91
 
93
- this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
92
+ this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
94
93
  if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
95
94
  this.latestEpochNumber = epochToProve;
96
95
  }
97
96
  }
98
97
 
99
98
  private async getEpochNumberToProve() {
100
- const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
99
+ const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
101
100
  const firstBlockToProve = BlockNumber(lastBlockProven + 1);
102
- const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
101
+ const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
103
102
  if (!firstBlockHeaderToProve) {
104
103
  return { epochToProve: undefined, blockNumber: firstBlockToProve };
105
104
  }
@@ -1,5 +1,5 @@
1
1
  import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
2
- import { AZTEC_MAX_EPOCH_DURATION } from '@aztec/constants';
2
+ import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
3
3
  import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
4
4
  import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
5
5
  import { makeTuple } from '@aztec/foundation/array';
@@ -8,20 +8,22 @@ import { areArraysEqual } from '@aztec/foundation/collection';
8
8
  import { Fr } from '@aztec/foundation/curves/bn254';
9
9
  import { EthAddress } from '@aztec/foundation/eth-address';
10
10
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
11
+ import { retryUntil } from '@aztec/foundation/retry';
11
12
  import type { Tuple } from '@aztec/foundation/serialize';
12
13
  import { Timer } from '@aztec/foundation/timer';
13
14
  import { RollupAbi } from '@aztec/l1-artifacts';
14
15
  import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
15
16
  import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
17
+ import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
16
18
  import type { Proof } from '@aztec/stdlib/proofs';
17
19
  import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
18
20
  import type { L1PublishProofStats } from '@aztec/stdlib/stats';
19
21
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
20
22
 
21
23
  import { inspect } from 'util';
22
- import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem';
24
+ import { type Hex, type TransactionReceipt, encodeFunctionData, formatEther, formatGwei } from 'viem';
23
25
 
24
- import { ProverNodePublisherMetrics } from './metrics.js';
26
+ import { type EstimatedSubmitProofStats, ProverNodePublisherMetrics } from './metrics.js';
25
27
 
26
28
  /** Arguments to the submitEpochProof method of the rollup contract */
27
29
  export type L1SubmitEpochProofArgs = {
@@ -31,7 +33,7 @@ export type L1SubmitEpochProofArgs = {
31
33
  endTimestamp: Fr;
32
34
  outHash: Fr;
33
35
  proverId: Fr;
34
- fees: Tuple<FeeRecipient, typeof AZTEC_MAX_EPOCH_DURATION>;
36
+ fees: Tuple<FeeRecipient, typeof MAX_CHECKPOINTS_PER_EPOCH>;
35
37
  proof: Proof;
36
38
  };
37
39
 
@@ -101,6 +103,11 @@ export class ProverNodePublisher {
101
103
  const ctx = { epochNumber, fromCheckpoint, toCheckpoint };
102
104
 
103
105
  if (!this.interrupted) {
106
+ if (!(await this.waitUntilStartBuildsOnProven(args))) {
107
+ this.log.verbose('Checkpoint data syncing interrupted', ctx);
108
+ return false;
109
+ }
110
+
104
111
  const timer = new Timer();
105
112
  // Validate epoch proof range and hashes are correct before submitting
106
113
  await this.validateEpochProofSubmission(args);
@@ -147,6 +154,53 @@ export class ProverNodePublisher {
147
154
  return false;
148
155
  }
149
156
 
157
+ private async waitUntilStartBuildsOnProven(args: { epochNumber: EpochNumber; fromCheckpoint: CheckpointNumber }) {
158
+ const { epochNumber, fromCheckpoint } = args;
159
+ const provenCheckpoint = await this.getProvenCheckpoint();
160
+ if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
161
+ return true;
162
+ }
163
+
164
+ const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
165
+ this.log.info(`Waiting for proven checkpoint to reach proof start`, {
166
+ epochNumber,
167
+ fromCheckpoint,
168
+ provenCheckpoint,
169
+ timeout,
170
+ });
171
+
172
+ await retryUntil(
173
+ async () => {
174
+ if (this.interrupted) {
175
+ return true;
176
+ }
177
+
178
+ const proven = await this.getProvenCheckpoint();
179
+ this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, { epochNumber });
180
+ return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
181
+ },
182
+ `proven checkpoint to reach ${fromCheckpoint - 1}`,
183
+ timeout,
184
+ 4,
185
+ );
186
+
187
+ return !this.interrupted;
188
+ }
189
+
190
+ private async getProvenCheckpoint() {
191
+ return (await this.rollupContract.getTips()).proven;
192
+ }
193
+
194
+ private isStartBuildingOnProven(fromCheckpoint: CheckpointNumber, provenCheckpoint: CheckpointNumber) {
195
+ return fromCheckpoint - 1 <= provenCheckpoint;
196
+ }
197
+
198
+ private async getSecondsUntilProofSubmissionWindowEnd(epochNumber: EpochNumber) {
199
+ const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
200
+ const now = BigInt(Math.floor(Date.now() / 1000));
201
+ return Math.max(Number(deadline - now), 0.001);
202
+ }
203
+
150
204
  private async validateEpochProofSubmission(args: {
151
205
  fromCheckpoint: CheckpointNumber;
152
206
  toCheckpoint: CheckpointNumber;
@@ -168,7 +222,7 @@ export class ProverNodePublisher {
168
222
  // toCheckpoint can't be greater than pending
169
223
  if (toCheckpoint > pending) {
170
224
  throw new Error(
171
- `Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as pending checkpoint is ${pending}`,
225
+ `Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as proposed checkpoint is ${pending}`,
172
226
  );
173
227
  }
174
228
 
@@ -203,13 +257,85 @@ export class ProverNodePublisher {
203
257
  const argsPublicInputs = [...publicInputs.toFields()];
204
258
 
205
259
  if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b) => a.equals(b))) {
206
- const fmt = (inputs: Fr[] | readonly string[]) => inputs.map(x => x.toString()).join(', ');
207
- throw new Error(
208
- `Root rollup public inputs mismatch:\nRollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
209
- );
260
+ throw await reportPublicInputsMismatch({
261
+ rollupPublicInputs,
262
+ argsPublicInputs,
263
+ fromCheckpoint,
264
+ toCheckpoint,
265
+ rollupContract: this.rollupContract,
266
+ log: this.log,
267
+ });
210
268
  }
211
269
  }
212
270
 
271
+ /**
272
+ * Estimates what submitting the epoch proof would have cost on L1 without actually sending it.
273
+ * Runs the same validation as `submitEpochProof`, encodes the calldata, estimates gas, and records metrics.
274
+ * Used when proof publishing is disabled (e.g. PROVER_NODE_DISABLE_PROOF_PUBLISH=true on mainnet).
275
+ */
276
+ public async analyzeEpochProofSubmission(args: {
277
+ epochNumber: EpochNumber;
278
+ fromCheckpoint: CheckpointNumber;
279
+ toCheckpoint: CheckpointNumber;
280
+ publicInputs: RootRollupPublicInputs;
281
+ proof: Proof;
282
+ batchedBlobInputs: BatchedBlob;
283
+ attestations: ViemCommitteeAttestation[];
284
+ }): Promise<void> {
285
+ const { epochNumber, fromCheckpoint, toCheckpoint } = args;
286
+
287
+ await this.validateEpochProofSubmission(args);
288
+
289
+ const data = this.encodeSubmitEpochProofCalldata(args);
290
+ const senderAddress = this.l1TxUtils.getSenderAddress();
291
+
292
+ const [gasLimit, gasPrice, latestBlock] = await Promise.all([
293
+ this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.rollupContract.address, data }),
294
+ this.l1TxUtils.getGasPrice(),
295
+ this.l1TxUtils.client.getBlock({ blockTag: 'latest' }),
296
+ ]);
297
+
298
+ const baseFeePerGas = latestBlock.baseFeePerGas ?? 0n;
299
+ const { maxPriorityFeePerGas } = gasPrice;
300
+
301
+ const effectiveFeePerGas = baseFeePerGas + maxPriorityFeePerGas;
302
+ const estimatedTotalFee = gasLimit * effectiveFeePerGas;
303
+
304
+ const stats: EstimatedSubmitProofStats = {
305
+ gasLimit,
306
+ baseFeePerGas,
307
+ maxPriorityFeePerGas,
308
+ estimatedTotalFee,
309
+ };
310
+
311
+ this.log.info(`Estimated epoch proof submission cost (not submitted)`, {
312
+ epochNumber,
313
+ fromCheckpoint,
314
+ toCheckpoint,
315
+ gasLimit: gasLimit.toString(),
316
+ baseFeePerGas: formatGwei(baseFeePerGas),
317
+ maxPriorityFeePerGas: formatGwei(maxPriorityFeePerGas),
318
+ estimatedTotalFeeEth: formatEther(estimatedTotalFee),
319
+ });
320
+
321
+ this.metrics.recordEstimatedSubmitProof(stats);
322
+ }
323
+
324
+ private encodeSubmitEpochProofCalldata(args: {
325
+ fromCheckpoint: CheckpointNumber;
326
+ toCheckpoint: CheckpointNumber;
327
+ publicInputs: RootRollupPublicInputs;
328
+ proof: Proof;
329
+ batchedBlobInputs: BatchedBlob;
330
+ attestations: ViemCommitteeAttestation[];
331
+ }): Hex {
332
+ return encodeFunctionData({
333
+ abi: RollupAbi,
334
+ functionName: 'submitEpochRootProof',
335
+ args: [this.getSubmitEpochProofArgs(args)],
336
+ });
337
+ }
338
+
213
339
  private async sendSubmitEpochProofTx(args: {
214
340
  fromCheckpoint: CheckpointNumber;
215
341
  toCheckpoint: CheckpointNumber;
@@ -271,7 +397,7 @@ export class ProverNodePublisher {
271
397
  outHash: args.publicInputs.outHash.toString(),
272
398
  proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
273
399
  } /*_args*/,
274
- makeTuple(AZTEC_MAX_EPOCH_DURATION * 2, i =>
400
+ makeTuple(MAX_CHECKPOINTS_PER_EPOCH * 2, i =>
275
401
  i % 2 === 0
276
402
  ? args.publicInputs.fees[i / 2].recipient.toField().toString()
277
403
  : args.publicInputs.fees[(i - 1) / 2].value.toString(),
@@ -296,11 +422,108 @@ export class ProverNodePublisher {
296
422
  end: argsArray[1],
297
423
  args: argsArray[2],
298
424
  fees: argsArray[3],
299
- attestations: new CommitteeAttestationsAndSigners(
425
+ attestations: CommitteeAttestationsAndSigners.packAttestations(
300
426
  args.attestations.map(a => CommitteeAttestation.fromViem(a)),
301
- ).getPackedAttestations(),
427
+ ),
302
428
  blobInputs: argsArray[4],
303
429
  proof: proofHex,
304
430
  };
305
431
  }
306
432
  }
433
+
434
+ /**
435
+ * Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
436
+ * mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
437
+ * Error with a human-readable summary.
438
+ *
439
+ * Layout of `RootRollupPublicInputs.toFields()`:
440
+ * [0] previousArchiveRoot
441
+ * [1] endArchiveRoot
442
+ * [2] outHash
443
+ * [3 .. 3+N-1] checkpointHeaderHashes[i] for i in 0..N-1 (N = MAX_CHECKPOINTS_PER_EPOCH)
444
+ * [3+N .. 3+3N-1] fees[i] = (recipient, value) for i in 0..N-1
445
+ * [3+3N .. 3+3N+4] EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
446
+ * [3+3N+5 ..] blobPublicInputs (FinalBlobAccumulator)
447
+ */
448
+ async function reportPublicInputsMismatch(input: {
449
+ rollupPublicInputs: readonly Fr[];
450
+ argsPublicInputs: readonly Fr[];
451
+ fromCheckpoint: CheckpointNumber;
452
+ toCheckpoint: CheckpointNumber;
453
+ rollupContract: RollupContract;
454
+ log: Logger;
455
+ }): Promise<Error> {
456
+ const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
457
+ const N = MAX_CHECKPOINTS_PER_EPOCH;
458
+ const constantsStart = 3 + 3 * N;
459
+ const blobStart = constantsStart + 5;
460
+ const constantLabels = ['chainId', 'version', 'vkTreeRoot', 'protocolContractsHash', 'proverId'];
461
+
462
+ const diffs: { index: number; label: string; rollup: Fr; computed: Fr; checkpointIndex?: number }[] = [];
463
+ const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
464
+ for (let i = 0; i < len; i++) {
465
+ const a = rollupPublicInputs[i] ?? Fr.ZERO;
466
+ const b = argsPublicInputs[i] ?? Fr.ZERO;
467
+ if (a.equals(b)) {
468
+ continue;
469
+ }
470
+ let label: string;
471
+ let checkpointIndex: number | undefined;
472
+ if (i === 0) {
473
+ label = 'previousArchiveRoot';
474
+ } else if (i === 1) {
475
+ label = 'endArchiveRoot';
476
+ } else if (i === 2) {
477
+ label = 'outHash';
478
+ } else if (i < 3 + N) {
479
+ checkpointIndex = i - 3;
480
+ label = `checkpointHeaderHashes[${checkpointIndex}]`;
481
+ } else if (i < 3 + 3 * N) {
482
+ const feePairIndex = i - (3 + N);
483
+ const feeIndex = Math.floor(feePairIndex / 2);
484
+ const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
485
+ label = `fees[${feeIndex}].${sub}`;
486
+ } else if (i < blobStart) {
487
+ label = `constants.${constantLabels[i - constantsStart]}`;
488
+ } else {
489
+ label = `blobPublicInputs[${i - blobStart}]`;
490
+ }
491
+ diffs.push({ index: i, label, rollup: a, computed: b, checkpointIndex });
492
+ }
493
+
494
+ // For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
495
+ // see what was published on-chain alongside the prover's recomputed hash.
496
+ const onChainCheckpoints = await Promise.all(
497
+ diffs
498
+ .filter(d => d.checkpointIndex !== undefined)
499
+ .map(async d => {
500
+ const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex!);
501
+ try {
502
+ const cp = await rollupContract.getCheckpoint(checkpointNumber);
503
+ return { checkpointIndex: d.checkpointIndex!, checkpointNumber, headerHash: cp.headerHash.toString() };
504
+ } catch (err) {
505
+ return { checkpointIndex: d.checkpointIndex!, checkpointNumber, error: (err as Error).message };
506
+ }
507
+ }),
508
+ );
509
+
510
+ log.error(`Root rollup public inputs mismatch`, undefined, {
511
+ fromCheckpoint,
512
+ toCheckpoint,
513
+ numDiffs: diffs.length,
514
+ diffs: diffs.map(d => ({
515
+ index: d.index,
516
+ label: d.label,
517
+ rollup: d.rollup.toString(),
518
+ computed: d.computed.toString(),
519
+ })),
520
+ onChainCheckpoints,
521
+ });
522
+
523
+ const fmt = (inputs: readonly Fr[]) => inputs.map(x => x.toString()).join(', ');
524
+ const summary = diffs.map(d => `[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
525
+ return new Error(
526
+ `Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` +
527
+ `Rollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
528
+ );
529
+ }
@@ -1,13 +1,12 @@
1
1
  import type { Archiver } from '@aztec/archiver';
2
2
  import type { RollupContract } from '@aztec/ethereum/contracts';
3
+ import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
3
4
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
4
5
  import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
5
6
  import type { Fr } from '@aztec/foundation/curves/bn254';
6
7
  import { memoize } from '@aztec/foundation/decorators';
7
8
  import { createLogger } from '@aztec/foundation/log';
8
9
  import { DateProvider } from '@aztec/foundation/timer';
9
- import type { DataStoreConfig } from '@aztec/kv-store/config';
10
- import type { P2PClient } from '@aztec/p2p';
11
10
  import { PublicProcessorFactory } from '@aztec/simulator/server';
12
11
  import type { L2BlockSource } from '@aztec/stdlib/block';
13
12
  import type { Checkpoint } from '@aztec/stdlib/checkpoint';
@@ -16,15 +15,15 @@ import type { ContractDataSource } from '@aztec/stdlib/contract';
16
15
  import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
17
16
  import {
18
17
  type EpochProverManager,
19
- EpochProvingJobTerminalState,
18
+ type ITxProvider,
20
19
  type ProverNodeApi,
21
20
  type Service,
22
21
  type WorldStateSyncStatus,
23
22
  type WorldStateSynchronizer,
24
23
  tryStop,
25
24
  } from '@aztec/stdlib/interfaces/server';
25
+ import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
26
26
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
27
- import type { P2PClientType } from '@aztec/stdlib/p2p';
28
27
  import type { Tx } from '@aztec/stdlib/tx';
29
28
  import {
30
29
  Attributes,
@@ -39,7 +38,7 @@ import {
39
38
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
40
39
  import type { SpecificProverNodeConfig } from './config.js';
41
40
  import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
42
- import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
41
+ import { EpochProvingJob, type EpochProvingJobOptions, type EpochProvingJobState } from './job/epoch-proving-job.js';
43
42
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
44
43
  import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
45
44
  import type { ProverNodePublisher } from './prover-node-publisher.js';
@@ -55,12 +54,12 @@ type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfi
55
54
  */
56
55
  export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
57
56
  private log = createLogger('prover-node');
58
- private dateProvider = new DateProvider();
59
57
 
60
58
  private jobs: Map<string, EpochProvingJob> = new Map();
61
59
  private config: ProverNodeOptions;
62
60
  private jobMetrics: ProverNodeJobMetrics;
63
61
  private rewardsMetrics: ProverNodeRewardsMetrics;
62
+ private startingProofEpochs: Set<EpochNumber> = new Set();
64
63
 
65
64
  public readonly tracer: Tracer;
66
65
 
@@ -73,17 +72,19 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
73
72
  protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
74
73
  protected readonly contractDataSource: ContractDataSource,
75
74
  protected readonly worldState: WorldStateSynchronizer,
76
- protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
75
+ protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
77
76
  protected readonly epochsMonitor: EpochMonitor,
78
77
  protected readonly rollupContract: RollupContract,
79
78
  protected readonly l1Metrics: L1Metrics,
80
79
  config: Partial<ProverNodeOptions> = {},
81
80
  protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
81
+ private delayer?: Delayer,
82
+ private readonly dateProvider: DateProvider = new DateProvider(),
82
83
  ) {
83
84
  this.config = {
84
85
  proverNodePollingIntervalMs: 1_000,
85
86
  proverNodeMaxPendingJobs: 100,
86
- proverNodeMaxParallelBlocksPerEpoch: 32,
87
+ proverNodeMaxParallelBlocksPerEpoch: 0,
87
88
  txGatheringIntervalMs: 1_000,
88
89
  txGatheringBatchSize: 10,
89
90
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -111,6 +112,11 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
111
112
  return this.p2pClient;
112
113
  }
113
114
 
115
+ /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */
116
+ public getDelayer(): Delayer | undefined {
117
+ return this.delayer;
118
+ }
119
+
114
120
  /**
115
121
  * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
116
122
  * @param epochNumber - The epoch number that was just completed.
@@ -121,15 +127,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
121
127
  this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
122
128
  jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
123
129
  });
124
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
125
- if (activeJobs.length > 0) {
126
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
127
- activeJobs: activeJobs.map(job => job.uuid),
128
- });
129
- return true;
130
- }
131
- await this.startProof(epochNumber);
132
- return true;
130
+ return await this.startProofIfNeeded(epochNumber);
133
131
  } catch (err) {
134
132
  if (err instanceof EmptyEpochError) {
135
133
  this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
@@ -155,17 +153,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
155
153
 
156
154
  /**
157
155
  * Stops the prover node and all its dependencies.
156
+ * Resources not owned by this node (shared with the parent aztec-node) are skipped.
158
157
  */
159
158
  async stop() {
160
159
  this.log.info('Stopping ProverNode');
161
160
  await this.epochsMonitor.stop();
162
- await this.prover.stop();
163
- await tryStop(this.p2pClient);
164
- await tryStop(this.l2BlockSource);
165
- await tryStop(this.publisherFactory);
166
161
  this.publisher?.interrupt();
167
162
  await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
168
- await this.worldState.stop();
163
+ await this.prover.stop();
164
+ await tryStop(this.publisherFactory);
169
165
  this.rewardsMetrics.stop();
170
166
  this.l1Metrics.stop();
171
167
  await this.telemetryClient.stop();
@@ -187,7 +183,38 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
187
183
  * Starts a proving process and returns immediately.
188
184
  */
189
185
  public async startProof(epochNumber: EpochNumber) {
190
- const job = await this.createProvingJob(epochNumber, { skipEpochCheck: true });
186
+ await this.startProofIfNeeded(epochNumber, { skipEpochCheck: true });
187
+ }
188
+
189
+ private async startProofIfNeeded(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}): Promise<boolean> {
190
+ if (this.startingProofEpochs.has(epochNumber)) {
191
+ this.log.warn(`Not starting proof for ${epochNumber} since a proof is already being started for the epoch`, {
192
+ epochNumber,
193
+ });
194
+ return false;
195
+ }
196
+
197
+ this.startingProofEpochs.add(epochNumber);
198
+
199
+ try {
200
+ const activeJobs = await this.activeJobsCoverEpoch(epochNumber);
201
+ if (activeJobs.length > 0) {
202
+ this.log.warn(`Not starting proof for ${epochNumber} since an active job already covers the epoch`, {
203
+ epochNumber,
204
+ activeJobs,
205
+ });
206
+ return true;
207
+ }
208
+
209
+ await this.startProofInternal(epochNumber, opts);
210
+ return true;
211
+ } finally {
212
+ this.startingProofEpochs.delete(epochNumber);
213
+ }
214
+ }
215
+
216
+ private async startProofInternal(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}) {
217
+ const job = await this.createProvingJob(epochNumber, opts);
191
218
  void this.runJob(job);
192
219
  }
193
220
 
@@ -241,20 +268,42 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
241
268
  * Returns an array of jobs being processed.
242
269
  */
243
270
  public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
244
- return Promise.resolve(
245
- Array.from(this.jobs.entries()).map(([uuid, job]) => ({
246
- uuid,
247
- status: job.getState(),
248
- epochNumber: job.getEpochNumber(),
249
- })),
250
- );
271
+ return Promise.resolve(this.getJobsInternal());
251
272
  }
252
273
 
253
- protected async getActiveJobsForEpoch(
254
- epochNumber: EpochNumber,
255
- ): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
256
- const jobs = await this.getJobs();
257
- return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
274
+ private getJobsInternal(): { uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[] {
275
+ return Array.from(this.jobs.entries()).map(([uuid, job]) => ({
276
+ uuid,
277
+ status: job.getState(),
278
+ epochNumber: job.getEpochNumber(),
279
+ }));
280
+ }
281
+
282
+ private async activeJobsCoverEpoch(epochNumber: EpochNumber): Promise<string[]> {
283
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: epochNumber });
284
+ if (checkpoints.length === 0) {
285
+ return [];
286
+ }
287
+
288
+ const firstCheckpoint = checkpoints.at(0)!.checkpointNumber;
289
+ const latestCheckpoint = checkpoints.at(-1)!.checkpointNumber;
290
+
291
+ const jobs: string[] = [];
292
+ for (const job of this.jobs.values()) {
293
+ if (job.getEpochNumber() !== epochNumber) {
294
+ continue;
295
+ }
296
+
297
+ const jobCheckpoints = job.getProvingData().checkpoints;
298
+ const checkpointOverlap =
299
+ jobCheckpoints.at(0)!.number <= firstCheckpoint && jobCheckpoints.at(-1)!.number >= latestCheckpoint;
300
+
301
+ if (checkpointOverlap && !['failed', 'stopped', 'timed-out'].includes(job.getState())) {
302
+ jobs.push(job.getId());
303
+ }
304
+ }
305
+
306
+ return jobs;
258
307
  }
259
308
 
260
309
  private checkMaximumPendingJobs() {
@@ -275,13 +324,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
275
324
  const fromCheckpoint = epochData.checkpoints[0].number;
276
325
  const toCheckpoint = epochData.checkpoints.at(-1)!.number;
277
326
  const fromBlock = epochData.checkpoints[0].blocks[0].number;
278
- const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
327
+ const lastBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!;
328
+ const toBlock = lastBlock.number;
279
329
  this.log.verbose(
280
330
  `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
281
331
  );
282
332
 
283
333
  // Fast forward world state to right before the target block and get a fork
284
- await this.worldState.syncImmediate(toBlock);
334
+ const lastBlockHash = await lastBlock.header.hash();
335
+ await this.worldState.syncImmediate(toBlock, lastBlockHash);
285
336
 
286
337
  // Create a processor factory
287
338
  const publicProcessorFactory = new PublicProcessorFactory(
@@ -306,26 +357,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
306
357
 
307
358
  @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
308
359
  private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
309
- const checkpoints = await this.gatherCheckpoints(epochNumber);
360
+ const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({ epoch: epochNumber });
361
+ if (publishedCheckpoints.length === 0) {
362
+ throw new EmptyEpochError(epochNumber);
363
+ }
364
+ const checkpoints = publishedCheckpoints.map(p => p.checkpoint);
365
+ const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
310
366
  const txArray = await this.gatherTxs(epochNumber, checkpoints);
311
367
  const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
312
368
  const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
313
369
  const [firstBlock] = checkpoints[0].blocks;
314
370
  const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
315
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
316
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
317
371
 
318
372
  return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
319
373
  }
320
374
 
321
- private async gatherCheckpoints(epochNumber: EpochNumber) {
322
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
323
- if (checkpoints.length === 0) {
324
- throw new EmptyEpochError(epochNumber);
325
- }
326
- return checkpoints;
327
- }
328
-
329
375
  private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
330
376
  const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
331
377
  const txProvider = this.p2pClient.getTxProvider();
@@ -354,16 +400,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
354
400
  }
355
401
 
356
402
  private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
357
- const header = await (previousBlockNumber === 0
358
- ? this.worldState.getCommitted().getInitialHeader()
359
- : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
360
-
361
- if (!header) {
403
+ const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
404
+ if (!data?.header) {
362
405
  throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
363
406
  }
364
407
 
365
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
366
- return header;
408
+ this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
409
+ return data.header;
367
410
  }
368
411
 
369
412
  /** Extracted for testing purposes. */
@@ -2,14 +2,14 @@ import type { RollupContract } from '@aztec/ethereum/contracts';
2
2
  import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
3
3
  import type { PublisherManager } from '@aztec/ethereum/publisher-manager';
4
4
  import type { LoggerBindings } from '@aztec/foundation/log';
5
- import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
5
+ import type { ProverPublisherConfig, ProverTxSenderConfig } from '@aztec/sequencer-client';
6
6
  import type { TelemetryClient } from '@aztec/telemetry-client';
7
7
 
8
8
  import { ProverNodePublisher } from './prover-node-publisher.js';
9
9
 
10
10
  export class ProverPublisherFactory {
11
11
  constructor(
12
- private config: TxSenderConfig & PublisherConfig,
12
+ private config: ProverTxSenderConfig & ProverPublisherConfig,
13
13
  private deps: {
14
14
  rollupContract: RollupContract;
15
15
  publisherManager: PublisherManager<L1TxUtils>;
@@ -19,11 +19,11 @@ export class ProverPublisherFactory {
19
19
  ) {}
20
20
 
21
21
  public async start() {
22
- await this.deps.publisherManager.loadState();
22
+ await this.deps.publisherManager.start();
23
23
  }
24
24
 
25
- public stop() {
26
- this.deps.publisherManager.interrupt();
25
+ public async stop() {
26
+ await this.deps.publisherManager.stop();
27
27
  }
28
28
 
29
29
  /**