@aztec/prover-node 0.0.1-commit.04852196a → 0.0.1-commit.04d373f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dest/actions/download-epoch-proving-job.js +1 -1
  2. package/dest/actions/rerun-epoch-proving-job.d.ts +3 -2
  3. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  4. package/dest/actions/rerun-epoch-proving-job.js +8 -8
  5. package/dest/bin/run-failed-epoch.js +1 -3
  6. package/dest/config.js +1 -1
  7. package/dest/factory.d.ts +1 -1
  8. package/dest/factory.d.ts.map +1 -1
  9. package/dest/factory.js +26 -6
  10. package/dest/job/epoch-proving-job.d.ts +6 -2
  11. package/dest/job/epoch-proving-job.d.ts.map +1 -1
  12. package/dest/job/epoch-proving-job.js +198 -38
  13. package/dest/metrics.d.ts +21 -1
  14. package/dest/metrics.d.ts.map +1 -1
  15. package/dest/metrics.js +47 -0
  16. package/dest/monitors/epoch-monitor.d.ts +1 -1
  17. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  18. package/dest/monitors/epoch-monitor.js +11 -9
  19. package/dest/prover-node-publisher.d.ts +20 -1
  20. package/dest/prover-node-publisher.d.ts.map +1 -1
  21. package/dest/prover-node-publisher.js +200 -5
  22. package/dest/prover-node.d.ts +1 -2
  23. package/dest/prover-node.d.ts.map +1 -1
  24. package/dest/prover-node.js +21 -19
  25. package/dest/prover-publisher-factory.d.ts +2 -2
  26. package/dest/prover-publisher-factory.d.ts.map +1 -1
  27. package/dest/prover-publisher-factory.js +3 -3
  28. package/package.json +23 -22
  29. package/src/actions/download-epoch-proving-job.ts +1 -1
  30. package/src/actions/rerun-epoch-proving-job.ts +16 -5
  31. package/src/bin/run-failed-epoch.ts +1 -2
  32. package/src/config.ts +1 -1
  33. package/src/factory.ts +21 -4
  34. package/src/job/epoch-proving-job.ts +120 -38
  35. package/src/metrics.ts +71 -0
  36. package/src/monitors/epoch-monitor.ts +5 -6
  37. package/src/prover-node-publisher.ts +232 -9
  38. package/src/prover-node.ts +17 -23
  39. package/src/prover-publisher-factory.ts +3 -3
package/src/metrics.ts CHANGED
@@ -25,6 +25,12 @@ export class ProverNodeJobMetrics {
25
25
  provingJobBlocks: Gauge;
26
26
  provingJobTransactions: Gauge;
27
27
 
28
+ private blobProcessingDuration: Gauge;
29
+ private chonkVerifierDuration: Gauge;
30
+ private blockProcessingDuration: Histogram;
31
+ private checkpointProcessingDuration: Histogram;
32
+ private allCheckpointsProcessingDuration: Gauge;
33
+
28
34
  constructor(
29
35
  private meter: Meter,
30
36
  public readonly tracer: Tracer,
@@ -35,6 +41,14 @@ export class ProverNodeJobMetrics {
35
41
  this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
36
42
  this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
37
43
  this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS);
44
+
45
+ this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
46
+ this.chonkVerifierDuration = this.meter.createGauge(Metrics.PROVER_NODE_CHONK_VERIFIER_LAST_DURATION);
47
+ this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
48
+ this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
49
+ this.allCheckpointsProcessingDuration = this.meter.createGauge(
50
+ Metrics.PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION,
51
+ );
38
52
  }
39
53
 
40
54
  public recordProvingJob(
@@ -50,6 +64,26 @@ export class ProverNodeJobMetrics {
50
64
  this.provingJobBlocks.record(Math.floor(numBlocks));
51
65
  this.provingJobTransactions.record(Math.floor(numTxs));
52
66
  }
67
+
68
+ public recordBlobProcessing(durationMs: number) {
69
+ this.blobProcessingDuration.record(Math.ceil(durationMs));
70
+ }
71
+
72
+ public recordChonkVerifier(durationMs: number) {
73
+ this.chonkVerifierDuration.record(Math.ceil(durationMs));
74
+ }
75
+
76
+ public recordBlockProcessing(durationMs: number) {
77
+ this.blockProcessingDuration.record(Math.ceil(durationMs));
78
+ }
79
+
80
+ public recordCheckpointProcessing(durationMs: number) {
81
+ this.checkpointProcessingDuration.record(Math.ceil(durationMs));
82
+ }
83
+
84
+ public recordAllCheckpointsProcessing(durationMs: number) {
85
+ this.allCheckpointsProcessingDuration.record(Math.ceil(durationMs));
86
+ }
53
87
  }
54
88
 
55
89
  export class ProverNodeRewardsMetrics {
@@ -106,6 +140,13 @@ export class ProverNodeRewardsMetrics {
106
140
  };
107
141
  }
108
142
 
143
+ export type EstimatedSubmitProofStats = {
144
+ gasLimit: bigint;
145
+ baseFeePerGas: bigint;
146
+ maxPriorityFeePerGas: bigint;
147
+ estimatedTotalFee: bigint;
148
+ };
149
+
109
150
  export class ProverNodePublisherMetrics {
110
151
  gasPrice: Histogram;
111
152
  txCount: UpDownCounter;
@@ -117,6 +158,10 @@ export class ProverNodePublisherMetrics {
117
158
  txBlobDataGasCost: Histogram;
118
159
  txTotalFee: Histogram;
119
160
 
161
+ private txGasEstimated: Histogram;
162
+ private gasPriceEstimated: Histogram;
163
+ private txTotalFeeEstimated: Histogram;
164
+
120
165
  private senderBalance: Gauge;
121
166
  private meter: Meter;
122
167
 
@@ -148,6 +193,12 @@ export class ProverNodePublisherMetrics {
148
193
 
149
194
  this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE);
150
195
 
196
+ this.txGasEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS);
197
+
198
+ this.gasPriceEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE);
199
+
200
+ this.txTotalFeeEstimated = this.meter.createHistogram(Metrics.PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE);
201
+
151
202
  this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE);
152
203
  }
153
204
 
@@ -162,6 +213,26 @@ export class ProverNodePublisherMetrics {
162
213
  this.recordTx(durationMs, stats);
163
214
  }
164
215
 
216
+ public recordEstimatedSubmitProof(stats: EstimatedSubmitProofStats) {
217
+ const attributes = { [Attributes.L1_TX_TYPE]: 'submitProof' } as const;
218
+
219
+ this.txGasEstimated.record(Number(stats.gasLimit), attributes);
220
+
221
+ try {
222
+ this.gasPriceEstimated.record(
223
+ parseInt(formatEther(stats.baseFeePerGas + stats.maxPriorityFeePerGas, 'gwei'), 10),
224
+ );
225
+ } catch {
226
+ // ignore
227
+ }
228
+
229
+ try {
230
+ this.txTotalFeeEstimated.record(parseFloat(formatEther(stats.estimatedTotalFee)));
231
+ } catch {
232
+ // ignore
233
+ }
234
+ }
235
+
165
236
  public recordSenderBalance(wei: bigint, senderAddress: string) {
166
237
  const eth = parseFloat(formatEther(wei, 'wei'));
167
238
  this.senderBalance.record(eth, {
@@ -69,9 +69,8 @@ export class EpochMonitor implements Traceable {
69
69
 
70
70
  public async work() {
71
71
  const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
72
- this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
73
72
  if (epochToProve === undefined) {
74
- this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
73
+ this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { epochToProve, blockNumber, slotNumber });
75
74
  return;
76
75
  }
77
76
  if (this.latestEpochNumber !== undefined && epochToProve <= this.latestEpochNumber) {
@@ -86,20 +85,20 @@ export class EpochMonitor implements Traceable {
86
85
  }
87
86
 
88
87
  if (this.options.provingDelayMs) {
89
- this.log.debug(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
88
+ this.log.warn(`Waiting ${this.options.provingDelayMs}ms before proving epoch ${epochToProve}`);
90
89
  await sleep(this.options.provingDelayMs);
91
90
  }
92
91
 
93
- this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
92
+ this.log.verbose(`Epoch ${epochToProve} is ready to be proven`);
94
93
  if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
95
94
  this.latestEpochNumber = epochToProve;
96
95
  }
97
96
  }
98
97
 
99
98
  private async getEpochNumberToProve() {
100
- const lastBlockProven = await this.l2BlockSource.getProvenBlockNumber();
99
+ const lastBlockProven = (await this.l2BlockSource.getBlockNumber({ tag: 'proven' })) ?? BlockNumber.ZERO;
101
100
  const firstBlockToProve = BlockNumber(lastBlockProven + 1);
102
- const firstBlockHeaderToProve = await this.l2BlockSource.getBlockHeader(firstBlockToProve);
101
+ const firstBlockHeaderToProve = (await this.l2BlockSource.getBlockData({ number: firstBlockToProve }))?.header;
103
102
  if (!firstBlockHeaderToProve) {
104
103
  return { epochToProve: undefined, blockNumber: firstBlockToProve };
105
104
  }
@@ -8,20 +8,22 @@ import { areArraysEqual } from '@aztec/foundation/collection';
8
8
  import { Fr } from '@aztec/foundation/curves/bn254';
9
9
  import { EthAddress } from '@aztec/foundation/eth-address';
10
10
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
11
+ import { retryUntil } from '@aztec/foundation/retry';
11
12
  import type { Tuple } from '@aztec/foundation/serialize';
12
13
  import { Timer } from '@aztec/foundation/timer';
13
14
  import { RollupAbi } from '@aztec/l1-artifacts';
14
15
  import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
15
16
  import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
17
+ import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
16
18
  import type { Proof } from '@aztec/stdlib/proofs';
17
19
  import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
18
20
  import type { L1PublishProofStats } from '@aztec/stdlib/stats';
19
21
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
20
22
 
21
23
  import { inspect } from 'util';
22
- import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem';
24
+ import { type Hex, type TransactionReceipt, encodeFunctionData, formatEther, formatGwei } from 'viem';
23
25
 
24
- import { ProverNodePublisherMetrics } from './metrics.js';
26
+ import { type EstimatedSubmitProofStats, ProverNodePublisherMetrics } from './metrics.js';
25
27
 
26
28
  /** Arguments to the submitEpochProof method of the rollup contract */
27
29
  export type L1SubmitEpochProofArgs = {
@@ -101,6 +103,11 @@ export class ProverNodePublisher {
101
103
  const ctx = { epochNumber, fromCheckpoint, toCheckpoint };
102
104
 
103
105
  if (!this.interrupted) {
106
+ if (!(await this.waitUntilStartBuildsOnProven(args))) {
107
+ this.log.verbose('Checkpoint data syncing interrupted', ctx);
108
+ return false;
109
+ }
110
+
104
111
  const timer = new Timer();
105
112
  // Validate epoch proof range and hashes are correct before submitting
106
113
  await this.validateEpochProofSubmission(args);
@@ -147,6 +154,53 @@ export class ProverNodePublisher {
147
154
  return false;
148
155
  }
149
156
 
157
+ private async waitUntilStartBuildsOnProven(args: { epochNumber: EpochNumber; fromCheckpoint: CheckpointNumber }) {
158
+ const { epochNumber, fromCheckpoint } = args;
159
+ const provenCheckpoint = await this.getProvenCheckpoint();
160
+ if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
161
+ return true;
162
+ }
163
+
164
+ const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
165
+ this.log.info(`Waiting for proven checkpoint to reach proof start`, {
166
+ epochNumber,
167
+ fromCheckpoint,
168
+ provenCheckpoint,
169
+ timeout,
170
+ });
171
+
172
+ await retryUntil(
173
+ async () => {
174
+ if (this.interrupted) {
175
+ return true;
176
+ }
177
+
178
+ const proven = await this.getProvenCheckpoint();
179
+ this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, { epochNumber });
180
+ return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
181
+ },
182
+ `proven checkpoint to reach ${fromCheckpoint - 1}`,
183
+ timeout,
184
+ 4,
185
+ );
186
+
187
+ return !this.interrupted;
188
+ }
189
+
190
+ private async getProvenCheckpoint() {
191
+ return (await this.rollupContract.getTips()).proven;
192
+ }
193
+
194
+ private isStartBuildingOnProven(fromCheckpoint: CheckpointNumber, provenCheckpoint: CheckpointNumber) {
195
+ return fromCheckpoint - 1 <= provenCheckpoint;
196
+ }
197
+
198
+ private async getSecondsUntilProofSubmissionWindowEnd(epochNumber: EpochNumber) {
199
+ const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
200
+ const now = BigInt(Math.floor(Date.now() / 1000));
201
+ return Math.max(Number(deadline - now), 0.001);
202
+ }
203
+
150
204
  private async validateEpochProofSubmission(args: {
151
205
  fromCheckpoint: CheckpointNumber;
152
206
  toCheckpoint: CheckpointNumber;
@@ -168,7 +222,7 @@ export class ProverNodePublisher {
168
222
  // toCheckpoint can't be greater than pending
169
223
  if (toCheckpoint > pending) {
170
224
  throw new Error(
171
- `Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as pending checkpoint is ${pending}`,
225
+ `Cannot submit epoch proof for ${fromCheckpoint}-${toCheckpoint} as proposed checkpoint is ${pending}`,
172
226
  );
173
227
  }
174
228
 
@@ -203,13 +257,85 @@ export class ProverNodePublisher {
203
257
  const argsPublicInputs = [...publicInputs.toFields()];
204
258
 
205
259
  if (!areArraysEqual(rollupPublicInputs, argsPublicInputs, (a, b) => a.equals(b))) {
206
- const fmt = (inputs: Fr[] | readonly string[]) => inputs.map(x => x.toString()).join(', ');
207
- throw new Error(
208
- `Root rollup public inputs mismatch:\nRollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
209
- );
260
+ throw await reportPublicInputsMismatch({
261
+ rollupPublicInputs,
262
+ argsPublicInputs,
263
+ fromCheckpoint,
264
+ toCheckpoint,
265
+ rollupContract: this.rollupContract,
266
+ log: this.log,
267
+ });
210
268
  }
211
269
  }
212
270
 
271
+ /**
272
+ * Estimates what submitting the epoch proof would have cost on L1 without actually sending it.
273
+ * Runs the same validation as `submitEpochProof`, encodes the calldata, estimates gas, and records metrics.
274
+ * Used when proof publishing is disabled (e.g. PROVER_NODE_DISABLE_PROOF_PUBLISH=true on mainnet).
275
+ */
276
+ public async analyzeEpochProofSubmission(args: {
277
+ epochNumber: EpochNumber;
278
+ fromCheckpoint: CheckpointNumber;
279
+ toCheckpoint: CheckpointNumber;
280
+ publicInputs: RootRollupPublicInputs;
281
+ proof: Proof;
282
+ batchedBlobInputs: BatchedBlob;
283
+ attestations: ViemCommitteeAttestation[];
284
+ }): Promise<void> {
285
+ const { epochNumber, fromCheckpoint, toCheckpoint } = args;
286
+
287
+ await this.validateEpochProofSubmission(args);
288
+
289
+ const data = this.encodeSubmitEpochProofCalldata(args);
290
+ const senderAddress = this.l1TxUtils.getSenderAddress();
291
+
292
+ const [gasLimit, gasPrice, latestBlock] = await Promise.all([
293
+ this.l1TxUtils.estimateGas(senderAddress.toString() as `0x${string}`, { to: this.rollupContract.address, data }),
294
+ this.l1TxUtils.getGasPrice(),
295
+ this.l1TxUtils.client.getBlock({ blockTag: 'latest' }),
296
+ ]);
297
+
298
+ const baseFeePerGas = latestBlock.baseFeePerGas ?? 0n;
299
+ const { maxPriorityFeePerGas } = gasPrice;
300
+
301
+ const effectiveFeePerGas = baseFeePerGas + maxPriorityFeePerGas;
302
+ const estimatedTotalFee = gasLimit * effectiveFeePerGas;
303
+
304
+ const stats: EstimatedSubmitProofStats = {
305
+ gasLimit,
306
+ baseFeePerGas,
307
+ maxPriorityFeePerGas,
308
+ estimatedTotalFee,
309
+ };
310
+
311
+ this.log.info(`Estimated epoch proof submission cost (not submitted)`, {
312
+ epochNumber,
313
+ fromCheckpoint,
314
+ toCheckpoint,
315
+ gasLimit: gasLimit.toString(),
316
+ baseFeePerGas: formatGwei(baseFeePerGas),
317
+ maxPriorityFeePerGas: formatGwei(maxPriorityFeePerGas),
318
+ estimatedTotalFeeEth: formatEther(estimatedTotalFee),
319
+ });
320
+
321
+ this.metrics.recordEstimatedSubmitProof(stats);
322
+ }
323
+
324
+ private encodeSubmitEpochProofCalldata(args: {
325
+ fromCheckpoint: CheckpointNumber;
326
+ toCheckpoint: CheckpointNumber;
327
+ publicInputs: RootRollupPublicInputs;
328
+ proof: Proof;
329
+ batchedBlobInputs: BatchedBlob;
330
+ attestations: ViemCommitteeAttestation[];
331
+ }): Hex {
332
+ return encodeFunctionData({
333
+ abi: RollupAbi,
334
+ functionName: 'submitEpochRootProof',
335
+ args: [this.getSubmitEpochProofArgs(args)],
336
+ });
337
+ }
338
+
213
339
  private async sendSubmitEpochProofTx(args: {
214
340
  fromCheckpoint: CheckpointNumber;
215
341
  toCheckpoint: CheckpointNumber;
@@ -296,11 +422,108 @@ export class ProverNodePublisher {
296
422
  end: argsArray[1],
297
423
  args: argsArray[2],
298
424
  fees: argsArray[3],
299
- attestations: new CommitteeAttestationsAndSigners(
425
+ attestations: CommitteeAttestationsAndSigners.packAttestations(
300
426
  args.attestations.map(a => CommitteeAttestation.fromViem(a)),
301
- ).getPackedAttestations(),
427
+ ),
302
428
  blobInputs: argsArray[4],
303
429
  proof: proofHex,
304
430
  };
305
431
  }
306
432
  }
433
+
434
+ /**
435
+ * Decodes a `Root rollup public inputs mismatch`, fetches the on-chain CheckpointLog for any
436
+ * mismatching `checkpointHeaderHashes[i]`, emits a structured error log, and returns a thrown-ready
437
+ * Error with a human-readable summary.
438
+ *
439
+ * Layout of `RootRollupPublicInputs.toFields()`:
440
+ * [0] previousArchiveRoot
441
+ * [1] endArchiveRoot
442
+ * [2] outHash
443
+ * [3 .. 3+N-1] checkpointHeaderHashes[i] for i in 0..N-1 (N = MAX_CHECKPOINTS_PER_EPOCH)
444
+ * [3+N .. 3+3N-1] fees[i] = (recipient, value) for i in 0..N-1
445
+ * [3+3N .. 3+3N+4] EpochConstantData (chainId, version, vkTreeRoot, protocolContractsHash, proverId)
446
+ * [3+3N+5 ..] blobPublicInputs (FinalBlobAccumulator)
447
+ */
448
+ async function reportPublicInputsMismatch(input: {
449
+ rollupPublicInputs: readonly Fr[];
450
+ argsPublicInputs: readonly Fr[];
451
+ fromCheckpoint: CheckpointNumber;
452
+ toCheckpoint: CheckpointNumber;
453
+ rollupContract: RollupContract;
454
+ log: Logger;
455
+ }): Promise<Error> {
456
+ const { rollupPublicInputs, argsPublicInputs, fromCheckpoint, toCheckpoint, rollupContract, log } = input;
457
+ const N = MAX_CHECKPOINTS_PER_EPOCH;
458
+ const constantsStart = 3 + 3 * N;
459
+ const blobStart = constantsStart + 5;
460
+ const constantLabels = ['chainId', 'version', 'vkTreeRoot', 'protocolContractsHash', 'proverId'];
461
+
462
+ const diffs: { index: number; label: string; rollup: Fr; computed: Fr; checkpointIndex?: number }[] = [];
463
+ const len = Math.max(rollupPublicInputs.length, argsPublicInputs.length);
464
+ for (let i = 0; i < len; i++) {
465
+ const a = rollupPublicInputs[i] ?? Fr.ZERO;
466
+ const b = argsPublicInputs[i] ?? Fr.ZERO;
467
+ if (a.equals(b)) {
468
+ continue;
469
+ }
470
+ let label: string;
471
+ let checkpointIndex: number | undefined;
472
+ if (i === 0) {
473
+ label = 'previousArchiveRoot';
474
+ } else if (i === 1) {
475
+ label = 'endArchiveRoot';
476
+ } else if (i === 2) {
477
+ label = 'outHash';
478
+ } else if (i < 3 + N) {
479
+ checkpointIndex = i - 3;
480
+ label = `checkpointHeaderHashes[${checkpointIndex}]`;
481
+ } else if (i < 3 + 3 * N) {
482
+ const feePairIndex = i - (3 + N);
483
+ const feeIndex = Math.floor(feePairIndex / 2);
484
+ const sub = feePairIndex % 2 === 0 ? 'recipient' : 'value';
485
+ label = `fees[${feeIndex}].${sub}`;
486
+ } else if (i < blobStart) {
487
+ label = `constants.${constantLabels[i - constantsStart]}`;
488
+ } else {
489
+ label = `blobPublicInputs[${i - blobStart}]`;
490
+ }
491
+ diffs.push({ index: i, label, rollup: a, computed: b, checkpointIndex });
492
+ }
493
+
494
+ // For each mismatching checkpointHeaderHash, fetch the L1 CheckpointLog so the operator can
495
+ // see what was published on-chain alongside the prover's recomputed hash.
496
+ const onChainCheckpoints = await Promise.all(
497
+ diffs
498
+ .filter(d => d.checkpointIndex !== undefined)
499
+ .map(async d => {
500
+ const checkpointNumber = CheckpointNumber(fromCheckpoint + d.checkpointIndex!);
501
+ try {
502
+ const cp = await rollupContract.getCheckpoint(checkpointNumber);
503
+ return { checkpointIndex: d.checkpointIndex!, checkpointNumber, headerHash: cp.headerHash.toString() };
504
+ } catch (err) {
505
+ return { checkpointIndex: d.checkpointIndex!, checkpointNumber, error: (err as Error).message };
506
+ }
507
+ }),
508
+ );
509
+
510
+ log.error(`Root rollup public inputs mismatch`, undefined, {
511
+ fromCheckpoint,
512
+ toCheckpoint,
513
+ numDiffs: diffs.length,
514
+ diffs: diffs.map(d => ({
515
+ index: d.index,
516
+ label: d.label,
517
+ rollup: d.rollup.toString(),
518
+ computed: d.computed.toString(),
519
+ })),
520
+ onChainCheckpoints,
521
+ });
522
+
523
+ const fmt = (inputs: readonly Fr[]) => inputs.map(x => x.toString()).join(', ');
524
+ const summary = diffs.map(d => `[${d.index} ${d.label}] L1=${d.rollup} prover=${d.computed}`).join('\n');
525
+ return new Error(
526
+ `Root rollup public inputs mismatch (${diffs.length} fields differ):\n${summary}\n` +
527
+ `Rollup: ${fmt(rollupPublicInputs)}\nComputed:${fmt(argsPublicInputs)}`,
528
+ );
529
+ }
@@ -84,7 +84,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
84
84
  this.config = {
85
85
  proverNodePollingIntervalMs: 1_000,
86
86
  proverNodeMaxPendingJobs: 100,
87
- proverNodeMaxParallelBlocksPerEpoch: 32,
87
+ proverNodeMaxParallelBlocksPerEpoch: 0,
88
88
  txGatheringIntervalMs: 1_000,
89
89
  txGatheringBatchSize: 10,
90
90
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -166,10 +166,10 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
166
166
  async stop() {
167
167
  this.log.info('Stopping ProverNode');
168
168
  await this.epochsMonitor.stop();
169
- await this.prover.stop();
170
- await tryStop(this.publisherFactory);
171
169
  this.publisher?.interrupt();
172
170
  await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
171
+ await this.prover.stop();
172
+ await tryStop(this.publisherFactory);
173
173
  this.rewardsMetrics.stop();
174
174
  this.l1Metrics.stop();
175
175
  await this.telemetryClient.stop();
@@ -279,13 +279,15 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
279
279
  const fromCheckpoint = epochData.checkpoints[0].number;
280
280
  const toCheckpoint = epochData.checkpoints.at(-1)!.number;
281
281
  const fromBlock = epochData.checkpoints[0].blocks[0].number;
282
- const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
282
+ const lastBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!;
283
+ const toBlock = lastBlock.number;
283
284
  this.log.verbose(
284
285
  `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
285
286
  );
286
287
 
287
288
  // Fast forward world state to right before the target block and get a fork
288
- await this.worldState.syncImmediate(toBlock);
289
+ const lastBlockHash = await lastBlock.header.hash();
290
+ await this.worldState.syncImmediate(toBlock, lastBlockHash);
289
291
 
290
292
  // Create a processor factory
291
293
  const publicProcessorFactory = new PublicProcessorFactory(
@@ -310,26 +312,21 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
310
312
 
311
313
  @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
312
314
  private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
313
- const checkpoints = await this.gatherCheckpoints(epochNumber);
315
+ const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({ epoch: epochNumber });
316
+ if (publishedCheckpoints.length === 0) {
317
+ throw new EmptyEpochError(epochNumber);
318
+ }
319
+ const checkpoints = publishedCheckpoints.map(p => p.checkpoint);
320
+ const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
314
321
  const txArray = await this.gatherTxs(epochNumber, checkpoints);
315
322
  const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
316
323
  const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
317
324
  const [firstBlock] = checkpoints[0].blocks;
318
325
  const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
319
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
320
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
321
326
 
322
327
  return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
323
328
  }
324
329
 
325
- private async gatherCheckpoints(epochNumber: EpochNumber) {
326
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
327
- if (checkpoints.length === 0) {
328
- throw new EmptyEpochError(epochNumber);
329
- }
330
- return checkpoints;
331
- }
332
-
333
330
  private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
334
331
  const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
335
332
  const txProvider = this.p2pClient.getTxProvider();
@@ -358,16 +355,13 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
358
355
  }
359
356
 
360
357
  private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
361
- const header = await (previousBlockNumber === 0
362
- ? this.worldState.getCommitted().getInitialHeader()
363
- : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
364
-
365
- if (!header) {
358
+ const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
359
+ if (!data?.header) {
366
360
  throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
367
361
  }
368
362
 
369
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
370
- return header;
363
+ this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
364
+ return data.header;
371
365
  }
372
366
 
373
367
  /** Extracted for testing purposes. */
@@ -19,11 +19,11 @@ export class ProverPublisherFactory {
19
19
  ) {}
20
20
 
21
21
  public async start() {
22
- await this.deps.publisherManager.loadState();
22
+ await this.deps.publisherManager.start();
23
23
  }
24
24
 
25
- public stop() {
26
- this.deps.publisherManager.interrupt();
25
+ public async stop() {
26
+ await this.deps.publisherManager.stop();
27
27
  }
28
28
 
29
29
  /**