@aztec/prover-node 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +572 -0
  2. package/dest/actions/download-epoch-proving-job.js +1 -1
  3. package/dest/actions/rerun-epoch-proving-job.d.ts +14 -4
  4. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  5. package/dest/actions/rerun-epoch-proving-job.js +146 -24
  6. package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
  7. package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
  8. package/dest/bin/run-failed-epoch.js +6 -5
  9. package/dest/checkpoint-store.d.ts +95 -0
  10. package/dest/checkpoint-store.d.ts.map +1 -0
  11. package/dest/checkpoint-store.js +178 -0
  12. package/dest/config.d.ts +7 -8
  13. package/dest/config.d.ts.map +1 -1
  14. package/dest/config.js +24 -20
  15. package/dest/factory.d.ts +19 -13
  16. package/dest/factory.d.ts.map +1 -1
  17. package/dest/factory.js +41 -65
  18. package/dest/index.d.ts +2 -1
  19. package/dest/index.d.ts.map +1 -1
  20. package/dest/index.js +1 -0
  21. package/dest/job/checkpoint-prover.d.ts +165 -0
  22. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  23. package/dest/job/checkpoint-prover.js +405 -0
  24. package/dest/job/epoch-session.d.ts +160 -0
  25. package/dest/job/epoch-session.d.ts.map +1 -0
  26. package/dest/job/{epoch-proving-job.js → epoch-session.js} +301 -298
  27. package/dest/job/top-tree-job.d.ts +82 -0
  28. package/dest/job/top-tree-job.d.ts.map +1 -0
  29. package/dest/job/top-tree-job.js +152 -0
  30. package/dest/metrics.d.ts +40 -3
  31. package/dest/metrics.d.ts.map +1 -1
  32. package/dest/metrics.js +101 -4
  33. package/dest/monitors/epoch-monitor.d.ts +1 -1
  34. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  35. package/dest/monitors/epoch-monitor.js +11 -9
  36. package/dest/proof-publishing-service.d.ts +161 -0
  37. package/dest/proof-publishing-service.d.ts.map +1 -0
  38. package/dest/proof-publishing-service.js +335 -0
  39. package/dest/prover-node-publisher.d.ts +25 -15
  40. package/dest/prover-node-publisher.d.ts.map +1 -1
  41. package/dest/prover-node-publisher.js +201 -62
  42. package/dest/prover-node.d.ts +144 -69
  43. package/dest/prover-node.d.ts.map +1 -1
  44. package/dest/prover-node.js +543 -221
  45. package/dest/prover-publisher-factory.d.ts +6 -4
  46. package/dest/prover-publisher-factory.d.ts.map +1 -1
  47. package/dest/prover-publisher-factory.js +4 -3
  48. package/dest/session-manager.d.ts +158 -0
  49. package/dest/session-manager.d.ts.map +1 -0
  50. package/dest/session-manager.js +492 -0
  51. package/dest/test/index.d.ts +7 -6
  52. package/dest/test/index.d.ts.map +1 -1
  53. package/package.json +23 -22
  54. package/src/actions/download-epoch-proving-job.ts +1 -1
  55. package/src/actions/rerun-epoch-proving-job.ts +178 -33
  56. package/src/actions/upload-epoch-proof-failure.ts +1 -1
  57. package/src/bin/run-failed-epoch.ts +5 -3
  58. package/src/checkpoint-store.ts +212 -0
  59. package/src/config.ts +35 -32
  60. package/src/factory.ts +69 -111
  61. package/src/index.ts +1 -0
  62. package/src/job/checkpoint-prover.ts +538 -0
  63. package/src/job/epoch-session.ts +462 -0
  64. package/src/job/top-tree-job.ts +227 -0
  65. package/src/metrics.ts +123 -10
  66. package/src/monitors/epoch-monitor.ts +5 -6
  67. package/src/proof-publishing-service.ts +427 -0
  68. package/src/prover-node-publisher.ts +236 -78
  69. package/src/prover-node.ts +628 -247
  70. package/src/prover-publisher-factory.ts +8 -5
  71. package/src/session-manager.ts +592 -0
  72. package/src/test/index.ts +6 -6
  73. package/dest/job/epoch-proving-job.d.ts +0 -63
  74. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  75. package/src/job/epoch-proving-job.ts +0 -435
@@ -1,89 +1,136 @@
1
1
  import type { Archiver } from '@aztec/archiver';
2
2
  import type { RollupContract } from '@aztec/ethereum/contracts';
3
+ import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
3
4
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
4
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
5
- import type { Fr } from '@aztec/foundation/curves/bn254';
5
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
6
6
  import { memoize } from '@aztec/foundation/decorators';
7
7
  import { createLogger } from '@aztec/foundation/log';
8
- import { DateProvider } from '@aztec/foundation/timer';
9
- import type { DataStoreConfig } from '@aztec/kv-store/config';
10
- import type { P2PClient } from '@aztec/p2p';
8
+ import { RunningPromise } from '@aztec/foundation/running-promise';
9
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
10
+ import type { EpochProverFactory } from '@aztec/prover-client';
11
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
12
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
11
13
  import { PublicProcessorFactory } from '@aztec/simulator/server';
12
- import type { L2BlockSource } from '@aztec/stdlib/block';
13
- import type { Checkpoint } from '@aztec/stdlib/checkpoint';
14
+ import {
15
+ EventDrivenL2BlockStream,
16
+ type L2BlockId,
17
+ type L2BlockSource,
18
+ type L2BlockStreamEvent,
19
+ type L2BlockStreamEventHandler,
20
+ L2TipsMemoryStore,
21
+ } from '@aztec/stdlib/block';
22
+ import type { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
14
23
  import type { ChainConfig } from '@aztec/stdlib/config';
15
24
  import type { ContractDataSource } from '@aztec/stdlib/contract';
16
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
25
+ import { type L1RollupConstants, getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
17
26
  import {
18
27
  type EpochProverManager,
28
+ type EpochProvingJobState,
19
29
  EpochProvingJobTerminalState,
30
+ type ITxProvider,
20
31
  type ProverNodeApi,
21
32
  type Service,
22
- type WorldStateSyncStatus,
23
33
  type WorldStateSynchronizer,
24
34
  tryStop,
25
35
  } from '@aztec/stdlib/interfaces/server';
36
+ import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
26
37
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
27
- import type { P2PClientType } from '@aztec/stdlib/p2p';
28
- import type { Tx } from '@aztec/stdlib/tx';
38
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
29
39
  import {
30
- Attributes,
31
40
  L1Metrics,
32
41
  type TelemetryClient,
33
42
  type Traceable,
34
43
  type Tracer,
35
44
  getTelemetryClient,
36
- trackSpan,
37
45
  } from '@aztec/telemetry-client';
38
46
 
39
47
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
48
+ import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
40
49
  import type { SpecificProverNodeConfig } from './config.js';
41
- import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
42
- import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
50
+ import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
51
+ import type { EpochSessionHooks } from './job/epoch-session.js';
43
52
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
44
- import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
45
- import type { ProverNodePublisher } from './prover-node-publisher.js';
53
+ import { ProofPublishingService } from './proof-publishing-service.js';
46
54
  import type { ProverPublisherFactory } from './prover-publisher-factory.js';
55
+ import { SessionManager } from './session-manager.js';
47
56
 
48
57
  type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
49
58
  type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
50
59
 
51
60
  /**
52
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
53
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
54
- * proof for the epoch, and submits it to L1.
61
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
62
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
63
+ * the wait rather than letting `stop()` hang indefinitely.
64
+ */
65
+ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
66
+
67
+ /**
68
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
69
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
70
+ * complete.
71
+ *
72
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
73
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
74
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
75
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
55
76
  */
56
- export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
77
+ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Traceable {
57
78
  private log = createLogger('prover-node');
58
- private dateProvider = new DateProvider();
59
79
 
60
- private jobs: Map<string, EpochProvingJob> = new Map();
61
- private config: ProverNodeOptions;
62
- private jobMetrics: ProverNodeJobMetrics;
63
- private rewardsMetrics: ProverNodeRewardsMetrics;
80
+ protected readonly checkpointStore: CheckpointStore;
81
+ protected readonly chonkCache: ChonkCache;
82
+ protected sessionManager: SessionManager | undefined;
83
+
84
+ private readonly config: ProverNodeOptions;
85
+ private readonly jobMetrics: ProverNodeJobMetrics;
86
+ private readonly rewardsMetrics: ProverNodeRewardsMetrics;
87
+
88
+ /** In-memory store for the L2BlockStream's local data provider. */
89
+ private tipsStore: L2TipsMemoryStore;
90
+ /** Block stream for checkpoint and reorg detection. */
91
+ private blockStream: EventDrivenL2BlockStream | undefined;
92
+ /**
93
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
94
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
95
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
96
+ * submission deadline. Protected so tests can verify the start() seeding.
97
+ */
98
+ protected lastExpiredEpoch: EpochNumber | undefined;
99
+
100
+ /**
101
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
102
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
103
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
104
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
105
+ */
106
+ protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
107
+
108
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
109
+ private expiryTicker: RunningPromise | undefined;
64
110
 
65
111
  public readonly tracer: Tracer;
66
112
 
67
- protected publisher: ProverNodePublisher | undefined;
113
+ protected publishingService: ProofPublishingService | undefined;
68
114
 
69
115
  constructor(
70
- protected readonly prover: EpochProverManager,
116
+ protected readonly prover: EpochProverManager & EpochProverFactory,
71
117
  protected readonly publisherFactory: ProverPublisherFactory,
72
118
  protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
73
119
  protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
74
120
  protected readonly contractDataSource: ContractDataSource,
75
121
  protected readonly worldState: WorldStateSynchronizer,
76
- protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
77
- protected readonly epochsMonitor: EpochMonitor,
122
+ protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
78
123
  protected readonly rollupContract: RollupContract,
79
124
  protected readonly l1Metrics: L1Metrics,
80
125
  config: Partial<ProverNodeOptions> = {},
81
126
  protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
127
+ private delayer?: Delayer,
128
+ private readonly dateProvider: DateProvider = new DateProvider(),
82
129
  ) {
83
130
  this.config = {
84
131
  proverNodePollingIntervalMs: 1_000,
85
132
  proverNodeMaxPendingJobs: 100,
86
- proverNodeMaxParallelBlocksPerEpoch: 32,
133
+ proverNodeMaxParallelBlocksPerEpoch: 0,
87
134
  txGatheringIntervalMs: 1_000,
88
135
  txGatheringBatchSize: 10,
89
136
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -99,8 +146,35 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
99
146
  this.tracer = telemetryClient.getTracer('ProverNode');
100
147
 
101
148
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
102
-
103
149
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
150
+
151
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
152
+
153
+ this.chonkCache = new ChonkCache(this.log.getBindings());
154
+ this.checkpointStore = new CheckpointStore(
155
+ this.l2BlockSource,
156
+ {
157
+ proverFactory: this.prover,
158
+ chonkCache: this.chonkCache,
159
+ publicProcessorFactory: new PublicProcessorFactory(
160
+ this.contractDataSource,
161
+ this.dateProvider,
162
+ this.telemetryClient,
163
+ this.log.getBindings(),
164
+ ),
165
+ dbProvider: this.worldState,
166
+ txProvider: this.p2pClient.getTxProvider(),
167
+ dateProvider: this.dateProvider,
168
+ proverId: this.prover.getProverId(),
169
+ metrics: this.jobMetrics,
170
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
171
+ deadline: undefined,
172
+ // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
173
+ // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
174
+ onFailed: prover => void this.tryUploadCheckpointFailure(prover),
175
+ },
176
+ this.log.getBindings(),
177
+ );
104
178
  }
105
179
 
106
180
  public getProverId() {
@@ -111,287 +185,598 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
111
185
  return this.p2pClient;
112
186
  }
113
187
 
188
+ /** Test-only: the shared L1 tx delayer, if enabled. */
189
+ public getDelayer(): Delayer | undefined {
190
+ return this.delayer;
191
+ }
192
+
193
+ /** Observability summary for the ProverNodeApi. */
194
+ public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
195
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
196
+ }
197
+
198
+ /** Tests inspect this when validating reconcile behaviour. */
199
+ public getCheckpointStore(): CheckpointStore {
200
+ return this.checkpointStore;
201
+ }
202
+
203
+ /** Tests inspect this to verify chonk-cache release semantics. */
204
+ public getChonkCache(): ChonkCache {
205
+ return this.chonkCache;
206
+ }
207
+
208
+ /** Tests inspect this when looking up live sessions. */
209
+ public getSessionManager(): SessionManager {
210
+ if (!this.sessionManager) {
211
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
212
+ }
213
+ return this.sessionManager;
214
+ }
215
+
216
+ /** Returns the underlying prover instance. */
217
+ public getProver() {
218
+ return this.prover;
219
+ }
220
+
221
+ // ---------------- L2BlockStream handler ----------------
222
+
223
+ public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
224
+ switch (event.type) {
225
+ case 'chain-checkpointed':
226
+ await this.processCheckpointJump(event.checkpoint.number);
227
+ break;
228
+ case 'chain-pruned':
229
+ await this.handlePruneEvent(event.block);
230
+ break;
231
+ case 'chain-proven':
232
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
233
+ break;
234
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
235
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
236
+ // `chain-finalized` carries nothing the prover-node acts on.
237
+ case 'chain-proposed':
238
+ case 'chain-finalized':
239
+ case 'blocks-added':
240
+ break;
241
+ default: {
242
+ const _: never = event;
243
+ break;
244
+ }
245
+ }
246
+ // Advance the local tips store only after the proving-side handling (registration / prune) has
247
+ // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
248
+ // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
249
+ await this.tipsStore.handleBlockStreamEvent(event);
250
+ }
251
+
114
252
  /**
115
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
116
- * @param epochNumber - The epoch number that was just completed.
117
- * @returns false if there is an error, true otherwise
253
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
254
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
255
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
256
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
257
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
258
+ *
259
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
260
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
261
+ * behind so the next pass retries from the first checkpoint that did not complete.
118
262
  */
119
- async handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean> {
120
- try {
121
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
122
- jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
263
+ private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
264
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
265
+ return;
266
+ }
267
+ const l1Constants = await this.getL1Constants();
268
+
269
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
270
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
271
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
272
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
273
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
274
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
275
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
276
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
277
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
278
+ from,
279
+ cappedFrom,
280
+ targetCheckpoint,
281
+ maxCheckpoints,
123
282
  });
124
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
125
- if (activeJobs.length > 0) {
126
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
127
- activeJobs: activeJobs.map(job => job.uuid),
128
- });
129
- return true;
283
+ // Advance the cursor past the skipped checkpoints so they are never retried.
284
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
285
+ from = cappedFrom;
286
+ }
287
+ const limit = Number(targetCheckpoint - from) + 1;
288
+ const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
289
+
290
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
291
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
292
+ // never drop an individual checkpoint inside an epoch we will prove.
293
+ const epochSkippable = new Map<EpochNumber, boolean>();
294
+ for (const metadata of metadatas) {
295
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
296
+ let skippable = epochSkippable.get(epochNumber);
297
+ if (skippable === undefined) {
298
+ skippable =
299
+ (await this.isEpochFullyProven(epochNumber, l1Constants)) ||
300
+ (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
301
+ epochSkippable.set(epochNumber, skippable);
130
302
  }
131
- await this.startProof(epochNumber);
132
- return true;
133
- } catch (err) {
134
- if (err instanceof EmptyEpochError) {
135
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
303
+ if (skippable) {
304
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
136
305
  } else {
137
- this.log.error(`Error handling epoch completed`, err);
306
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
138
307
  }
308
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
309
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
310
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
311
+ }
312
+ }
313
+
314
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
315
+ private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
316
+ const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
317
+ if (!published) {
318
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
319
+ }
320
+ const checkpoint = published.checkpoint;
321
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
322
+ checkpointNumber: checkpoint.number,
323
+ epochNumber,
324
+ slotNumber: checkpoint.header.slotNumber,
325
+ });
326
+
327
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
328
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
329
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
330
+
331
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
332
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
333
+ await this.tipsStore.recordBlockHashes(
334
+ await Promise.all(
335
+ checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
336
+ ),
337
+ );
338
+ }
339
+
340
+ /**
341
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
342
+ * and the archive sibling path.
343
+ */
344
+ private async collectRegisterData(
345
+ checkpoint: Checkpoint,
346
+ attestations: PublishedCheckpoint['attestations'],
347
+ ): Promise<RegisterCheckpointData> {
348
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
349
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
350
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
351
+ const lastBlock = checkpoint.blocks.at(-1)!;
352
+ const lastBlockHash = await lastBlock.header.hash();
353
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
354
+ const previousArchiveSiblingPath = await getLastSiblingPath(
355
+ MerkleTreeId.ARCHIVE,
356
+ this.worldState.getSnapshot(previousBlockNumber),
357
+ );
358
+ return {
359
+ attestations,
360
+ previousBlockHeader,
361
+ l1ToL2Messages,
362
+ previousArchiveSiblingPath,
363
+ };
364
+ }
365
+
366
+ /**
367
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
368
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
369
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
370
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
371
+ * fails and the prune is retried next iteration.
372
+ */
373
+ private async handlePruneEvent(prunedToBlock: L2BlockId) {
374
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
375
+
376
+ // Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
377
+ // so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
378
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
379
+ let cursorFloor: CheckpointNumber;
380
+ if (prunedToBlock.number === 0) {
381
+ cursorFloor = CheckpointNumber.ZERO;
382
+ } else {
383
+ const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
384
+ if (targetData === undefined) {
385
+ throw new Error(
386
+ `No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
387
+ );
388
+ }
389
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
390
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
391
+ // would permanently skip a rebuilt same-number checkpoint.
392
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
393
+ }
394
+
395
+ const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
396
+
397
+ if (this.lastProcessedCheckpoint > cursorFloor) {
398
+ this.lastProcessedCheckpoint = cursorFloor;
399
+ }
400
+
401
+ if (affected.length === 0) {
402
+ return;
403
+ }
404
+ const l1Constants = await this.getL1Constants();
405
+ const affectedEpochs = Array.from(
406
+ new Set(affected.map(p => Number(getEpochAtSlot(p.slotNumber, l1Constants)))),
407
+ ).map(n => EpochNumber(n));
408
+ // The session manager cancels every affected session, which in turn calls
409
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
410
+ // publishing service is needed.
411
+ await this.sessionManager?.onPrune(affectedEpochs);
412
+ }
413
+
414
+ /**
415
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
416
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
417
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
418
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
419
+ * be read yet.
420
+ */
421
+ private async isEpochPastProofSubmissionWindow(
422
+ epochNumber: EpochNumber,
423
+ l1Constants: L1RollupConstants,
424
+ ): Promise<boolean> {
425
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
426
+ if (latestSlot === undefined) {
139
427
  return false;
140
428
  }
429
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
430
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
431
+ }
432
+
433
+ /**
434
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
435
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
436
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
437
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
438
+ * the last check or the archiver's slot can't be read.
439
+ */
440
+ private async checkEpochExpiry(): Promise<void> {
441
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
442
+ if (latestSlot === undefined) {
443
+ return;
444
+ }
445
+ const l1Constants = await this.getL1Constants();
446
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
447
+ const offset = l1Constants.proofSubmissionEpochs + 1;
448
+ if (latestEpoch < offset) {
449
+ return;
450
+ }
451
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
452
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
453
+ if (newlyExpiredUpTo < from) {
454
+ return;
455
+ }
456
+ for (let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)) {
457
+ await this.expireEpoch(e);
458
+ }
459
+ this.lastExpiredEpoch = newlyExpiredUpTo;
141
460
  }
142
461
 
143
462
  /**
144
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
145
- * starts proving jobs for them.
463
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
464
+ * CheckpointProver in the store whose epoch is at or below it.
146
465
  */
466
+ private async expireEpoch(epoch: EpochNumber): Promise<void> {
467
+ try {
468
+ const blocks = await this.l2BlockSource.getBlocks({ epoch, onlyCheckpointed: true });
469
+ if (blocks.length > 0) {
470
+ this.chonkCache.releaseForBlocks(blocks);
471
+ }
472
+ } catch (err) {
473
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
474
+ }
475
+ this.checkpointStore.reapExpired(epoch);
476
+ }
477
+
478
+ // ---------------- public API ----------------
479
+
480
+ /**
481
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
482
+ */
483
+ public async startProof(epochNumber: EpochNumber): Promise<string> {
484
+ if (!this.sessionManager) {
485
+ throw new Error('ProverNode not started');
486
+ }
487
+ return await this.sessionManager.startProof(epochNumber);
488
+ }
489
+
490
+ // ---------------- Service lifecycle ----------------
491
+
147
492
  async start() {
148
- this.epochsMonitor.start(this);
493
+ await this.checkpointStore.start();
494
+
149
495
  await this.publisherFactory.start();
150
- this.publisher = await this.publisherFactory.create();
496
+ this.publishingService = new ProofPublishingService({
497
+ publisherFactory: this.publisherFactory,
498
+ l2BlockSource: this.l2BlockSource,
499
+ dateProvider: this.dateProvider,
500
+ config: { skipSubmitProof: !!this.config.proverNodeDisableProofPublish },
501
+ bindings: this.log.getBindings(),
502
+ });
503
+ this.sessionManager = this.createSessionManager(this.publishingService);
504
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
505
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
506
+ // the previous epoch is proven on L1.
507
+ this.sessionManager.start();
508
+ // Now that the store + manager exist, arm the live-state observable gauges.
509
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
510
+
511
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
512
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
513
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
514
+ this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
515
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
516
+ tipsOnly: true,
517
+ });
518
+ this.blockStream.start();
519
+
520
+ // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
521
+ // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
522
+ // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
523
+ this.expiryTicker = new RunningPromise(
524
+ () => this.checkEpochExpiry(),
525
+ this.log,
526
+ this.config.proverNodePollingIntervalMs,
527
+ );
528
+ this.expiryTicker.start();
529
+
151
530
  await this.rewardsMetrics.start();
152
531
  this.l1Metrics.start();
153
532
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
154
533
  }
155
534
 
156
- /**
157
- * Stops the prover node and all its dependencies.
158
- */
159
535
  async stop() {
160
536
  this.log.info('Stopping ProverNode');
161
- await this.epochsMonitor.stop();
537
+ this.jobMetrics.stopObservingState();
538
+ await this.blockStream?.stop();
539
+ await this.expiryTicker?.stop();
540
+ if (this.sessionManager) {
541
+ await this.sessionManager.stop();
542
+ }
543
+ if (this.publishingService) {
544
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
545
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
546
+ // the tx may still mine, but shutdown must not hang on it.
547
+ const publishingService = this.publishingService;
548
+ await executeTimeout(
549
+ () => publishingService.stop(),
550
+ PUBLISHING_SERVICE_STOP_TIMEOUT_MS,
551
+ 'prover-node publishing-service stop',
552
+ ).catch(err => this.log.warn(`Timed out stopping proof publishing service`, err));
553
+ }
554
+ await this.checkpointStore.stop();
555
+ this.chonkCache.stop();
162
556
  await this.prover.stop();
163
- await tryStop(this.p2pClient);
164
- await tryStop(this.l2BlockSource);
165
557
  await tryStop(this.publisherFactory);
166
- this.publisher?.interrupt();
167
- await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
168
- await this.worldState.stop();
169
558
  this.rewardsMetrics.stop();
170
559
  this.l1Metrics.stop();
171
560
  await this.telemetryClient.stop();
172
561
  this.log.info('Stopped ProverNode');
173
562
  }
174
563
 
175
- /** Returns world state status. */
176
- public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
177
- const { syncSummary } = await this.worldState.status();
178
- return syncSummary;
564
+ /**
565
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap the
566
+ * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
567
+ * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
568
+ * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
569
+ */
570
+ protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
571
+ return new SessionManager({
572
+ checkpointStore: this.checkpointStore,
573
+ l2BlockSource: this.l2BlockSource,
574
+ proverFactory: this.prover,
575
+ proverId: this.prover.getProverId(),
576
+ publishingService,
577
+ metrics: this.jobMetrics,
578
+ dateProvider: this.dateProvider,
579
+ config: {
580
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
581
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
582
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
583
+ },
584
+ onSessionFailed: async session => {
585
+ await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
586
+ },
587
+ bindings: this.log.getBindings(),
588
+ });
179
589
  }
180
590
 
181
- /** Returns archiver status. */
182
- public getL2Tips() {
183
- return this.l2BlockSource.getL2Tips();
591
+ /**
592
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
593
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
594
+ * Applies to every session constructed after this call.
595
+ */
596
+ public setSessionHooks(hooks: EpochSessionHooks): void {
597
+ if (!this.sessionManager) {
598
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
599
+ }
600
+ this.sessionManager.setSessionHooks(hooks);
184
601
  }
185
602
 
186
603
  /**
187
- * Starts a proving process and returns immediately.
604
+ * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
605
+ * CheckpointProver constructed after this call. For the e2e harness only.
188
606
  */
189
- public async startProof(epochNumber: EpochNumber) {
190
- const job = await this.createProvingJob(epochNumber, { skipEpochCheck: true });
191
- void this.runJob(job);
607
+ public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
608
+ this.checkpointStore.setTestHooks(hooks);
192
609
  }
193
610
 
194
- private async runJob(job: EpochProvingJob) {
195
- const epochNumber = job.getEpochNumber();
196
- const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
197
-
198
- try {
199
- await job.run();
200
- const state = job.getState();
201
- ctx.state = state;
202
-
203
- if (state === 'reorg') {
204
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
205
- await this.createProvingJob(epochNumber);
206
- } else if (state === 'failed') {
207
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
208
- await this.tryUploadEpochFailure(job);
209
- } else {
210
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
211
- }
212
- } catch (err) {
213
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
214
- } finally {
215
- this.jobs.delete(job.getId());
611
+ /**
612
+ * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
613
+ * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
614
+ * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
615
+ * store is configured or the checkpoint set is empty.
616
+ */
617
+ public async tryUploadEpochFailure(
618
+ id: string,
619
+ checkpoints: readonly CheckpointProver[],
620
+ ): Promise<string | undefined> {
621
+ if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
622
+ return undefined;
216
623
  }
624
+ const data = await SessionManager.buildProvingData(checkpoints);
625
+ return await uploadEpochProofFailure(
626
+ this.config.proverNodeFailedEpochStore,
627
+ // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
628
+ id,
629
+ data,
630
+ this.l2BlockSource as Archiver,
631
+ this.worldState,
632
+ assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
633
+ this.log,
634
+ );
217
635
  }
218
636
 
219
- protected async tryUploadEpochFailure(job: EpochProvingJob) {
220
- if (this.config.proverNodeFailedEpochStore) {
637
+ /**
638
+ * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
639
+ * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
640
+ * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
641
+ * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
642
+ * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
643
+ */
644
+ public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
645
+ if (!this.config.proverNodeFailedEpochStore) {
646
+ return undefined;
647
+ }
648
+ try {
649
+ // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
650
+ // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
651
+ // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
652
+ // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
653
+ // the snapshot (full world-state + archiver) is expensive to produce and store.
654
+ if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
655
+ this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
656
+ checkpointNumber: prover.checkpoint.number,
657
+ });
658
+ return undefined;
659
+ }
660
+ const data = await SessionManager.buildProvingData([prover]);
221
661
  return await uploadEpochProofFailure(
222
662
  this.config.proverNodeFailedEpochStore,
223
- job.getId(),
224
- job.getProvingData(),
663
+ // The prover's content-addressed id; the epoch number is already in the upload path.
664
+ prover.id,
665
+ data,
225
666
  this.l2BlockSource as Archiver,
226
667
  this.worldState,
227
668
  assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
228
669
  this.log,
229
670
  );
671
+ } catch (err) {
672
+ this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
673
+ return undefined;
230
674
  }
231
675
  }
232
676
 
233
- /**
234
- * Returns the prover instance.
235
- */
236
- public getProver() {
237
- return this.prover;
238
- }
677
+ // ---------------- helpers ----------------
239
678
 
240
679
  /**
241
- * Returns an array of jobs being processed.
680
+ * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
681
+ * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
682
+ * so this returns false. Protected for direct unit-test access.
242
683
  */
243
- public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
244
- return Promise.resolve(
245
- Array.from(this.jobs.entries()).map(([uuid, job]) => ({
246
- uuid,
247
- status: job.getState(),
248
- epochNumber: job.getEpochNumber(),
249
- })),
250
- );
251
- }
252
-
253
- protected async getActiveJobsForEpoch(
254
- epochNumber: EpochNumber,
255
- ): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
256
- const jobs = await this.getJobs();
257
- return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
258
- }
259
-
260
- private checkMaximumPendingJobs() {
261
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
262
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
263
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
684
+ protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
685
+ const lastBlock = checkpoint.blocks.at(-1);
686
+ if (!lastBlock) {
687
+ return false;
264
688
  }
265
- }
266
-
267
- @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
268
- private async createProvingJob(epochNumber: EpochNumber, opts: { skipEpochCheck?: boolean } = {}) {
269
- this.checkMaximumPendingJobs();
270
-
271
- this.publisher = await this.publisherFactory.create();
272
-
273
- // Gather all data for this epoch
274
- const epochData = await this.gatherEpochData(epochNumber);
275
- const fromCheckpoint = epochData.checkpoints[0].number;
276
- const toCheckpoint = epochData.checkpoints.at(-1)!.number;
277
- const fromBlock = epochData.checkpoints[0].blocks[0].number;
278
- const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
279
- this.log.verbose(
280
- `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
281
- );
282
-
283
- // Fast forward world state to right before the target block and get a fork
284
- await this.worldState.syncImmediate(toBlock);
285
-
286
- // Create a processor factory
287
- const publicProcessorFactory = new PublicProcessorFactory(
288
- this.contractDataSource,
289
- this.dateProvider,
290
- this.telemetryClient,
291
- this.log.getBindings(),
292
- );
293
-
294
- // Set deadline for this job to run. It will abort if it takes too long.
295
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
296
- const deadline = new Date(Number(deadlineTs) * 1000);
297
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
298
- this.jobs.set(job.getId(), job);
299
- return job;
689
+ const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
690
+ return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
300
691
  }
301
692
 
302
693
  @memoize
303
- private getL1Constants() {
694
+ private getL1Constants(): Promise<L1RollupConstants> {
304
695
  return this.l2BlockSource.getL1Constants();
305
696
  }
306
697
 
307
- @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
308
- private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
309
- const checkpoints = await this.gatherCheckpoints(epochNumber);
310
- const txArray = await this.gatherTxs(epochNumber, checkpoints);
311
- const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
312
- const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
313
- const [firstBlock] = checkpoints[0].blocks;
314
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
315
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
316
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
317
-
318
- return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
319
- }
320
-
321
- private async gatherCheckpoints(epochNumber: EpochNumber) {
322
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
323
- if (checkpoints.length === 0) {
324
- throw new EmptyEpochError(epochNumber);
698
+ /**
699
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
700
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
701
+ */
702
+ protected async isEpochFullyProven(
703
+ epochNumber: EpochNumber,
704
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
705
+ ): Promise<boolean> {
706
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
707
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
708
+ return false;
325
709
  }
326
- return checkpoints;
327
- }
328
-
329
- private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
330
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
331
- const txProvider = this.p2pClient.getTxProvider();
332
- const blocks = checkpoints.flatMap(checkpoint => checkpoint.blocks);
333
- const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
334
- const txs = txsByBlock.map(({ txs }) => txs).flat();
335
- const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
336
-
337
- if (missingTxs.length === 0) {
338
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
339
- return txs;
710
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
711
+ if (!provenHeader) {
712
+ return false;
340
713
  }
341
-
342
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map(hash => hash.toString()).join(', ')}`);
714
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
715
+ if (epochNumber < provenEpoch) {
716
+ return true;
717
+ }
718
+ if (epochNumber > provenEpoch) {
719
+ return false;
720
+ }
721
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
343
722
  }
344
723
 
345
- private async gatherMessages(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
346
- const messages = await Promise.all(checkpoints.map(c => this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
347
- const messageCount = sum(messages.map(m => m.length));
348
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
349
- const messagesByCheckpoint: Record<CheckpointNumber, Fr[]> = {};
350
- for (let i = 0; i < checkpoints.length; i++) {
351
- messagesByCheckpoint[checkpoints[i].number] = messages[i];
724
+ /** Protected for direct unit-test access. */
725
+ protected async isProvenBlockLastOfItsEpoch(
726
+ provenBlockNumber: BlockNumber,
727
+ provenEpoch: EpochNumber,
728
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
729
+ ): Promise<boolean> {
730
+ const nextHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber + 1) }))?.header;
731
+ if (nextHeader) {
732
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
352
733
  }
353
- return messagesByCheckpoint;
734
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
354
735
  }
355
736
 
356
- private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
357
- const header = await (previousBlockNumber === 0
358
- ? this.worldState.getCommitted().getInitialHeader()
359
- : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
360
-
361
- if (!header) {
362
- throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
737
+ /**
738
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
739
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
740
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
741
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
742
+ */
743
+ protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
744
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
745
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
746
+ return { lastFullyProvenEpoch: undefined };
363
747
  }
364
-
365
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
366
- return header;
748
+ const l1Constants = await this.getL1Constants();
749
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
750
+ if (!provenHeader) {
751
+ return { lastFullyProvenEpoch: undefined };
752
+ }
753
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
754
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
755
+ return { lastFullyProvenEpoch: provenEpoch };
756
+ }
757
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
758
+ return { lastFullyProvenEpoch };
367
759
  }
368
760
 
369
- /** Extracted for testing purposes. */
370
- protected doCreateEpochProvingJob(
371
- data: EpochProvingJobData,
372
- deadline: Date | undefined,
373
- publicProcessorFactory: PublicProcessorFactory,
374
- publisher: ProverNodePublisher,
375
- opts: { skipEpochCheck?: boolean } = {},
376
- ) {
377
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
378
- return new EpochProvingJob(
379
- data,
380
- this.worldState,
381
- this.prover.createEpochProver(),
382
- publicProcessorFactory,
383
- publisher,
384
- this.l2BlockSource,
385
- this.jobMetrics,
386
- deadline,
387
- { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
388
- this.log.getBindings(),
389
- );
761
+ /**
762
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
763
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
764
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
765
+ */
766
+ protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
767
+ if (lastFullyProvenEpoch === undefined) {
768
+ return CheckpointNumber.ZERO;
769
+ }
770
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
771
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
390
772
  }
391
773
 
392
- /** Extracted for testing purposes. */
393
- protected async triggerMonitors() {
394
- await this.epochsMonitor.work();
774
+ private async gatherPreviousBlockHeader(previousBlockNumber: number) {
775
+ const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
776
+ if (!data?.header) {
777
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
778
+ }
779
+ return data.header;
395
780
  }
396
781
 
397
782
  private validateConfig() {
@@ -410,9 +795,5 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
410
795
  }
411
796
  }
412
797
 
413
- class EmptyEpochError extends Error {
414
- constructor(epochNumber: EpochNumber) {
415
- super(`No blocks found for epoch ${epochNumber}`);
416
- this.name = 'EmptyEpochError';
417
- }
418
- }
798
+ // Re-export so handlers can compare states externally.
799
+ export { EpochProvingJobTerminalState };