@aztec/prover-node 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +572 -0
  2. package/dest/actions/download-epoch-proving-job.js +1 -1
  3. package/dest/actions/rerun-epoch-proving-job.d.ts +14 -4
  4. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  5. package/dest/actions/rerun-epoch-proving-job.js +244 -24
  6. package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
  7. package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
  8. package/dest/bin/run-failed-epoch.js +6 -5
  9. package/dest/checkpoint-store.d.ts +95 -0
  10. package/dest/checkpoint-store.d.ts.map +1 -0
  11. package/dest/checkpoint-store.js +178 -0
  12. package/dest/config.d.ts +7 -8
  13. package/dest/config.d.ts.map +1 -1
  14. package/dest/config.js +24 -20
  15. package/dest/factory.d.ts +22 -13
  16. package/dest/factory.d.ts.map +1 -1
  17. package/dest/factory.js +41 -65
  18. package/dest/index.d.ts +2 -1
  19. package/dest/index.d.ts.map +1 -1
  20. package/dest/index.js +1 -0
  21. package/dest/job/checkpoint-prover.d.ts +165 -0
  22. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  23. package/dest/job/checkpoint-prover.js +405 -0
  24. package/dest/job/epoch-session.d.ts +160 -0
  25. package/dest/job/epoch-session.d.ts.map +1 -0
  26. package/dest/job/{epoch-proving-job.js → epoch-session.js} +301 -298
  27. package/dest/job/top-tree-job.d.ts +82 -0
  28. package/dest/job/top-tree-job.d.ts.map +1 -0
  29. package/dest/job/top-tree-job.js +152 -0
  30. package/dest/metrics.d.ts +40 -3
  31. package/dest/metrics.d.ts.map +1 -1
  32. package/dest/metrics.js +101 -4
  33. package/dest/monitors/epoch-monitor.d.ts +1 -1
  34. package/dest/monitors/epoch-monitor.d.ts.map +1 -1
  35. package/dest/monitors/epoch-monitor.js +11 -9
  36. package/dest/proof-publishing-service.d.ts +161 -0
  37. package/dest/proof-publishing-service.d.ts.map +1 -0
  38. package/dest/proof-publishing-service.js +335 -0
  39. package/dest/prover-node-publisher.d.ts +25 -15
  40. package/dest/prover-node-publisher.d.ts.map +1 -1
  41. package/dest/prover-node-publisher.js +202 -63
  42. package/dest/prover-node.d.ts +146 -69
  43. package/dest/prover-node.d.ts.map +1 -1
  44. package/dest/prover-node.js +545 -221
  45. package/dest/prover-publisher-factory.d.ts +6 -4
  46. package/dest/prover-publisher-factory.d.ts.map +1 -1
  47. package/dest/prover-publisher-factory.js +4 -3
  48. package/dest/session-manager.d.ts +158 -0
  49. package/dest/session-manager.d.ts.map +1 -0
  50. package/dest/session-manager.js +492 -0
  51. package/dest/test/index.d.ts +7 -6
  52. package/dest/test/index.d.ts.map +1 -1
  53. package/package.json +24 -22
  54. package/src/actions/download-epoch-proving-job.ts +1 -1
  55. package/src/actions/rerun-epoch-proving-job.ts +190 -31
  56. package/src/actions/upload-epoch-proof-failure.ts +1 -1
  57. package/src/bin/run-failed-epoch.ts +5 -3
  58. package/src/checkpoint-store.ts +212 -0
  59. package/src/config.ts +35 -32
  60. package/src/factory.ts +73 -111
  61. package/src/index.ts +1 -0
  62. package/src/job/checkpoint-prover.ts +538 -0
  63. package/src/job/epoch-session.ts +462 -0
  64. package/src/job/top-tree-job.ts +227 -0
  65. package/src/metrics.ts +123 -10
  66. package/src/monitors/epoch-monitor.ts +5 -6
  67. package/src/proof-publishing-service.ts +427 -0
  68. package/src/prover-node-publisher.ts +237 -79
  69. package/src/prover-node.ts +631 -248
  70. package/src/prover-publisher-factory.ts +8 -5
  71. package/src/session-manager.ts +592 -0
  72. package/src/test/index.ts +6 -6
  73. package/dest/job/epoch-proving-job.d.ts +0 -63
  74. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  75. package/src/job/epoch-proving-job.ts +0 -435
@@ -1,89 +1,137 @@
1
1
  import type { Archiver } from '@aztec/archiver';
2
2
  import type { RollupContract } from '@aztec/ethereum/contracts';
3
+ import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
3
4
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
4
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
5
- import type { Fr } from '@aztec/foundation/curves/bn254';
5
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
6
6
  import { memoize } from '@aztec/foundation/decorators';
7
7
  import { createLogger } from '@aztec/foundation/log';
8
- import { DateProvider } from '@aztec/foundation/timer';
9
- import type { DataStoreConfig } from '@aztec/kv-store/config';
10
- import type { P2PClient } from '@aztec/p2p';
11
- import { PublicProcessorFactory } from '@aztec/simulator/server';
12
- import type { L2BlockSource } from '@aztec/stdlib/block';
13
- import type { Checkpoint } from '@aztec/stdlib/checkpoint';
8
+ import { RunningPromise } from '@aztec/foundation/running-promise';
9
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
10
+ import type { EpochProverFactory } from '@aztec/prover-client';
11
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
12
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
13
+ import { type AvmSimulator, PublicProcessorFactory } from '@aztec/simulator/server';
14
+ import {
15
+ EventDrivenL2BlockStream,
16
+ type L2BlockId,
17
+ type L2BlockSource,
18
+ type L2BlockStreamEvent,
19
+ type L2BlockStreamEventHandler,
20
+ L2TipsMemoryStore,
21
+ } from '@aztec/stdlib/block';
22
+ import type { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
14
23
  import type { ChainConfig } from '@aztec/stdlib/config';
15
24
  import type { ContractDataSource } from '@aztec/stdlib/contract';
16
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
25
+ import { type L1RollupConstants, getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
17
26
  import {
18
27
  type EpochProverManager,
28
+ type EpochProvingJobState,
19
29
  EpochProvingJobTerminalState,
30
+ type ITxProvider,
20
31
  type ProverNodeApi,
21
32
  type Service,
22
- type WorldStateSyncStatus,
23
33
  type WorldStateSynchronizer,
24
34
  tryStop,
25
35
  } from '@aztec/stdlib/interfaces/server';
36
+ import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
26
37
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
27
- import type { P2PClientType } from '@aztec/stdlib/p2p';
28
- import type { Tx } from '@aztec/stdlib/tx';
38
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
29
39
  import {
30
- Attributes,
31
40
  L1Metrics,
32
41
  type TelemetryClient,
33
42
  type Traceable,
34
43
  type Tracer,
35
44
  getTelemetryClient,
36
- trackSpan,
37
45
  } from '@aztec/telemetry-client';
38
46
 
39
47
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
48
+ import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
40
49
  import type { SpecificProverNodeConfig } from './config.js';
41
- import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
42
- import { EpochProvingJob, type EpochProvingJobState } from './job/epoch-proving-job.js';
50
+ import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
51
+ import type { EpochSessionHooks } from './job/epoch-session.js';
43
52
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
44
- import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
45
- import type { ProverNodePublisher } from './prover-node-publisher.js';
53
+ import { ProofPublishingService } from './proof-publishing-service.js';
46
54
  import type { ProverPublisherFactory } from './prover-publisher-factory.js';
55
+ import { SessionManager } from './session-manager.js';
47
56
 
48
57
  type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
49
58
  type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
50
59
 
51
60
  /**
52
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
53
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
54
- * proof for the epoch, and submits it to L1.
61
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
62
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
63
+ * the wait rather than letting `stop()` hang indefinitely.
64
+ */
65
+ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
66
+
67
+ /**
68
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
69
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
70
+ * complete.
71
+ *
72
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
73
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
74
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
75
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
55
76
  */
56
- export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
77
+ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Traceable {
57
78
  private log = createLogger('prover-node');
58
- private dateProvider = new DateProvider();
59
79
 
60
- private jobs: Map<string, EpochProvingJob> = new Map();
61
- private config: ProverNodeOptions;
62
- private jobMetrics: ProverNodeJobMetrics;
63
- private rewardsMetrics: ProverNodeRewardsMetrics;
80
+ protected readonly checkpointStore: CheckpointStore;
81
+ protected readonly chonkCache: ChonkCache;
82
+ protected sessionManager: SessionManager | undefined;
83
+
84
+ private readonly config: ProverNodeOptions;
85
+ private readonly jobMetrics: ProverNodeJobMetrics;
86
+ private readonly rewardsMetrics: ProverNodeRewardsMetrics;
87
+
88
+ /** In-memory store for the L2BlockStream's local data provider. */
89
+ private tipsStore: L2TipsMemoryStore;
90
+ /** Block stream for checkpoint and reorg detection. */
91
+ private blockStream: EventDrivenL2BlockStream | undefined;
92
+ /**
93
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
94
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
95
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
96
+ * submission deadline. Protected so tests can verify the start() seeding.
97
+ */
98
+ protected lastExpiredEpoch: EpochNumber | undefined;
99
+
100
+ /**
101
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
102
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
103
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
104
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
105
+ */
106
+ protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
107
+
108
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
109
+ private expiryTicker: RunningPromise | undefined;
64
110
 
65
111
  public readonly tracer: Tracer;
66
112
 
67
- protected publisher: ProverNodePublisher | undefined;
113
+ protected publishingService: ProofPublishingService | undefined;
68
114
 
69
115
  constructor(
70
- protected readonly prover: EpochProverManager,
116
+ protected readonly prover: EpochProverManager & EpochProverFactory,
71
117
  protected readonly publisherFactory: ProverPublisherFactory,
72
118
  protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
73
119
  protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
74
120
  protected readonly contractDataSource: ContractDataSource,
75
121
  protected readonly worldState: WorldStateSynchronizer,
76
- protected readonly p2pClient: Pick<P2PClient<P2PClientType.Prover>, 'getTxProvider'> & Partial<Service>,
77
- protected readonly epochsMonitor: EpochMonitor,
122
+ protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
78
123
  protected readonly rollupContract: RollupContract,
79
124
  protected readonly l1Metrics: L1Metrics,
125
+ private readonly avmSimulator: AvmSimulator,
80
126
  config: Partial<ProverNodeOptions> = {},
81
127
  protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
128
+ private delayer?: Delayer,
129
+ private readonly dateProvider: DateProvider = new DateProvider(),
82
130
  ) {
83
131
  this.config = {
84
132
  proverNodePollingIntervalMs: 1_000,
85
133
  proverNodeMaxPendingJobs: 100,
86
- proverNodeMaxParallelBlocksPerEpoch: 32,
134
+ proverNodeMaxParallelBlocksPerEpoch: 0,
87
135
  txGatheringIntervalMs: 1_000,
88
136
  txGatheringBatchSize: 10,
89
137
  txGatheringMaxParallelRequestsPerNode: 100,
@@ -99,8 +147,36 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
99
147
  this.tracer = telemetryClient.getTracer('ProverNode');
100
148
 
101
149
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
102
-
103
150
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
151
+
152
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
153
+
154
+ this.chonkCache = new ChonkCache(this.log.getBindings());
155
+ this.checkpointStore = new CheckpointStore(
156
+ this.l2BlockSource,
157
+ {
158
+ proverFactory: this.prover,
159
+ chonkCache: this.chonkCache,
160
+ publicProcessorFactory: new PublicProcessorFactory(
161
+ this.contractDataSource,
162
+ this.avmSimulator,
163
+ this.dateProvider,
164
+ this.telemetryClient,
165
+ this.log.getBindings(),
166
+ ),
167
+ dbProvider: this.worldState,
168
+ txProvider: this.p2pClient.getTxProvider(),
169
+ dateProvider: this.dateProvider,
170
+ proverId: this.prover.getProverId(),
171
+ metrics: this.jobMetrics,
172
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
173
+ deadline: undefined,
174
+ // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
175
+ // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
176
+ onFailed: prover => void this.tryUploadCheckpointFailure(prover),
177
+ },
178
+ this.log.getBindings(),
179
+ );
104
180
  }
105
181
 
106
182
  public getProverId() {
@@ -111,287 +187,598 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
111
187
  return this.p2pClient;
112
188
  }
113
189
 
190
+ /** Test-only: the shared L1 tx delayer, if enabled. */
191
+ public getDelayer(): Delayer | undefined {
192
+ return this.delayer;
193
+ }
194
+
195
+ /** Observability summary for the ProverNodeApi. */
196
+ public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
197
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
198
+ }
199
+
200
+ /** Tests inspect this when validating reconcile behaviour. */
201
+ public getCheckpointStore(): CheckpointStore {
202
+ return this.checkpointStore;
203
+ }
204
+
205
+ /** Tests inspect this to verify chonk-cache release semantics. */
206
+ public getChonkCache(): ChonkCache {
207
+ return this.chonkCache;
208
+ }
209
+
210
+ /** Tests inspect this when looking up live sessions. */
211
+ public getSessionManager(): SessionManager {
212
+ if (!this.sessionManager) {
213
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
214
+ }
215
+ return this.sessionManager;
216
+ }
217
+
218
+ /** Returns the underlying prover instance. */
219
+ public getProver() {
220
+ return this.prover;
221
+ }
222
+
223
+ // ---------------- L2BlockStream handler ----------------
224
+
225
+ public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
226
+ switch (event.type) {
227
+ case 'chain-checkpointed':
228
+ await this.processCheckpointJump(event.checkpoint.number);
229
+ break;
230
+ case 'chain-pruned':
231
+ await this.handlePruneEvent(event.block);
232
+ break;
233
+ case 'chain-proven':
234
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
235
+ break;
236
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
237
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
238
+ // `chain-finalized` carries nothing the prover-node acts on.
239
+ case 'chain-proposed':
240
+ case 'chain-finalized':
241
+ case 'blocks-added':
242
+ break;
243
+ default: {
244
+ const _: never = event;
245
+ break;
246
+ }
247
+ }
248
+ // Advance the local tips store only after the proving-side handling (registration / prune) has
249
+ // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
250
+ // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
251
+ await this.tipsStore.handleBlockStreamEvent(event);
252
+ }
253
+
114
254
  /**
115
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
116
- * @param epochNumber - The epoch number that was just completed.
117
- * @returns false if there is an error, true otherwise
255
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
256
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
257
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
258
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
259
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
260
+ *
261
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
262
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
263
+ * behind so the next pass retries from the first checkpoint that did not complete.
118
264
  */
119
- async handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean> {
120
- try {
121
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
122
- jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
265
+ private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
266
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
267
+ return;
268
+ }
269
+ const l1Constants = await this.getL1Constants();
270
+
271
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
272
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
273
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
274
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
275
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
276
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
277
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
278
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
279
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
280
+ from,
281
+ cappedFrom,
282
+ targetCheckpoint,
283
+ maxCheckpoints,
123
284
  });
124
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
125
- if (activeJobs.length > 0) {
126
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
127
- activeJobs: activeJobs.map(job => job.uuid),
128
- });
129
- return true;
285
+ // Advance the cursor past the skipped checkpoints so they are never retried.
286
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
287
+ from = cappedFrom;
288
+ }
289
+ const limit = Number(targetCheckpoint - from) + 1;
290
+ const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
291
+
292
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
293
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
294
+ // never drop an individual checkpoint inside an epoch we will prove.
295
+ const epochSkippable = new Map<EpochNumber, boolean>();
296
+ for (const metadata of metadatas) {
297
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
298
+ let skippable = epochSkippable.get(epochNumber);
299
+ if (skippable === undefined) {
300
+ skippable =
301
+ (await this.isEpochFullyProven(epochNumber, l1Constants)) ||
302
+ (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
303
+ epochSkippable.set(epochNumber, skippable);
130
304
  }
131
- await this.startProof(epochNumber);
132
- return true;
133
- } catch (err) {
134
- if (err instanceof EmptyEpochError) {
135
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
305
+ if (skippable) {
306
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
136
307
  } else {
137
- this.log.error(`Error handling epoch completed`, err);
308
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
138
309
  }
310
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
311
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
312
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
313
+ }
314
+ }
315
+
316
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
317
+ private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
318
+ const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
319
+ if (!published) {
320
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
321
+ }
322
+ const checkpoint = published.checkpoint;
323
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
324
+ checkpointNumber: checkpoint.number,
325
+ epochNumber,
326
+ slotNumber: checkpoint.header.slotNumber,
327
+ });
328
+
329
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
330
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
331
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
332
+
333
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
334
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
335
+ await this.tipsStore.recordBlockHashes(
336
+ await Promise.all(
337
+ checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
338
+ ),
339
+ );
340
+ }
341
+
342
+ /**
343
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
344
+ * and the archive sibling path.
345
+ */
346
+ private async collectRegisterData(
347
+ checkpoint: Checkpoint,
348
+ attestations: PublishedCheckpoint['attestations'],
349
+ ): Promise<RegisterCheckpointData> {
350
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
351
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
352
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
353
+ const lastBlock = checkpoint.blocks.at(-1)!;
354
+ const lastBlockHash = await lastBlock.header.hash();
355
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
356
+ const previousArchiveSiblingPath = await getLastSiblingPath(
357
+ MerkleTreeId.ARCHIVE,
358
+ this.worldState.getSnapshot(previousBlockNumber),
359
+ );
360
+ return {
361
+ attestations,
362
+ previousBlockHeader,
363
+ l1ToL2Messages,
364
+ previousArchiveSiblingPath,
365
+ };
366
+ }
367
+
368
+ /**
369
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
370
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
371
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
372
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
373
+ * fails and the prune is retried next iteration.
374
+ */
375
+ private async handlePruneEvent(prunedToBlock: L2BlockId) {
376
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
377
+
378
+ // Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
379
+ // so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
380
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
381
+ let cursorFloor: CheckpointNumber;
382
+ if (prunedToBlock.number === 0) {
383
+ cursorFloor = CheckpointNumber.ZERO;
384
+ } else {
385
+ const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
386
+ if (targetData === undefined) {
387
+ throw new Error(
388
+ `No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
389
+ );
390
+ }
391
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
392
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
393
+ // would permanently skip a rebuilt same-number checkpoint.
394
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
395
+ }
396
+
397
+ const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
398
+
399
+ if (this.lastProcessedCheckpoint > cursorFloor) {
400
+ this.lastProcessedCheckpoint = cursorFloor;
401
+ }
402
+
403
+ if (affected.length === 0) {
404
+ return;
405
+ }
406
+ const l1Constants = await this.getL1Constants();
407
+ const affectedEpochs = Array.from(
408
+ new Set(affected.map(p => Number(getEpochAtSlot(p.slotNumber, l1Constants)))),
409
+ ).map(n => EpochNumber(n));
410
+ // The session manager cancels every affected session, which in turn calls
411
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
412
+ // publishing service is needed.
413
+ await this.sessionManager?.onPrune(affectedEpochs);
414
+ }
415
+
416
+ /**
417
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
418
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
419
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
420
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
421
+ * be read yet.
422
+ */
423
+ private async isEpochPastProofSubmissionWindow(
424
+ epochNumber: EpochNumber,
425
+ l1Constants: L1RollupConstants,
426
+ ): Promise<boolean> {
427
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
428
+ if (latestSlot === undefined) {
139
429
  return false;
140
430
  }
431
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
432
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
433
+ }
434
+
435
+ /**
436
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
437
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
438
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
439
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
440
+ * the last check or the archiver's slot can't be read.
441
+ */
442
+ private async checkEpochExpiry(): Promise<void> {
443
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
444
+ if (latestSlot === undefined) {
445
+ return;
446
+ }
447
+ const l1Constants = await this.getL1Constants();
448
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
449
+ const offset = l1Constants.proofSubmissionEpochs + 1;
450
+ if (latestEpoch < offset) {
451
+ return;
452
+ }
453
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
454
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
455
+ if (newlyExpiredUpTo < from) {
456
+ return;
457
+ }
458
+ for (let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)) {
459
+ await this.expireEpoch(e);
460
+ }
461
+ this.lastExpiredEpoch = newlyExpiredUpTo;
141
462
  }
142
463
 
143
464
  /**
144
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
145
- * starts proving jobs for them.
465
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
466
+ * CheckpointProver in the store whose epoch is at or below it.
146
467
  */
468
+ private async expireEpoch(epoch: EpochNumber): Promise<void> {
469
+ try {
470
+ const blocks = await this.l2BlockSource.getBlocks({ epoch, onlyCheckpointed: true });
471
+ if (blocks.length > 0) {
472
+ this.chonkCache.releaseForBlocks(blocks);
473
+ }
474
+ } catch (err) {
475
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
476
+ }
477
+ this.checkpointStore.reapExpired(epoch);
478
+ }
479
+
480
+ // ---------------- public API ----------------
481
+
482
+ /**
483
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
484
+ */
485
+ public async startProof(epochNumber: EpochNumber): Promise<string> {
486
+ if (!this.sessionManager) {
487
+ throw new Error('ProverNode not started');
488
+ }
489
+ return await this.sessionManager.startProof(epochNumber);
490
+ }
491
+
492
+ // ---------------- Service lifecycle ----------------
493
+
147
494
  async start() {
148
- this.epochsMonitor.start(this);
495
+ await this.checkpointStore.start();
496
+
149
497
  await this.publisherFactory.start();
150
- this.publisher = await this.publisherFactory.create();
498
+ this.publishingService = new ProofPublishingService({
499
+ publisherFactory: this.publisherFactory,
500
+ l2BlockSource: this.l2BlockSource,
501
+ dateProvider: this.dateProvider,
502
+ config: { skipSubmitProof: !!this.config.proverNodeDisableProofPublish },
503
+ bindings: this.log.getBindings(),
504
+ });
505
+ this.sessionManager = this.createSessionManager(this.publishingService);
506
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
507
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
508
+ // the previous epoch is proven on L1.
509
+ this.sessionManager.start();
510
+ // Now that the store + manager exist, arm the live-state observable gauges.
511
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
512
+
513
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
514
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
515
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
516
+ this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
517
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
518
+ tipsOnly: true,
519
+ });
520
+ this.blockStream.start();
521
+
522
+ // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
523
+ // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
524
+ // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
525
+ this.expiryTicker = new RunningPromise(
526
+ () => this.checkEpochExpiry(),
527
+ this.log,
528
+ this.config.proverNodePollingIntervalMs,
529
+ );
530
+ this.expiryTicker.start();
531
+
151
532
  await this.rewardsMetrics.start();
152
533
  this.l1Metrics.start();
153
534
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
154
535
  }
155
536
 
156
- /**
157
- * Stops the prover node and all its dependencies.
158
- */
159
537
  async stop() {
160
538
  this.log.info('Stopping ProverNode');
161
- await this.epochsMonitor.stop();
539
+ this.jobMetrics.stopObservingState();
540
+ await this.blockStream?.stop();
541
+ await this.expiryTicker?.stop();
542
+ if (this.sessionManager) {
543
+ await this.sessionManager.stop();
544
+ }
545
+ if (this.publishingService) {
546
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
547
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
548
+ // the tx may still mine, but shutdown must not hang on it.
549
+ const publishingService = this.publishingService;
550
+ await executeTimeout(
551
+ () => publishingService.stop(),
552
+ PUBLISHING_SERVICE_STOP_TIMEOUT_MS,
553
+ 'prover-node publishing-service stop',
554
+ ).catch(err => this.log.warn(`Timed out stopping proof publishing service`, err));
555
+ }
556
+ await this.checkpointStore.stop();
557
+ this.chonkCache.stop();
162
558
  await this.prover.stop();
163
- await tryStop(this.p2pClient);
164
- await tryStop(this.l2BlockSource);
165
559
  await tryStop(this.publisherFactory);
166
- this.publisher?.interrupt();
167
- await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
168
- await this.worldState.stop();
169
560
  this.rewardsMetrics.stop();
170
561
  this.l1Metrics.stop();
171
562
  await this.telemetryClient.stop();
172
563
  this.log.info('Stopped ProverNode');
173
564
  }
174
565
 
175
- /** Returns world state status. */
176
- public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
177
- const { syncSummary } = await this.worldState.status();
178
- return syncSummary;
566
+ /**
567
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap the
568
+ * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
569
+ * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
570
+ * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
571
+ */
572
+ protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
573
+ return new SessionManager({
574
+ checkpointStore: this.checkpointStore,
575
+ l2BlockSource: this.l2BlockSource,
576
+ proverFactory: this.prover,
577
+ proverId: this.prover.getProverId(),
578
+ publishingService,
579
+ metrics: this.jobMetrics,
580
+ dateProvider: this.dateProvider,
581
+ config: {
582
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
583
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
584
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
585
+ },
586
+ onSessionFailed: async session => {
587
+ await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
588
+ },
589
+ bindings: this.log.getBindings(),
590
+ });
179
591
  }
180
592
 
181
- /** Returns archiver status. */
182
- public getL2Tips() {
183
- return this.l2BlockSource.getL2Tips();
593
+ /**
594
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
595
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
596
+ * Applies to every session constructed after this call.
597
+ */
598
+ public setSessionHooks(hooks: EpochSessionHooks): void {
599
+ if (!this.sessionManager) {
600
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
601
+ }
602
+ this.sessionManager.setSessionHooks(hooks);
184
603
  }
185
604
 
186
605
  /**
187
- * Starts a proving process and returns immediately.
606
+ * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
607
+ * CheckpointProver constructed after this call. For the e2e harness only.
188
608
  */
189
- public async startProof(epochNumber: EpochNumber) {
190
- const job = await this.createProvingJob(epochNumber, { skipEpochCheck: true });
191
- void this.runJob(job);
609
+ public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
610
+ this.checkpointStore.setTestHooks(hooks);
192
611
  }
193
612
 
194
- private async runJob(job: EpochProvingJob) {
195
- const epochNumber = job.getEpochNumber();
196
- const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
197
-
198
- try {
199
- await job.run();
200
- const state = job.getState();
201
- ctx.state = state;
202
-
203
- if (state === 'reorg') {
204
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
205
- await this.createProvingJob(epochNumber);
206
- } else if (state === 'failed') {
207
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
208
- await this.tryUploadEpochFailure(job);
209
- } else {
210
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
211
- }
212
- } catch (err) {
213
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
214
- } finally {
215
- this.jobs.delete(job.getId());
613
+ /**
614
+ * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
615
+ * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
616
+ * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
617
+ * store is configured or the checkpoint set is empty.
618
+ */
619
+ public async tryUploadEpochFailure(
620
+ id: string,
621
+ checkpoints: readonly CheckpointProver[],
622
+ ): Promise<string | undefined> {
623
+ if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
624
+ return undefined;
216
625
  }
626
+ const data = await SessionManager.buildProvingData(checkpoints);
627
+ return await uploadEpochProofFailure(
628
+ this.config.proverNodeFailedEpochStore,
629
+ // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
630
+ id,
631
+ data,
632
+ this.l2BlockSource as Archiver,
633
+ this.worldState,
634
+ assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
635
+ this.log,
636
+ );
217
637
  }
218
638
 
219
- protected async tryUploadEpochFailure(job: EpochProvingJob) {
220
- if (this.config.proverNodeFailedEpochStore) {
639
+ /**
640
+ * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
641
+ * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
642
+ * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
643
+ * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
644
+ * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
645
+ */
646
+ public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
647
+ if (!this.config.proverNodeFailedEpochStore) {
648
+ return undefined;
649
+ }
650
+ try {
651
+ // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
652
+ // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
653
+ // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
654
+ // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
655
+ // the snapshot (full world-state + archiver) is expensive to produce and store.
656
+ if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
657
+ this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
658
+ checkpointNumber: prover.checkpoint.number,
659
+ });
660
+ return undefined;
661
+ }
662
+ const data = await SessionManager.buildProvingData([prover]);
221
663
  return await uploadEpochProofFailure(
222
664
  this.config.proverNodeFailedEpochStore,
223
- job.getId(),
224
- job.getProvingData(),
665
+ // The prover's content-addressed id; the epoch number is already in the upload path.
666
+ prover.id,
667
+ data,
225
668
  this.l2BlockSource as Archiver,
226
669
  this.worldState,
227
670
  assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
228
671
  this.log,
229
672
  );
673
+ } catch (err) {
674
+ this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
675
+ return undefined;
230
676
  }
231
677
  }
232
678
 
233
- /**
234
- * Returns the prover instance.
235
- */
236
- public getProver() {
237
- return this.prover;
238
- }
679
+ // ---------------- helpers ----------------
239
680
 
240
681
  /**
241
- * Returns an array of jobs being processed.
682
+ * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
683
+ * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
684
+ * so this returns false. Protected for direct unit-test access.
242
685
  */
243
- public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
244
- return Promise.resolve(
245
- Array.from(this.jobs.entries()).map(([uuid, job]) => ({
246
- uuid,
247
- status: job.getState(),
248
- epochNumber: job.getEpochNumber(),
249
- })),
250
- );
251
- }
252
-
253
- protected async getActiveJobsForEpoch(
254
- epochNumber: EpochNumber,
255
- ): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
256
- const jobs = await this.getJobs();
257
- return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
258
- }
259
-
260
- private checkMaximumPendingJobs() {
261
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
262
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
263
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
686
+ protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
687
+ const lastBlock = checkpoint.blocks.at(-1);
688
+ if (!lastBlock) {
689
+ return false;
264
690
  }
265
- }
266
-
267
- @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
268
- private async createProvingJob(epochNumber: EpochNumber, opts: { skipEpochCheck?: boolean } = {}) {
269
- this.checkMaximumPendingJobs();
270
-
271
- this.publisher = await this.publisherFactory.create();
272
-
273
- // Gather all data for this epoch
274
- const epochData = await this.gatherEpochData(epochNumber);
275
- const fromCheckpoint = epochData.checkpoints[0].number;
276
- const toCheckpoint = epochData.checkpoints.at(-1)!.number;
277
- const fromBlock = epochData.checkpoints[0].blocks[0].number;
278
- const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
279
- this.log.verbose(
280
- `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
281
- );
282
-
283
- // Fast forward world state to right before the target block and get a fork
284
- await this.worldState.syncImmediate(toBlock);
285
-
286
- // Create a processor factory
287
- const publicProcessorFactory = new PublicProcessorFactory(
288
- this.contractDataSource,
289
- this.dateProvider,
290
- this.telemetryClient,
291
- this.log.getBindings(),
292
- );
293
-
294
- // Set deadline for this job to run. It will abort if it takes too long.
295
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
296
- const deadline = new Date(Number(deadlineTs) * 1000);
297
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
298
- this.jobs.set(job.getId(), job);
299
- return job;
691
+ const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
692
+ return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
300
693
  }
301
694
 
302
695
  @memoize
303
- private getL1Constants() {
696
+ private getL1Constants(): Promise<L1RollupConstants> {
304
697
  return this.l2BlockSource.getL1Constants();
305
698
  }
306
699
 
307
- @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
308
- private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
309
- const checkpoints = await this.gatherCheckpoints(epochNumber);
310
- const txArray = await this.gatherTxs(epochNumber, checkpoints);
311
- const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
312
- const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
313
- const [firstBlock] = checkpoints[0].blocks;
314
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
315
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1)!.number, 1);
316
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
317
-
318
- return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
319
- }
320
-
321
- private async gatherCheckpoints(epochNumber: EpochNumber) {
322
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
323
- if (checkpoints.length === 0) {
324
- throw new EmptyEpochError(epochNumber);
700
+ /**
701
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
702
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
703
+ */
704
+ protected async isEpochFullyProven(
705
+ epochNumber: EpochNumber,
706
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
707
+ ): Promise<boolean> {
708
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
709
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
710
+ return false;
325
711
  }
326
- return checkpoints;
327
- }
328
-
329
- private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
330
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
331
- const txProvider = this.p2pClient.getTxProvider();
332
- const blocks = checkpoints.flatMap(checkpoint => checkpoint.blocks);
333
- const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
334
- const txs = txsByBlock.map(({ txs }) => txs).flat();
335
- const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
336
-
337
- if (missingTxs.length === 0) {
338
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
339
- return txs;
712
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
713
+ if (!provenHeader) {
714
+ return false;
340
715
  }
341
-
342
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map(hash => hash.toString()).join(', ')}`);
716
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
717
+ if (epochNumber < provenEpoch) {
718
+ return true;
719
+ }
720
+ if (epochNumber > provenEpoch) {
721
+ return false;
722
+ }
723
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
343
724
  }
344
725
 
345
- private async gatherMessages(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
346
- const messages = await Promise.all(checkpoints.map(c => this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
347
- const messageCount = sum(messages.map(m => m.length));
348
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
349
- const messagesByCheckpoint: Record<CheckpointNumber, Fr[]> = {};
350
- for (let i = 0; i < checkpoints.length; i++) {
351
- messagesByCheckpoint[checkpoints[i].number] = messages[i];
726
+ /** Protected for direct unit-test access. */
727
+ protected async isProvenBlockLastOfItsEpoch(
728
+ provenBlockNumber: BlockNumber,
729
+ provenEpoch: EpochNumber,
730
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
731
+ ): Promise<boolean> {
732
+ const nextHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber + 1) }))?.header;
733
+ if (nextHeader) {
734
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
352
735
  }
353
- return messagesByCheckpoint;
736
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
354
737
  }
355
738
 
356
- private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
357
- const header = await (previousBlockNumber === 0
358
- ? this.worldState.getCommitted().getInitialHeader()
359
- : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
360
-
361
- if (!header) {
362
- throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
739
+ /**
740
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
741
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
742
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
743
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
744
+ */
745
+ protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
746
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
747
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
748
+ return { lastFullyProvenEpoch: undefined };
363
749
  }
364
-
365
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
366
- return header;
750
+ const l1Constants = await this.getL1Constants();
751
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
752
+ if (!provenHeader) {
753
+ return { lastFullyProvenEpoch: undefined };
754
+ }
755
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
756
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
757
+ return { lastFullyProvenEpoch: provenEpoch };
758
+ }
759
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
760
+ return { lastFullyProvenEpoch };
367
761
  }
368
762
 
369
- /** Extracted for testing purposes. */
370
- protected doCreateEpochProvingJob(
371
- data: EpochProvingJobData,
372
- deadline: Date | undefined,
373
- publicProcessorFactory: PublicProcessorFactory,
374
- publisher: ProverNodePublisher,
375
- opts: { skipEpochCheck?: boolean } = {},
376
- ) {
377
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
378
- return new EpochProvingJob(
379
- data,
380
- this.worldState,
381
- this.prover.createEpochProver(),
382
- publicProcessorFactory,
383
- publisher,
384
- this.l2BlockSource,
385
- this.jobMetrics,
386
- deadline,
387
- { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
388
- this.log.getBindings(),
389
- );
763
+ /**
764
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
765
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
766
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
767
+ */
768
+ protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
769
+ if (lastFullyProvenEpoch === undefined) {
770
+ return CheckpointNumber.ZERO;
771
+ }
772
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
773
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
390
774
  }
391
775
 
392
- /** Extracted for testing purposes. */
393
- protected async triggerMonitors() {
394
- await this.epochsMonitor.work();
776
+ private async gatherPreviousBlockHeader(previousBlockNumber: number) {
777
+ const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
778
+ if (!data?.header) {
779
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
780
+ }
781
+ return data.header;
395
782
  }
396
783
 
397
784
  private validateConfig() {
@@ -410,9 +797,5 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
410
797
  }
411
798
  }
412
799
 
413
- class EmptyEpochError extends Error {
414
- constructor(epochNumber: EpochNumber) {
415
- super(`No blocks found for epoch ${epochNumber}`);
416
- this.name = 'EmptyEpochError';
417
- }
418
- }
800
+ // Re-export so handlers can compare states externally.
801
+ export { EpochProvingJobTerminalState };