@aztec/prover-node 0.0.1-commit.2f68f620 → 0.0.1-commit.321f6a9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +572 -0
  2. package/dest/actions/rerun-epoch-proving-job.d.ts +12 -3
  3. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  4. package/dest/actions/rerun-epoch-proving-job.js +170 -28
  5. package/dest/checkpoint-store.d.ts +95 -0
  6. package/dest/checkpoint-store.d.ts.map +1 -0
  7. package/dest/checkpoint-store.js +178 -0
  8. package/dest/config.d.ts +1 -1
  9. package/dest/config.d.ts.map +1 -1
  10. package/dest/config.js +1 -1
  11. package/dest/factory.d.ts +4 -1
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +1 -6
  14. package/dest/index.d.ts +2 -1
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +1 -0
  17. package/dest/job/checkpoint-prover.d.ts +154 -0
  18. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  19. package/dest/job/checkpoint-prover.js +363 -0
  20. package/dest/job/epoch-session.d.ts +160 -0
  21. package/dest/job/epoch-session.d.ts.map +1 -0
  22. package/dest/job/epoch-session.js +744 -0
  23. package/dest/job/top-tree-job.d.ts +82 -0
  24. package/dest/job/top-tree-job.d.ts.map +1 -0
  25. package/dest/job/top-tree-job.js +152 -0
  26. package/dest/metrics.d.ts +25 -8
  27. package/dest/metrics.d.ts.map +1 -1
  28. package/dest/metrics.js +64 -14
  29. package/dest/proof-publishing-service.d.ts +161 -0
  30. package/dest/proof-publishing-service.d.ts.map +1 -0
  31. package/dest/proof-publishing-service.js +335 -0
  32. package/dest/prover-node-publisher.d.ts +7 -19
  33. package/dest/prover-node-publisher.d.ts.map +1 -1
  34. package/dest/prover-node-publisher.js +38 -96
  35. package/dest/prover-node.d.ts +133 -66
  36. package/dest/prover-node.d.ts.map +1 -1
  37. package/dest/prover-node.js +529 -255
  38. package/dest/session-manager.d.ts +158 -0
  39. package/dest/session-manager.d.ts.map +1 -0
  40. package/dest/session-manager.js +486 -0
  41. package/dest/test/index.d.ts +7 -6
  42. package/dest/test/index.d.ts.map +1 -1
  43. package/package.json +24 -23
  44. package/src/actions/rerun-epoch-proving-job.ts +178 -30
  45. package/src/checkpoint-store.ts +212 -0
  46. package/src/config.ts +2 -1
  47. package/src/factory.ts +4 -8
  48. package/src/index.ts +1 -0
  49. package/src/job/checkpoint-prover.ts +496 -0
  50. package/src/job/epoch-session.ts +462 -0
  51. package/src/job/top-tree-job.ts +227 -0
  52. package/src/metrics.ts +65 -23
  53. package/src/proof-publishing-service.ts +427 -0
  54. package/src/prover-node-publisher.ts +52 -121
  55. package/src/prover-node.ts +617 -277
  56. package/src/session-manager.ts +589 -0
  57. package/src/test/index.ts +6 -6
  58. package/dest/job/epoch-proving-job.d.ts +0 -67
  59. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  60. package/dest/job/epoch-proving-job.js +0 -912
  61. package/src/job/epoch-proving-job.ts +0 -531
@@ -2,80 +2,127 @@ import type { Archiver } from '@aztec/archiver';
2
2
  import type { RollupContract } from '@aztec/ethereum/contracts';
3
3
  import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
4
4
  import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
5
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
6
- import type { Fr } from '@aztec/foundation/curves/bn254';
5
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
7
6
  import { memoize } from '@aztec/foundation/decorators';
8
7
  import { createLogger } from '@aztec/foundation/log';
9
- import { DateProvider } from '@aztec/foundation/timer';
10
- import { PublicProcessorFactory } from '@aztec/simulator/server';
11
- import type { L2BlockSource } from '@aztec/stdlib/block';
12
- import type { Checkpoint } from '@aztec/stdlib/checkpoint';
8
+ import { RunningPromise } from '@aztec/foundation/running-promise';
9
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
10
+ import type { EpochProverFactory } from '@aztec/prover-client';
11
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
12
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
13
+ import { type AvmSimulator, PublicProcessorFactory } from '@aztec/simulator/server';
14
+ import {
15
+ EventDrivenL2BlockStream,
16
+ type L2BlockId,
17
+ type L2BlockSource,
18
+ type L2BlockStreamEvent,
19
+ type L2BlockStreamEventHandler,
20
+ L2TipsMemoryStore,
21
+ } from '@aztec/stdlib/block';
22
+ import type { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
13
23
  import type { ChainConfig } from '@aztec/stdlib/config';
14
24
  import type { ContractDataSource } from '@aztec/stdlib/contract';
15
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
25
+ import { type L1RollupConstants, getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
16
26
  import {
17
27
  type EpochProverManager,
28
+ type EpochProvingJobState,
29
+ EpochProvingJobTerminalState,
18
30
  type ITxProvider,
19
31
  type ProverNodeApi,
20
32
  type Service,
21
- type WorldStateSyncStatus,
22
33
  type WorldStateSynchronizer,
23
34
  tryStop,
24
35
  } from '@aztec/stdlib/interfaces/server';
25
36
  import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
26
37
  import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
27
- import type { Tx } from '@aztec/stdlib/tx';
38
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
28
39
  import {
29
- Attributes,
30
40
  L1Metrics,
31
41
  type TelemetryClient,
32
42
  type Traceable,
33
43
  type Tracer,
34
44
  getTelemetryClient,
35
- trackSpan,
36
45
  } from '@aztec/telemetry-client';
37
46
 
38
47
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
48
+ import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
39
49
  import type { SpecificProverNodeConfig } from './config.js';
40
- import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
41
- import { EpochProvingJob, type EpochProvingJobOptions, type EpochProvingJobState } from './job/epoch-proving-job.js';
50
+ import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
51
+ import type { EpochSessionHooks } from './job/epoch-session.js';
42
52
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
43
- import type { EpochMonitor, EpochMonitorHandler } from './monitors/epoch-monitor.js';
44
- import type { ProverNodePublisher } from './prover-node-publisher.js';
53
+ import { ProofPublishingService } from './proof-publishing-service.js';
45
54
  import type { ProverPublisherFactory } from './prover-publisher-factory.js';
55
+ import { SessionManager } from './session-manager.js';
46
56
 
47
57
  type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
48
58
  type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
49
59
 
50
60
  /**
51
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
52
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
53
- * proof for the epoch, and submits it to L1.
61
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
62
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
63
+ * the wait rather than letting `stop()` hang indefinitely.
64
+ */
65
+ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
66
+
67
+ /**
68
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
69
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
70
+ * complete.
71
+ *
72
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
73
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
74
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
75
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
54
76
  */
55
- export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable {
77
+ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Traceable {
56
78
  private log = createLogger('prover-node');
57
79
 
58
- private jobs: Map<string, EpochProvingJob> = new Map();
59
- private config: ProverNodeOptions;
60
- private jobMetrics: ProverNodeJobMetrics;
61
- private rewardsMetrics: ProverNodeRewardsMetrics;
62
- private startingProofEpochs: Set<EpochNumber> = new Set();
80
+ protected readonly checkpointStore: CheckpointStore;
81
+ protected readonly chonkCache: ChonkCache;
82
+ protected sessionManager: SessionManager | undefined;
83
+
84
+ private readonly config: ProverNodeOptions;
85
+ private readonly jobMetrics: ProverNodeJobMetrics;
86
+ private readonly rewardsMetrics: ProverNodeRewardsMetrics;
87
+
88
+ /** In-memory store for the L2BlockStream's local data provider. */
89
+ private tipsStore: L2TipsMemoryStore;
90
+ /** Block stream for checkpoint and reorg detection. */
91
+ private blockStream: EventDrivenL2BlockStream | undefined;
92
+ /**
93
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
94
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
95
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
96
+ * submission deadline. Protected so tests can verify the start() seeding.
97
+ */
98
+ protected lastExpiredEpoch: EpochNumber | undefined;
99
+
100
+ /**
101
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
102
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
103
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
104
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
105
+ */
106
+ protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
107
+
108
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
109
+ private expiryTicker: RunningPromise | undefined;
63
110
 
64
111
  public readonly tracer: Tracer;
65
112
 
66
- protected publisher: ProverNodePublisher | undefined;
113
+ protected publishingService: ProofPublishingService | undefined;
67
114
 
68
115
  constructor(
69
- protected readonly prover: EpochProverManager,
116
+ protected readonly prover: EpochProverManager & EpochProverFactory,
70
117
  protected readonly publisherFactory: ProverPublisherFactory,
71
118
  protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
72
119
  protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
73
120
  protected readonly contractDataSource: ContractDataSource,
74
121
  protected readonly worldState: WorldStateSynchronizer,
75
122
  protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
76
- protected readonly epochsMonitor: EpochMonitor,
77
123
  protected readonly rollupContract: RollupContract,
78
124
  protected readonly l1Metrics: L1Metrics,
125
+ private readonly avmSimulator: AvmSimulator,
79
126
  config: Partial<ProverNodeOptions> = {},
80
127
  protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
81
128
  private delayer?: Delayer,
@@ -100,8 +147,36 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
100
147
  this.tracer = telemetryClient.getTracer('ProverNode');
101
148
 
102
149
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
103
-
104
150
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
151
+
152
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
153
+
154
+ this.chonkCache = new ChonkCache(this.log.getBindings());
155
+ this.checkpointStore = new CheckpointStore(
156
+ this.l2BlockSource,
157
+ {
158
+ proverFactory: this.prover,
159
+ chonkCache: this.chonkCache,
160
+ publicProcessorFactory: new PublicProcessorFactory(
161
+ this.contractDataSource,
162
+ this.avmSimulator,
163
+ this.dateProvider,
164
+ this.telemetryClient,
165
+ this.log.getBindings(),
166
+ ),
167
+ dbProvider: this.worldState,
168
+ txProvider: this.p2pClient.getTxProvider(),
169
+ dateProvider: this.dateProvider,
170
+ proverId: this.prover.getProverId(),
171
+ metrics: this.jobMetrics,
172
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
173
+ deadline: undefined,
174
+ // A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
175
+ // post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
176
+ onFailed: prover => void this.tryUploadCheckpointFailure(prover),
177
+ },
178
+ this.log.getBindings(),
179
+ );
105
180
  }
106
181
 
107
182
  public getProverId() {
@@ -112,54 +187,374 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
112
187
  return this.p2pClient;
113
188
  }
114
189
 
115
- /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */
190
+ /** Test-only: the shared L1 tx delayer, if enabled. */
116
191
  public getDelayer(): Delayer | undefined {
117
192
  return this.delayer;
118
193
  }
119
194
 
195
+ /** Observability summary for the ProverNodeApi. */
196
+ public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
197
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
198
+ }
199
+
200
+ /** Tests inspect this when validating reconcile behaviour. */
201
+ public getCheckpointStore(): CheckpointStore {
202
+ return this.checkpointStore;
203
+ }
204
+
205
+ /** Tests inspect this to verify chonk-cache release semantics. */
206
+ public getChonkCache(): ChonkCache {
207
+ return this.chonkCache;
208
+ }
209
+
210
+ /** Tests inspect this when looking up live sessions. */
211
+ public getSessionManager(): SessionManager {
212
+ if (!this.sessionManager) {
213
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
214
+ }
215
+ return this.sessionManager;
216
+ }
217
+
218
+ /** Returns the underlying prover instance. */
219
+ public getProver() {
220
+ return this.prover;
221
+ }
222
+
223
+ // ---------------- L2BlockStream handler ----------------
224
+
225
+ public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
226
+ switch (event.type) {
227
+ case 'chain-checkpointed':
228
+ await this.processCheckpointJump(event.checkpoint.number);
229
+ break;
230
+ case 'chain-pruned':
231
+ await this.handlePruneEvent(event.block);
232
+ break;
233
+ case 'chain-proven':
234
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
235
+ break;
236
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
237
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
238
+ // `chain-finalized` carries nothing the prover-node acts on.
239
+ case 'chain-proposed':
240
+ case 'chain-finalized':
241
+ case 'blocks-added':
242
+ break;
243
+ default: {
244
+ const _: never = event;
245
+ break;
246
+ }
247
+ }
248
+ // Advance the local tips store only after the proving-side handling (registration / prune) has
249
+ // succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
250
+ // pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
251
+ await this.tipsStore.handleBlockStreamEvent(event);
252
+ }
253
+
120
254
  /**
121
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
122
- * @param epochNumber - The epoch number that was just completed.
123
- * @returns false if there is an error, true otherwise
255
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
256
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
257
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
258
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
259
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
260
+ *
261
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
262
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
263
+ * behind so the next pass retries from the first checkpoint that did not complete.
124
264
  */
125
- async handleEpochReadyToProve(epochNumber: EpochNumber): Promise<boolean> {
126
- try {
127
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
128
- jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
265
+ private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
266
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
267
+ return;
268
+ }
269
+ const l1Constants = await this.getL1Constants();
270
+
271
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
272
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
273
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
274
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
275
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
276
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
277
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
278
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
279
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
280
+ from,
281
+ cappedFrom,
282
+ targetCheckpoint,
283
+ maxCheckpoints,
129
284
  });
130
- return await this.startProofIfNeeded(epochNumber);
131
- } catch (err) {
132
- if (err instanceof EmptyEpochError) {
133
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
285
+ // Advance the cursor past the skipped checkpoints so they are never retried.
286
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
287
+ from = cappedFrom;
288
+ }
289
+ const limit = Number(targetCheckpoint - from) + 1;
290
+ const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
291
+
292
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
293
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
294
+ // never drop an individual checkpoint inside an epoch we will prove.
295
+ const epochSkippable = new Map<EpochNumber, boolean>();
296
+ for (const metadata of metadatas) {
297
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
298
+ let skippable = epochSkippable.get(epochNumber);
299
+ if (skippable === undefined) {
300
+ skippable =
301
+ (await this.isEpochFullyProven(epochNumber, l1Constants)) ||
302
+ (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
303
+ epochSkippable.set(epochNumber, skippable);
304
+ }
305
+ if (skippable) {
306
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
134
307
  } else {
135
- this.log.error(`Error handling epoch completed`, err);
308
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
309
+ }
310
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
311
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
312
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
313
+ }
314
+ }
315
+
316
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
317
+ private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
318
+ const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
319
+ if (!published) {
320
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
321
+ }
322
+ const checkpoint = published.checkpoint;
323
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
324
+ checkpointNumber: checkpoint.number,
325
+ epochNumber,
326
+ slotNumber: checkpoint.header.slotNumber,
327
+ });
328
+
329
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
330
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
331
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
332
+
333
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
334
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
335
+ await this.tipsStore.recordBlockHashes(
336
+ await Promise.all(
337
+ checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
338
+ ),
339
+ );
340
+ }
341
+
342
+ /**
343
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
344
+ * and the archive sibling path.
345
+ */
346
+ private async collectRegisterData(
347
+ checkpoint: Checkpoint,
348
+ attestations: PublishedCheckpoint['attestations'],
349
+ ): Promise<RegisterCheckpointData> {
350
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
351
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
352
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
353
+ const lastBlock = checkpoint.blocks.at(-1)!;
354
+ const lastBlockHash = await lastBlock.header.hash();
355
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
356
+ const previousArchiveSiblingPath = await getLastSiblingPath(
357
+ MerkleTreeId.ARCHIVE,
358
+ this.worldState.getSnapshot(previousBlockNumber),
359
+ );
360
+ return {
361
+ attestations,
362
+ previousBlockHeader,
363
+ l1ToL2Messages,
364
+ previousArchiveSiblingPath,
365
+ };
366
+ }
367
+
368
+ /**
369
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
370
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
371
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
372
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
373
+ * fails and the prune is retried next iteration.
374
+ */
375
+ private async handlePruneEvent(prunedToBlock: L2BlockId) {
376
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
377
+
378
+ // Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
379
+ // so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
380
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
381
+ let cursorFloor: CheckpointNumber;
382
+ if (prunedToBlock.number === 0) {
383
+ cursorFloor = CheckpointNumber.ZERO;
384
+ } else {
385
+ const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
386
+ if (targetData === undefined) {
387
+ throw new Error(
388
+ `No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
389
+ );
136
390
  }
391
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
392
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
393
+ // would permanently skip a rebuilt same-number checkpoint.
394
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
395
+ }
396
+
397
+ const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
398
+
399
+ if (this.lastProcessedCheckpoint > cursorFloor) {
400
+ this.lastProcessedCheckpoint = cursorFloor;
401
+ }
402
+
403
+ if (affected.length === 0) {
404
+ return;
405
+ }
406
+ const l1Constants = await this.getL1Constants();
407
+ const affectedEpochs = Array.from(
408
+ new Set(affected.map(p => Number(getEpochAtSlot(p.slotNumber, l1Constants)))),
409
+ ).map(n => EpochNumber(n));
410
+ // The session manager cancels every affected session, which in turn calls
411
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
412
+ // publishing service is needed.
413
+ await this.sessionManager?.onPrune(affectedEpochs);
414
+ }
415
+
416
+ /**
417
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
418
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
419
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
420
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
421
+ * be read yet.
422
+ */
423
+ private async isEpochPastProofSubmissionWindow(
424
+ epochNumber: EpochNumber,
425
+ l1Constants: L1RollupConstants,
426
+ ): Promise<boolean> {
427
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
428
+ if (latestSlot === undefined) {
137
429
  return false;
138
430
  }
431
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
432
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
433
+ }
434
+
435
+ /**
436
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
437
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
438
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
439
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
440
+ * the last check or the archiver's slot can't be read.
441
+ */
442
+ private async checkEpochExpiry(): Promise<void> {
443
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
444
+ if (latestSlot === undefined) {
445
+ return;
446
+ }
447
+ const l1Constants = await this.getL1Constants();
448
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
449
+ const offset = l1Constants.proofSubmissionEpochs + 1;
450
+ if (latestEpoch < offset) {
451
+ return;
452
+ }
453
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
454
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
455
+ if (newlyExpiredUpTo < from) {
456
+ return;
457
+ }
458
+ for (let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)) {
459
+ await this.expireEpoch(e);
460
+ }
461
+ this.lastExpiredEpoch = newlyExpiredUpTo;
462
+ }
463
+
464
+ /**
465
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
466
+ * CheckpointProver in the store whose epoch is at or below it.
467
+ */
468
+ private async expireEpoch(epoch: EpochNumber): Promise<void> {
469
+ try {
470
+ const blocks = await this.l2BlockSource.getBlocks({ epoch, onlyCheckpointed: true });
471
+ if (blocks.length > 0) {
472
+ this.chonkCache.releaseForBlocks(blocks);
473
+ }
474
+ } catch (err) {
475
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
476
+ }
477
+ this.checkpointStore.reapExpired(epoch);
139
478
  }
140
479
 
480
+ // ---------------- public API ----------------
481
+
141
482
  /**
142
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
143
- * starts proving jobs for them.
483
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
144
484
  */
485
+ public async startProof(epochNumber: EpochNumber): Promise<string> {
486
+ if (!this.sessionManager) {
487
+ throw new Error('ProverNode not started');
488
+ }
489
+ return await this.sessionManager.startProof(epochNumber);
490
+ }
491
+
492
+ // ---------------- Service lifecycle ----------------
493
+
145
494
  async start() {
146
- this.epochsMonitor.start(this);
495
+ await this.checkpointStore.start();
496
+
147
497
  await this.publisherFactory.start();
148
- this.publisher = await this.publisherFactory.create();
498
+ this.publishingService = new ProofPublishingService({
499
+ publisherFactory: this.publisherFactory,
500
+ l2BlockSource: this.l2BlockSource,
501
+ dateProvider: this.dateProvider,
502
+ config: { skipSubmitProof: !!this.config.proverNodeDisableProofPublish },
503
+ bindings: this.log.getBindings(),
504
+ });
505
+ this.sessionManager = this.createSessionManager(this.publishingService);
506
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
507
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
508
+ // the previous epoch is proven on L1.
509
+ this.sessionManager.start();
510
+ // Now that the store + manager exist, arm the live-state observable gauges.
511
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
512
+
513
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
514
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
515
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
516
+ this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
517
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
518
+ tipsOnly: true,
519
+ });
520
+ this.blockStream.start();
521
+
522
+ // The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
523
+ // or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
524
+ // `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
525
+ this.expiryTicker = new RunningPromise(
526
+ () => this.checkEpochExpiry(),
527
+ this.log,
528
+ this.config.proverNodePollingIntervalMs,
529
+ );
530
+ this.expiryTicker.start();
531
+
149
532
  await this.rewardsMetrics.start();
150
533
  this.l1Metrics.start();
151
534
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
152
535
  }
153
536
 
154
- /**
155
- * Stops the prover node and all its dependencies.
156
- * Resources not owned by this node (shared with the parent aztec-node) are skipped.
157
- */
158
537
  async stop() {
159
538
  this.log.info('Stopping ProverNode');
160
- await this.epochsMonitor.stop();
161
- this.publisher?.interrupt();
162
- await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
539
+ this.jobMetrics.stopObservingState();
540
+ await this.blockStream?.stop();
541
+ await this.expiryTicker?.stop();
542
+ if (this.sessionManager) {
543
+ await this.sessionManager.stop();
544
+ }
545
+ if (this.publishingService) {
546
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
547
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
548
+ // the tx may still mine, but shutdown must not hang on it.
549
+ const publishingService = this.publishingService;
550
+ await executeTimeout(
551
+ () => publishingService.stop(),
552
+ PUBLISHING_SERVICE_STOP_TIMEOUT_MS,
553
+ 'prover-node publishing-service stop',
554
+ ).catch(err => this.log.warn(`Timed out stopping proof publishing service`, err));
555
+ }
556
+ await this.checkpointStore.stop();
557
+ this.chonkCache.stop();
163
558
  await this.prover.stop();
164
559
  await tryStop(this.publisherFactory);
165
560
  this.rewardsMetrics.stop();
@@ -168,275 +563,224 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
168
563
  this.log.info('Stopped ProverNode');
169
564
  }
170
565
 
171
- /** Returns world state status. */
172
- public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
173
- const { syncSummary } = await this.worldState.status();
174
- return syncSummary;
566
+ /**
567
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap the
568
+ * implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
569
+ * (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
570
+ * a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
571
+ */
572
+ protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
573
+ return new SessionManager({
574
+ checkpointStore: this.checkpointStore,
575
+ l2BlockSource: this.l2BlockSource,
576
+ proverFactory: this.prover,
577
+ proverId: this.prover.getProverId(),
578
+ publishingService,
579
+ metrics: this.jobMetrics,
580
+ dateProvider: this.dateProvider,
581
+ config: {
582
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
583
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
584
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
585
+ },
586
+ onSessionFailed: async session => {
587
+ await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
588
+ },
589
+ bindings: this.log.getBindings(),
590
+ });
175
591
  }
176
592
 
177
- /** Returns archiver status. */
178
- public getL2Tips() {
179
- return this.l2BlockSource.getL2Tips();
593
+ /**
594
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
595
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
596
+ * Applies to every session constructed after this call.
597
+ */
598
+ public setSessionHooks(hooks: EpochSessionHooks): void {
599
+ if (!this.sessionManager) {
600
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
601
+ }
602
+ this.sessionManager.setSessionHooks(hooks);
180
603
  }
181
604
 
182
605
  /**
183
- * Starts a proving process and returns immediately.
606
+ * Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
607
+ * CheckpointProver constructed after this call. For the e2e harness only.
184
608
  */
185
- public async startProof(epochNumber: EpochNumber) {
186
- await this.startProofIfNeeded(epochNumber, { skipEpochCheck: true });
609
+ public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
610
+ this.checkpointStore.setTestHooks(hooks);
187
611
  }
188
612
 
189
- private async startProofIfNeeded(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}): Promise<boolean> {
190
- if (this.startingProofEpochs.has(epochNumber)) {
191
- this.log.warn(`Not starting proof for ${epochNumber} since a proof is already being started for the epoch`, {
192
- epochNumber,
193
- });
194
- return false;
195
- }
196
-
197
- this.startingProofEpochs.add(epochNumber);
198
-
199
- try {
200
- const activeJobs = await this.activeJobsCoverEpoch(epochNumber);
201
- if (activeJobs.length > 0) {
202
- this.log.warn(`Not starting proof for ${epochNumber} since an active job already covers the epoch`, {
203
- epochNumber,
204
- activeJobs,
205
- });
206
- return true;
207
- }
208
-
209
- await this.startProofInternal(epochNumber, opts);
210
- return true;
211
- } finally {
212
- this.startingProofEpochs.delete(epochNumber);
613
+ /**
614
+ * Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
615
+ * session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
616
+ * genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
617
+ * store is configured or the checkpoint set is empty.
618
+ */
619
+ public async tryUploadEpochFailure(
620
+ id: string,
621
+ checkpoints: readonly CheckpointProver[],
622
+ ): Promise<string | undefined> {
623
+ if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
624
+ return undefined;
213
625
  }
626
+ const data = SessionManager.buildProvingData(checkpoints);
627
+ return await uploadEpochProofFailure(
628
+ this.config.proverNodeFailedEpochStore,
629
+ // The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
630
+ id,
631
+ data,
632
+ this.l2BlockSource as Archiver,
633
+ this.worldState,
634
+ assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
635
+ this.log,
636
+ );
214
637
  }
215
638
 
216
- private async startProofInternal(epochNumber: EpochNumber, opts: EpochProvingJobOptions = {}) {
217
- const job = await this.createProvingJob(epochNumber, opts);
218
- void this.runJob(job);
219
- }
220
-
221
- private async runJob(job: EpochProvingJob) {
222
- const epochNumber = job.getEpochNumber();
223
- const ctx = { id: job.getId(), epochNumber, state: undefined as EpochProvingJobState | undefined };
224
-
639
+ /**
640
+ * Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
641
+ * proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
642
+ * block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
643
+ * failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
644
+ * to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
645
+ */
646
+ public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
647
+ if (!this.config.proverNodeFailedEpochStore) {
648
+ return undefined;
649
+ }
225
650
  try {
226
- await job.run();
227
- const state = job.getState();
228
- ctx.state = state;
229
-
230
- if (state === 'reorg') {
231
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
232
- await this.createProvingJob(epochNumber);
233
- } else if (state === 'failed') {
234
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
235
- await this.tryUploadEpochFailure(job);
236
- } else {
237
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
651
+ // A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
652
+ // prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
653
+ // committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
654
+ // Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
655
+ // the snapshot (full world-state + archiver) is expensive to produce and store.
656
+ if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
657
+ this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
658
+ checkpointNumber: prover.checkpoint.number,
659
+ });
660
+ return undefined;
238
661
  }
239
- } catch (err) {
240
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
241
- } finally {
242
- this.jobs.delete(job.getId());
243
- }
244
- }
245
-
246
- protected async tryUploadEpochFailure(job: EpochProvingJob) {
247
- if (this.config.proverNodeFailedEpochStore) {
662
+ const data = SessionManager.buildProvingData([prover]);
248
663
  return await uploadEpochProofFailure(
249
664
  this.config.proverNodeFailedEpochStore,
250
- job.getId(),
251
- job.getProvingData(),
665
+ // The prover's content-addressed id; the epoch number is already in the upload path.
666
+ prover.id,
667
+ data,
252
668
  this.l2BlockSource as Archiver,
253
669
  this.worldState,
254
670
  assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
255
671
  this.log,
256
672
  );
673
+ } catch (err) {
674
+ this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
675
+ return undefined;
257
676
  }
258
677
  }
259
678
 
260
- /**
261
- * Returns the prover instance.
262
- */
263
- public getProver() {
264
- return this.prover;
265
- }
679
+ // ---------------- helpers ----------------
266
680
 
267
681
  /**
268
- * Returns an array of jobs being processed.
682
+ * True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
683
+ * block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
684
+ * so this returns false. Protected for direct unit-test access.
269
685
  */
270
- public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
271
- return Promise.resolve(this.getJobsInternal());
686
+ protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
687
+ const lastBlock = checkpoint.blocks.at(-1);
688
+ if (!lastBlock) {
689
+ return false;
690
+ }
691
+ const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
692
+ return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
272
693
  }
273
694
 
274
- private getJobsInternal(): { uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[] {
275
- return Array.from(this.jobs.entries()).map(([uuid, job]) => ({
276
- uuid,
277
- status: job.getState(),
278
- epochNumber: job.getEpochNumber(),
279
- }));
695
+ @memoize
696
+ private getL1Constants(): Promise<L1RollupConstants> {
697
+ return this.l2BlockSource.getL1Constants();
280
698
  }
281
699
 
282
- private async activeJobsCoverEpoch(epochNumber: EpochNumber): Promise<string[]> {
283
- const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: epochNumber });
284
- if (checkpoints.length === 0) {
285
- return [];
700
+ /**
701
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
702
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
703
+ */
704
+ protected async isEpochFullyProven(
705
+ epochNumber: EpochNumber,
706
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
707
+ ): Promise<boolean> {
708
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
709
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
710
+ return false;
286
711
  }
287
-
288
- const firstCheckpoint = checkpoints.at(0)!.checkpointNumber;
289
- const latestCheckpoint = checkpoints.at(-1)!.checkpointNumber;
290
-
291
- const jobs: string[] = [];
292
- for (const job of this.jobs.values()) {
293
- if (job.getEpochNumber() !== epochNumber) {
294
- continue;
295
- }
296
-
297
- const jobCheckpoints = job.getProvingData().checkpoints;
298
- const checkpointOverlap =
299
- jobCheckpoints.at(0)!.number <= firstCheckpoint && jobCheckpoints.at(-1)!.number >= latestCheckpoint;
300
-
301
- if (checkpointOverlap && !['failed', 'stopped', 'timed-out'].includes(job.getState())) {
302
- jobs.push(job.getId());
303
- }
712
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
713
+ if (!provenHeader) {
714
+ return false;
304
715
  }
305
-
306
- return jobs;
307
- }
308
-
309
- private checkMaximumPendingJobs() {
310
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
311
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
312
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
716
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
717
+ if (epochNumber < provenEpoch) {
718
+ return true;
313
719
  }
720
+ if (epochNumber > provenEpoch) {
721
+ return false;
722
+ }
723
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
314
724
  }
315
725
 
316
- @trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
317
- private async createProvingJob(epochNumber: EpochNumber, opts: { skipEpochCheck?: boolean } = {}) {
318
- this.checkMaximumPendingJobs();
319
-
320
- this.publisher = await this.publisherFactory.create();
321
-
322
- // Gather all data for this epoch
323
- const epochData = await this.gatherEpochData(epochNumber);
324
- const fromCheckpoint = epochData.checkpoints[0].number;
325
- const toCheckpoint = epochData.checkpoints.at(-1)!.number;
326
- const fromBlock = epochData.checkpoints[0].blocks[0].number;
327
- const lastBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!;
328
- const toBlock = lastBlock.number;
329
- this.log.verbose(
330
- `Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
331
- );
332
-
333
- // Fast forward world state to right before the target block and get a fork
334
- const lastBlockHash = await lastBlock.header.hash();
335
- await this.worldState.syncImmediate(toBlock, lastBlockHash);
336
-
337
- // Create a processor factory
338
- const publicProcessorFactory = new PublicProcessorFactory(
339
- this.contractDataSource,
340
- this.dateProvider,
341
- this.telemetryClient,
342
- this.log.getBindings(),
343
- );
344
-
345
- // Set deadline for this job to run. It will abort if it takes too long.
346
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
347
- const deadline = new Date(Number(deadlineTs) * 1000);
348
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
349
- this.jobs.set(job.getId(), job);
350
- return job;
351
- }
352
-
353
- @memoize
354
- private getL1Constants() {
355
- return this.l2BlockSource.getL1Constants();
356
- }
357
-
358
- @trackSpan('ProverNode.gatherEpochData', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
359
- private async gatherEpochData(epochNumber: EpochNumber): Promise<EpochProvingJobData> {
360
- const publishedCheckpoints = await this.l2BlockSource.getCheckpoints({ epoch: epochNumber });
361
- if (publishedCheckpoints.length === 0) {
362
- throw new EmptyEpochError(epochNumber);
726
+ /** Protected for direct unit-test access. */
727
+ protected async isProvenBlockLastOfItsEpoch(
728
+ provenBlockNumber: BlockNumber,
729
+ provenEpoch: EpochNumber,
730
+ l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
731
+ ): Promise<boolean> {
732
+ const nextHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber + 1) }))?.header;
733
+ if (nextHeader) {
734
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
363
735
  }
364
- const checkpoints = publishedCheckpoints.map(p => p.checkpoint);
365
- const attestations = publishedCheckpoints.at(-1)?.attestations ?? [];
366
- const txArray = await this.gatherTxs(epochNumber, checkpoints);
367
- const txs = new Map<string, Tx>(txArray.map(tx => [tx.getTxHash().toString(), tx]));
368
- const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
369
- const [firstBlock] = checkpoints[0].blocks;
370
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
371
-
372
- return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
736
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
373
737
  }
374
738
 
375
- private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
376
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
377
- const txProvider = this.p2pClient.getTxProvider();
378
- const blocks = checkpoints.flatMap(checkpoint => checkpoint.blocks);
379
- const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
380
- const txs = txsByBlock.map(({ txs }) => txs).flat();
381
- const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
382
-
383
- if (missingTxs.length === 0) {
384
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
385
- return txs;
739
+ /**
740
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
741
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
742
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
743
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
744
+ */
745
+ protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
746
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
747
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
748
+ return { lastFullyProvenEpoch: undefined };
386
749
  }
387
-
388
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map(hash => hash.toString()).join(', ')}`);
750
+ const l1Constants = await this.getL1Constants();
751
+ const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
752
+ if (!provenHeader) {
753
+ return { lastFullyProvenEpoch: undefined };
754
+ }
755
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
756
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
757
+ return { lastFullyProvenEpoch: provenEpoch };
758
+ }
759
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
760
+ return { lastFullyProvenEpoch };
389
761
  }
390
762
 
391
- private async gatherMessages(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
392
- const messages = await Promise.all(checkpoints.map(c => this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
393
- const messageCount = sum(messages.map(m => m.length));
394
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, { epochNumber });
395
- const messagesByCheckpoint: Record<CheckpointNumber, Fr[]> = {};
396
- for (let i = 0; i < checkpoints.length; i++) {
397
- messagesByCheckpoint[checkpoints[i].number] = messages[i];
763
+ /**
764
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
765
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
766
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
767
+ */
768
+ protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
769
+ if (lastFullyProvenEpoch === undefined) {
770
+ return CheckpointNumber.ZERO;
398
771
  }
399
- return messagesByCheckpoint;
772
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
773
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
400
774
  }
401
775
 
402
- private async gatherPreviousBlockHeader(epochNumber: EpochNumber, previousBlockNumber: number) {
776
+ private async gatherPreviousBlockHeader(previousBlockNumber: number) {
403
777
  const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
404
778
  if (!data?.header) {
405
- throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
779
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
406
780
  }
407
-
408
- this.log.verbose(`Gathered previous block header ${data.header.getBlockNumber()} for epoch ${epochNumber}`);
409
781
  return data.header;
410
782
  }
411
783
 
412
- /** Extracted for testing purposes. */
413
- protected doCreateEpochProvingJob(
414
- data: EpochProvingJobData,
415
- deadline: Date | undefined,
416
- publicProcessorFactory: PublicProcessorFactory,
417
- publisher: ProverNodePublisher,
418
- opts: { skipEpochCheck?: boolean } = {},
419
- ) {
420
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
421
- return new EpochProvingJob(
422
- data,
423
- this.worldState,
424
- this.prover.createEpochProver(),
425
- publicProcessorFactory,
426
- publisher,
427
- this.l2BlockSource,
428
- this.jobMetrics,
429
- deadline,
430
- { parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
431
- this.log.getBindings(),
432
- );
433
- }
434
-
435
- /** Extracted for testing purposes. */
436
- protected async triggerMonitors() {
437
- await this.epochsMonitor.work();
438
- }
439
-
440
784
  private validateConfig() {
441
785
  if (
442
786
  this.config.proverNodeFailedEpochStore &&
@@ -453,9 +797,5 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
453
797
  }
454
798
  }
455
799
 
456
- class EmptyEpochError extends Error {
457
- constructor(epochNumber: EpochNumber) {
458
- super(`No blocks found for epoch ${epochNumber}`);
459
- this.name = 'EmptyEpochError';
460
- }
461
- }
800
+ // Re-export so handlers can compare states externally.
801
+ export { EpochProvingJobTerminalState };