@aztec/prover-node 5.0.0-rc.1 → 5.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -92,7 +92,7 @@ export type EpochSessionDeps = {
92
92
  *
93
93
  * Lifecycle (happy path):
94
94
  *
95
- * initialized → awaiting-checkpoints → completed
95
+ * initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
96
96
  *
97
97
  * Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
98
98
  * `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
@@ -320,6 +320,12 @@ export class EpochSession implements Traceable {
320
320
  0,
321
321
  );
322
322
 
323
+ // Reflect the publish phase. Guard against a terminal state set concurrently by cancel() — the
324
+ // post-submit isTerminal() check below relies on cancel still winning.
325
+ if (!this.isTerminal()) {
326
+ this.state = 'publishing-proof';
327
+ }
328
+
323
329
  const outcome = await this.deps.publishingService.submit({
324
330
  id: this.uuid,
325
331
  epoch: this.spec.epochNumber,
@@ -333,6 +339,7 @@ export class EpochSession implements Traceable {
333
339
  proof: proof.proof,
334
340
  batchedBlobInputs: proof.batchedBlobInputs,
335
341
  attestations,
342
+ headers: this.checkpoints.map(c => c.checkpoint.header),
336
343
  });
337
344
 
338
345
  if (this.isTerminal()) {
@@ -347,7 +354,7 @@ export class EpochSession implements Traceable {
347
354
  { uuid: this.uuid, ...this.spec },
348
355
  );
349
356
  this.state = 'completed';
350
- this.deps.metrics.recordProvingJob(timer.ms(), timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
357
+ this.deps.metrics.recordProvingJob(timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
351
358
  return;
352
359
  case 'superseded':
353
360
  this.log.info(`EpochSession ${this.uuid} superseded by a longer candidate`, {
@@ -410,15 +417,20 @@ export class EpochSession implements Traceable {
410
417
  }
411
418
  }
412
419
 
413
- private toTopTreeHooks(): TopTreeJobHooks | undefined {
420
+ private toTopTreeHooks(): TopTreeJobHooks {
414
421
  const hooks = this.deps.hooks;
415
- if (!hooks?.beforeTopTreeProve && !hooks?.afterTopTreeProve && !hooks?.topTreeProveOverride) {
416
- return undefined;
417
- }
418
422
  return {
419
- beforeProve: hooks.beforeTopTreeProve,
420
- afterProve: hooks.afterTopTreeProve,
421
- proveOverride: hooks.topTreeProveOverride,
423
+ // `beforeProve` fires once the sub-tree (checkpoint block) proofs are ready and the root prove is
424
+ // about to start — the boundary between `awaiting-checkpoints` and proving the top tree. Don't
425
+ // clobber a terminal state set concurrently by cancel().
426
+ beforeProve: async () => {
427
+ if (!this.isTerminal()) {
428
+ this.state = 'awaiting-root';
429
+ }
430
+ await hooks?.beforeTopTreeProve?.();
431
+ },
432
+ afterProve: hooks?.afterTopTreeProve,
433
+ proveOverride: hooks?.topTreeProveOverride,
422
434
  };
423
435
  }
424
436
  }
package/src/metrics.ts CHANGED
@@ -22,7 +22,6 @@ import type { CheckpointStore } from './checkpoint-store.js';
22
22
  import type { SessionManager } from './session-manager.js';
23
23
 
24
24
  export class ProverNodeJobMetrics {
25
- proverEpochExecutionDuration: Histogram;
26
25
  provingJobDuration: Histogram;
27
26
  provingJobCheckpoints: Gauge;
28
27
  provingJobBlocks: Gauge;
@@ -31,6 +30,9 @@ export class ProverNodeJobMetrics {
31
30
  private blobProcessingDuration: Gauge;
32
31
  private blockProcessingDuration: Histogram;
33
32
  private checkpointProcessingDuration: Histogram;
33
+ private checkpointProvingDuration: Histogram;
34
+ private checkpointBlocks: Histogram;
35
+ private checkpointTransactions: Histogram;
34
36
 
35
37
  /** Observable gauges for live state. Registered via `observeState(...)` once the
36
38
  * CheckpointStore and SessionManager are available. */
@@ -44,7 +46,6 @@ export class ProverNodeJobMetrics {
44
46
  public readonly tracer: Tracer,
45
47
  private logger = createLogger('prover-node:publisher:metrics'),
46
48
  ) {
47
- this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
48
49
  this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
49
50
  this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
50
51
  this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
@@ -53,16 +54,12 @@ export class ProverNodeJobMetrics {
53
54
  this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
54
55
  this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
55
56
  this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
57
+ this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
58
+ this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
59
+ this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
56
60
  }
57
61
 
58
- public recordProvingJob(
59
- executionTimeMs: number,
60
- totalTimeMs: number,
61
- numCheckpoints: number,
62
- numBlocks: number,
63
- numTxs: number,
64
- ) {
65
- this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
62
+ public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
66
63
  this.provingJobDuration.record(totalTimeMs / 1000);
67
64
  this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
68
65
  this.provingJobBlocks.record(Math.floor(numBlocks));
@@ -77,8 +74,14 @@ export class ProverNodeJobMetrics {
77
74
  this.blockProcessingDuration.record(Math.ceil(durationMs));
78
75
  }
79
76
 
80
- public recordCheckpointProcessing(durationMs: number) {
77
+ public recordCheckpointProcessing(durationMs: number, numBlocks: number, numTxs: number) {
81
78
  this.checkpointProcessingDuration.record(Math.ceil(durationMs));
79
+ this.checkpointBlocks.record(Math.floor(numBlocks));
80
+ this.checkpointTransactions.record(Math.floor(numTxs));
81
+ }
82
+
83
+ public recordCheckpointProving(durationMs: number) {
84
+ this.checkpointProvingDuration.record(Math.ceil(durationMs));
82
85
  }
83
86
 
84
87
  /**
@@ -7,7 +7,7 @@ import { SerialQueue } from '@aztec/foundation/queue';
7
7
  import type { DateProvider } from '@aztec/foundation/timer';
8
8
  import type { L2BlockSource } from '@aztec/stdlib/block';
9
9
  import type { Proof } from '@aztec/stdlib/proofs';
10
- import type { RootRollupPublicInputs } from '@aztec/stdlib/rollup';
10
+ import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
11
11
 
12
12
  import type { ProverNodePublisher } from './prover-node-publisher.js';
13
13
  import type { ProverPublisherFactory } from './prover-publisher-factory.js';
@@ -45,6 +45,8 @@ export type PublishCandidate = {
45
45
  proof: Proof;
46
46
  batchedBlobInputs: BatchedBlob;
47
47
  attestations: ViemCommitteeAttestation[];
48
+ /** Committee-attested checkpoint headers for the range, supplying the L1-verified fee recipient/value. */
49
+ headers: CheckpointHeader[];
48
50
  };
49
51
 
50
52
  /** Terminal outcome for a candidate. The promise from `submit()` resolves with one of these. */
@@ -333,6 +335,7 @@ export class ProofPublishingService {
333
335
  proof: candidate.proof,
334
336
  batchedBlobInputs: candidate.batchedBlobInputs,
335
337
  attestations: candidate.attestations,
338
+ headers: candidate.headers,
336
339
  // Stop the L1 tx retrying past the candidate's submission-window deadline.
337
340
  deadline: candidate.deadline,
338
341
  };
@@ -2,19 +2,17 @@ import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
2
2
  import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
3
3
  import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
4
4
  import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
5
- import { makeTuple } from '@aztec/foundation/array';
6
5
  import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
7
6
  import { areArraysEqual } from '@aztec/foundation/collection';
8
7
  import { Fr } from '@aztec/foundation/curves/bn254';
9
8
  import { EthAddress } from '@aztec/foundation/eth-address';
10
9
  import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
11
- import type { Tuple } from '@aztec/foundation/serialize';
12
10
  import { Timer } from '@aztec/foundation/timer';
13
11
  import { RollupAbi } from '@aztec/l1-artifacts';
14
12
  import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
15
13
  import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
16
14
  import type { Proof } from '@aztec/stdlib/proofs';
17
- import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
15
+ import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
18
16
  import type { L1PublishProofStats } from '@aztec/stdlib/stats';
19
17
  import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
20
18
 
@@ -31,7 +29,7 @@ export type L1SubmitEpochProofArgs = {
31
29
  endTimestamp: Fr;
32
30
  outHash: Fr;
33
31
  proverId: Fr;
34
- fees: Tuple<FeeRecipient, typeof MAX_CHECKPOINTS_PER_EPOCH>;
32
+ headers: CheckpointHeader[];
35
33
  proof: Proof;
36
34
  };
37
35
 
@@ -78,6 +76,7 @@ export class ProverNodePublisher {
78
76
  proof: Proof;
79
77
  batchedBlobInputs: BatchedBlob;
80
78
  attestations: ViemCommitteeAttestation[];
79
+ headers: CheckpointHeader[];
81
80
  /** Wall-clock deadline (proof-submission window end) past which the L1 tx should stop retrying. */
82
81
  deadline?: Date;
83
82
  }): Promise<boolean> {
@@ -134,6 +133,7 @@ export class ProverNodePublisher {
134
133
  proof: Proof;
135
134
  batchedBlobInputs: BatchedBlob;
136
135
  attestations: ViemCommitteeAttestation[];
136
+ headers: CheckpointHeader[];
137
137
  }) {
138
138
  const { fromCheckpoint, toCheckpoint, publicInputs, batchedBlobInputs } = args;
139
139
 
@@ -207,6 +207,7 @@ export class ProverNodePublisher {
207
207
  proof: Proof;
208
208
  batchedBlobInputs: BatchedBlob;
209
209
  attestations: ViemCommitteeAttestation[];
210
+ headers: CheckpointHeader[];
210
211
  }): Promise<void> {
211
212
  const { epochNumber, fromCheckpoint, toCheckpoint } = args;
212
213
 
@@ -254,6 +255,7 @@ export class ProverNodePublisher {
254
255
  proof: Proof;
255
256
  batchedBlobInputs: BatchedBlob;
256
257
  attestations: ViemCommitteeAttestation[];
258
+ headers: CheckpointHeader[];
257
259
  }): Hex {
258
260
  return encodeFunctionData({
259
261
  abi: RollupAbi,
@@ -270,6 +272,7 @@ export class ProverNodePublisher {
270
272
  proof: Proof;
271
273
  batchedBlobInputs: BatchedBlob;
272
274
  attestations: ViemCommitteeAttestation[];
275
+ headers: CheckpointHeader[];
273
276
  }): Promise<TransactionReceipt | undefined> {
274
277
  const txArgs = [this.getSubmitEpochProofArgs(args)] as const;
275
278
 
@@ -316,6 +319,7 @@ export class ProverNodePublisher {
316
319
  publicInputs: RootRollupPublicInputs;
317
320
  batchedBlobInputs: BatchedBlob;
318
321
  attestations: ViemCommitteeAttestation[];
322
+ headers: CheckpointHeader[];
319
323
  }) {
320
324
  // Returns arguments for EpochProofLib.sol -> getEpochProofPublicInputs()
321
325
  return [
@@ -327,11 +331,7 @@ export class ProverNodePublisher {
327
331
  outHash: args.publicInputs.outHash.toString(),
328
332
  proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
329
333
  } /*_args*/,
330
- makeTuple(MAX_CHECKPOINTS_PER_EPOCH * 2, i =>
331
- i % 2 === 0
332
- ? args.publicInputs.fees[i / 2].recipient.toField().toString()
333
- : args.publicInputs.fees[(i - 1) / 2].value.toString(),
334
- ) /*_fees*/,
334
+ args.headers.map(header => header.toViem()) /*_headers*/,
335
335
  getEthBlobEvaluationInputs(args.batchedBlobInputs) /*_blobPublicInputs*/,
336
336
  ] as const;
337
337
  }
@@ -343,6 +343,7 @@ export class ProverNodePublisher {
343
343
  proof: Proof;
344
344
  batchedBlobInputs: BatchedBlob;
345
345
  attestations: ViemCommitteeAttestation[];
346
+ headers: CheckpointHeader[];
346
347
  }) {
347
348
  // Returns arguments for EpochProofLib.sol -> submitEpochRootProof()
348
349
  const proofHex: Hex = `0x${args.proof.withoutPublicInputs().toString('hex')}`;
@@ -351,7 +352,7 @@ export class ProverNodePublisher {
351
352
  start: argsArray[0],
352
353
  end: argsArray[1],
353
354
  args: argsArray[2],
354
- fees: argsArray[3],
355
+ headers: argsArray[3],
355
356
  attestations: CommitteeAttestationsAndSigners.packAttestations(
356
357
  args.attestations.map(a => CommitteeAttestation.fromViem(a)),
357
358
  ),
@@ -5,12 +5,14 @@ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/br
5
5
  import { assertRequired, compact, pick } from '@aztec/foundation/collection';
6
6
  import { memoize } from '@aztec/foundation/decorators';
7
7
  import { createLogger } from '@aztec/foundation/log';
8
+ import { RunningPromise } from '@aztec/foundation/running-promise';
8
9
  import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
9
10
  import type { EpochProverFactory } from '@aztec/prover-client';
10
11
  import { getLastSiblingPath } from '@aztec/prover-client/helpers';
11
12
  import { ChonkCache } from '@aztec/prover-client/orchestrator';
12
13
  import { PublicProcessorFactory } from '@aztec/simulator/server';
13
14
  import {
15
+ type L2BlockId,
14
16
  type L2BlockSource,
15
17
  L2BlockStream,
16
18
  type L2BlockStreamEvent,
@@ -28,7 +30,6 @@ import {
28
30
  type ITxProvider,
29
31
  type ProverNodeApi,
30
32
  type Service,
31
- type WorldStateSyncStatus,
32
33
  type WorldStateSynchronizer,
33
34
  tryStop,
34
35
  } from '@aztec/stdlib/interfaces/server';
@@ -95,6 +96,17 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
95
96
  */
96
97
  protected lastExpiredEpoch: EpochNumber | undefined;
97
98
 
99
+ /**
100
+ * Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
101
+ * The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
102
+ * checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
103
+ * than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
104
+ */
105
+ protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
106
+
107
+ /** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
108
+ private expiryTicker: RunningPromise | undefined;
109
+
98
110
  public readonly tracer: Tracer;
99
111
 
100
112
  protected publishingService: ProofPublishingService | undefined;
@@ -198,17 +210,6 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
198
210
  return this.sessionManager;
199
211
  }
200
212
 
201
- /** Returns world state status. */
202
- public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
203
- const { syncSummary } = await this.worldState.status();
204
- return syncSummary;
205
- }
206
-
207
- /** Returns archiver status. */
208
- public getL2Tips() {
209
- return this.l2BlockSource.getL2Tips();
210
- }
211
-
212
213
  /** Returns the underlying prover instance. */
213
214
  public getProver() {
214
215
  return this.prover;
@@ -219,17 +220,25 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
219
220
  public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
220
221
  switch (event.type) {
221
222
  case 'chain-checkpointed':
222
- await this.handleCheckpointEvent(event.checkpoint);
223
+ await this.processCheckpointJump(event.checkpoint.number);
223
224
  break;
224
225
  case 'chain-pruned':
225
- await this.handlePruneEvent(event.checkpointed.checkpoint);
226
+ await this.handlePruneEvent(event.block);
226
227
  break;
227
228
  case 'chain-proven':
228
229
  this.publishingService?.onChainProven(BlockNumber(event.block.number));
229
230
  break;
231
+ // The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
232
+ // tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
233
+ // `chain-finalized` carries nothing the prover-node acts on.
234
+ case 'chain-proposed':
230
235
  case 'chain-finalized':
231
236
  case 'blocks-added':
232
237
  break;
238
+ default: {
239
+ const _: never = event;
240
+ break;
241
+ }
233
242
  }
234
243
  // Expiry is driven by the archiver's latest synced L2 slot
235
244
  await this.checkEpochExpiry();
@@ -239,34 +248,92 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
239
248
  await this.tipsStore.handleBlockStreamEvent(event);
240
249
  }
241
250
 
242
- /** Register a new checkpoint with the store and notify the session manager. */
243
- private async handleCheckpointEvent(publishedCheckpoint: PublishedCheckpoint) {
244
- const checkpoint = publishedCheckpoint.checkpoint;
245
- const slotNumber = checkpoint.header.slotNumber;
246
- const l1Constants = await this.getL1Constants();
247
- const epochNumber = getEpochAtSlot(slotNumber, l1Constants);
248
-
249
- if (await this.isEpochFullyProven(epochNumber, l1Constants)) {
250
- this.log.debug(`Skipping checkpoint ${checkpoint.number} for already-proven epoch ${epochNumber}`);
251
+ /**
252
+ * Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
253
+ * each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
254
+ * `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
255
+ * catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
256
+ * `getCheckpoints` fetch only for checkpoints in provable epochs.
257
+ *
258
+ * The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
259
+ * fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
260
+ * behind so the next pass retries from the first checkpoint that did not complete.
261
+ */
262
+ private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
263
+ if (targetCheckpoint <= this.lastProcessedCheckpoint) {
251
264
  return;
252
265
  }
266
+ const l1Constants = await this.getL1Constants();
253
267
 
254
- if (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants)) {
255
- this.log.debug(
256
- `Skipping checkpoint ${checkpoint.number} for epoch ${epochNumber} past its proof-submission window`,
257
- );
258
- return;
268
+ // Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
269
+ // When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
270
+ // load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
271
+ // its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
272
+ const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
273
+ let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
274
+ if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
275
+ const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
276
+ this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
277
+ from,
278
+ cappedFrom,
279
+ targetCheckpoint,
280
+ maxCheckpoints,
281
+ });
282
+ // Advance the cursor past the skipped checkpoints so they are never retried.
283
+ this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
284
+ from = cappedFrom;
259
285
  }
286
+ const limit = Number(targetCheckpoint - from) + 1;
287
+ const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
288
+
289
+ // Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
290
+ // only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
291
+ // never drop an individual checkpoint inside an epoch we will prove.
292
+ const epochSkippable = new Map<EpochNumber, boolean>();
293
+ for (const metadata of metadatas) {
294
+ const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
295
+ let skippable = epochSkippable.get(epochNumber);
296
+ if (skippable === undefined) {
297
+ skippable =
298
+ (await this.isEpochFullyProven(epochNumber, l1Constants)) ||
299
+ (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
300
+ epochSkippable.set(epochNumber, skippable);
301
+ }
302
+ if (skippable) {
303
+ this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
304
+ } else {
305
+ await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
306
+ }
307
+ // Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
308
+ // throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
309
+ this.lastProcessedCheckpoint = metadata.checkpointNumber;
310
+ }
311
+ }
260
312
 
313
+ /** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
314
+ private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
315
+ const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
316
+ if (!published) {
317
+ throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
318
+ }
319
+ const checkpoint = published.checkpoint;
261
320
  this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
262
321
  checkpointNumber: checkpoint.number,
263
322
  epochNumber,
264
- slotNumber,
323
+ slotNumber: checkpoint.header.slotNumber,
265
324
  });
266
325
 
267
- const registerData = await this.collectRegisterData(checkpoint, publishedCheckpoint.attestations);
326
+ const registerData = await this.collectRegisterData(checkpoint, published.attestations);
268
327
  await this.checkpointStore.addOrUpdate(checkpoint, registerData);
269
328
  await this.sessionManager?.onCheckpointAdded(epochNumber);
329
+
330
+ // Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
331
+ // range then prunes at the true divergence instead of the nearest sparse tip anchor.
332
+ await this.tipsStore.recordBlockHashes(
333
+ await Promise.all(
334
+ checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
335
+ ),
336
+ );
270
337
  }
271
338
 
272
339
  /**
@@ -295,10 +362,41 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
295
362
  };
296
363
  }
297
364
 
298
- /** Mark every prover above the prune threshold as pruned and notify the session manager. */
299
- private async handlePruneEvent(prunedCheckpoint: { number: CheckpointNumber; hash: string }) {
300
- this.log.warn(`Chain pruned to checkpoint ${prunedCheckpoint.number}`, { prunedCheckpoint });
301
- const affected = this.checkpointStore.markPrunedAfter(prunedCheckpoint.number);
365
+ /**
366
+ * Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
367
+ * checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
368
+ * rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
369
+ * orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
370
+ * fails and the prune is retried next iteration.
371
+ */
372
+ private async handlePruneEvent(prunedToBlock: L2BlockId) {
373
+ this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
374
+
375
+ // Resolve the cursor floor BEFORE marking provers: markPrunedAboveBlock returns only newly-marked provers, so a
376
+ // throw after marking would leave a retry pass with nothing to act on. Resolving first means a throw leaves
377
+ // everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
378
+ let cursorFloor: CheckpointNumber;
379
+ if (prunedToBlock.number === 0) {
380
+ cursorFloor = CheckpointNumber.ZERO;
381
+ } else {
382
+ const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
383
+ if (targetData === undefined) {
384
+ throw new Error(
385
+ `No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
386
+ );
387
+ }
388
+ // Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
389
+ // reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
390
+ // would permanently skip a rebuilt same-number checkpoint.
391
+ cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
392
+ }
393
+
394
+ const affected = this.checkpointStore.markPrunedAboveBlock(prunedToBlock.number);
395
+
396
+ if (this.lastProcessedCheckpoint > cursorFloor) {
397
+ this.lastProcessedCheckpoint = cursorFloor;
398
+ }
399
+
302
400
  if (affected.length === 0) {
303
401
  return;
304
402
  }
@@ -409,14 +507,24 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
409
507
  // Now that the store + manager exist, arm the live-state observable gauges.
410
508
  this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
411
509
 
412
- const { startingBlock, lastFullyProvenEpoch } = await this.computeStartupState();
510
+ const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
413
511
  this.lastExpiredEpoch = lastFullyProvenEpoch;
512
+ this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
414
513
  this.blockStream = new L2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
415
514
  pollIntervalMS: this.config.proverNodePollingIntervalMs,
416
- startingBlock,
515
+ tipsOnly: true,
417
516
  });
418
517
  this.blockStream.start();
419
518
 
519
+ // With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
520
+ // from a periodic tick so epochs still expire during idle/no-event periods.
521
+ this.expiryTicker = new RunningPromise(
522
+ () => this.checkEpochExpiry(),
523
+ this.log,
524
+ this.config.proverNodePollingIntervalMs,
525
+ );
526
+ this.expiryTicker.start();
527
+
420
528
  await this.rewardsMetrics.start();
421
529
  this.l1Metrics.start();
422
530
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
@@ -426,6 +534,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
426
534
  this.log.info('Stopping ProverNode');
427
535
  this.jobMetrics.stopObservingState();
428
536
  await this.blockStream?.stop();
537
+ await this.expiryTicker?.stop();
429
538
  if (this.sessionManager) {
430
539
  await this.sessionManager.stop();
431
540
  }
@@ -552,38 +661,40 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
552
661
  }
553
662
 
554
663
  /**
555
- * Resolves the L2BlockStream's starting block and the last fully-proven epoch in one
556
- * pass. The starting block is the first block of the next unproven epoch (or the start
557
- * of the partially-proven epoch if the proven tip falls mid-epoch). The fully-proven
558
- * epoch is `provenEpoch` when the proven tip is the last block of its epoch, otherwise
559
- * `provenEpoch - 1`, or `undefined` if no block is proven yet.
664
+ * Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
665
+ * `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
666
+ * proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
667
+ * proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
560
668
  */
561
- protected async computeStartupState(): Promise<{
562
- startingBlock: BlockNumber;
563
- lastFullyProvenEpoch: EpochNumber | undefined;
564
- }> {
669
+ protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
565
670
  const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
566
671
  if (!provenBlockNumber || provenBlockNumber <= 0) {
567
- return { startingBlock: BlockNumber(1), lastFullyProvenEpoch: undefined };
672
+ return { lastFullyProvenEpoch: undefined };
568
673
  }
569
674
  const l1Constants = await this.getL1Constants();
570
675
  const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
571
676
  if (!provenHeader) {
572
- return { startingBlock: BlockNumber(provenBlockNumber + 1), lastFullyProvenEpoch: undefined };
677
+ return { lastFullyProvenEpoch: undefined };
573
678
  }
574
679
  const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
575
680
  if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
576
- return { startingBlock: BlockNumber(provenBlockNumber + 1), lastFullyProvenEpoch: provenEpoch };
681
+ return { lastFullyProvenEpoch: provenEpoch };
577
682
  }
578
- const epochCheckpoints = await this.l2BlockSource.getCheckpointsData({ epoch: provenEpoch });
579
- const firstBlockOfEpoch =
580
- epochCheckpoints.length > 0 ? epochCheckpoints[0].startBlock : BlockNumber(provenBlockNumber);
581
- this.log.info(
582
- `Starting L2BlockStream at block ${firstBlockOfEpoch} (start of partially-proven epoch ${provenEpoch})`,
583
- { provenBlockNumber, provenEpoch, firstBlockOfEpoch },
584
- );
585
683
  const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
586
- return { startingBlock: firstBlockOfEpoch, lastFullyProvenEpoch };
684
+ return { lastFullyProvenEpoch };
685
+ }
686
+
687
+ /**
688
+ * Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
689
+ * from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
690
+ * partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
691
+ */
692
+ protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
693
+ if (lastFullyProvenEpoch === undefined) {
694
+ return CheckpointNumber.ZERO;
695
+ }
696
+ const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
697
+ return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
587
698
  }
588
699
 
589
700
  private async gatherPreviousBlockHeader(previousBlockNumber: number) {