@aztec/prover-node 0.0.1-commit.c31f2472 → 0.0.1-commit.c52d6e7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +572 -0
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +14 -4
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +244 -24
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +6 -5
- package/dest/checkpoint-store.d.ts +95 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +178 -0
- package/dest/config.d.ts +7 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +24 -20
- package/dest/factory.d.ts +22 -13
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +41 -65
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +165 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +405 -0
- package/dest/job/epoch-session.d.ts +160 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/{epoch-proving-job.js → epoch-session.js} +301 -298
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +40 -3
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +101 -4
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +11 -9
- package/dest/proof-publishing-service.d.ts +161 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +335 -0
- package/dest/prover-node-publisher.d.ts +25 -15
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +202 -63
- package/dest/prover-node.d.ts +146 -69
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +545 -221
- package/dest/prover-publisher-factory.d.ts +6 -4
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +4 -3
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +492 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +24 -22
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +190 -31
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +5 -3
- package/src/checkpoint-store.ts +212 -0
- package/src/config.ts +35 -32
- package/src/factory.ts +73 -111
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +538 -0
- package/src/job/epoch-session.ts +462 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +123 -10
- package/src/monitors/epoch-monitor.ts +5 -6
- package/src/proof-publishing-service.ts +427 -0
- package/src/prover-node-publisher.ts +237 -79
- package/src/prover-node.ts +631 -248
- package/src/prover-publisher-factory.ts +8 -5
- package/src/session-manager.ts +592 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -63
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/src/job/epoch-proving-job.ts +0 -435
package/src/prover-node.ts
CHANGED
|
@@ -1,89 +1,137 @@
|
|
|
1
1
|
import type { Archiver } from '@aztec/archiver';
|
|
2
2
|
import type { RollupContract } from '@aztec/ethereum/contracts';
|
|
3
|
+
import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
|
|
3
4
|
import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
4
|
-
import { assertRequired, compact, pick
|
|
5
|
-
import type { Fr } from '@aztec/foundation/curves/bn254';
|
|
5
|
+
import { assertRequired, compact, pick } from '@aztec/foundation/collection';
|
|
6
6
|
import { memoize } from '@aztec/foundation/decorators';
|
|
7
7
|
import { createLogger } from '@aztec/foundation/log';
|
|
8
|
-
import {
|
|
9
|
-
import
|
|
10
|
-
import type {
|
|
11
|
-
import {
|
|
12
|
-
import
|
|
13
|
-
import type
|
|
8
|
+
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
9
|
+
import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
|
|
10
|
+
import type { EpochProverFactory } from '@aztec/prover-client';
|
|
11
|
+
import { getLastSiblingPath } from '@aztec/prover-client/helpers';
|
|
12
|
+
import { ChonkCache } from '@aztec/prover-client/orchestrator';
|
|
13
|
+
import { type AvmSimulator, PublicProcessorFactory } from '@aztec/simulator/server';
|
|
14
|
+
import {
|
|
15
|
+
EventDrivenL2BlockStream,
|
|
16
|
+
type L2BlockId,
|
|
17
|
+
type L2BlockSource,
|
|
18
|
+
type L2BlockStreamEvent,
|
|
19
|
+
type L2BlockStreamEventHandler,
|
|
20
|
+
L2TipsMemoryStore,
|
|
21
|
+
} from '@aztec/stdlib/block';
|
|
22
|
+
import type { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
14
23
|
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
15
24
|
import type { ContractDataSource } from '@aztec/stdlib/contract';
|
|
16
|
-
import {
|
|
25
|
+
import { type L1RollupConstants, getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
|
|
17
26
|
import {
|
|
18
27
|
type EpochProverManager,
|
|
28
|
+
type EpochProvingJobState,
|
|
19
29
|
EpochProvingJobTerminalState,
|
|
30
|
+
type ITxProvider,
|
|
20
31
|
type ProverNodeApi,
|
|
21
32
|
type Service,
|
|
22
|
-
type WorldStateSyncStatus,
|
|
23
33
|
type WorldStateSynchronizer,
|
|
24
34
|
tryStop,
|
|
25
35
|
} from '@aztec/stdlib/interfaces/server';
|
|
36
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
26
37
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
27
|
-
import
|
|
28
|
-
import type { Tx } from '@aztec/stdlib/tx';
|
|
38
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
29
39
|
import {
|
|
30
|
-
Attributes,
|
|
31
40
|
L1Metrics,
|
|
32
41
|
type TelemetryClient,
|
|
33
42
|
type Traceable,
|
|
34
43
|
type Tracer,
|
|
35
44
|
getTelemetryClient,
|
|
36
|
-
trackSpan,
|
|
37
45
|
} from '@aztec/telemetry-client';
|
|
38
46
|
|
|
39
47
|
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
48
|
+
import { CheckpointStore, type RegisterCheckpointData } from './checkpoint-store.js';
|
|
40
49
|
import type { SpecificProverNodeConfig } from './config.js';
|
|
41
|
-
import type {
|
|
42
|
-
import {
|
|
50
|
+
import type { CheckpointProver, CheckpointProverTestHooks } from './job/checkpoint-prover.js';
|
|
51
|
+
import type { EpochSessionHooks } from './job/epoch-session.js';
|
|
43
52
|
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
44
|
-
import
|
|
45
|
-
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
53
|
+
import { ProofPublishingService } from './proof-publishing-service.js';
|
|
46
54
|
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
55
|
+
import { SessionManager } from './session-manager.js';
|
|
47
56
|
|
|
48
57
|
type ProverNodeOptions = SpecificProverNodeConfig & Partial<DataStoreOptions>;
|
|
49
58
|
type DataStoreOptions = Pick<DataStoreConfig, 'dataDirectory'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'>;
|
|
50
59
|
|
|
51
60
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
61
|
+
* Grace period for the proof-publishing service to settle during shutdown. The service waits for
|
|
62
|
+
* any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
|
|
63
|
+
* the wait rather than letting `stop()` hang indefinitely.
|
|
64
|
+
*/
|
|
65
|
+
const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
|
|
69
|
+
* starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
|
|
70
|
+
* complete.
|
|
71
|
+
*
|
|
72
|
+
* The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
|
|
73
|
+
* `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
|
|
74
|
+
* manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
|
|
75
|
+
* the `SessionManager`. Each chain event is translated here into a single method call on it.
|
|
55
76
|
*/
|
|
56
|
-
export class ProverNode implements
|
|
77
|
+
export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Traceable {
|
|
57
78
|
private log = createLogger('prover-node');
|
|
58
|
-
private dateProvider = new DateProvider();
|
|
59
79
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
80
|
+
protected readonly checkpointStore: CheckpointStore;
|
|
81
|
+
protected readonly chonkCache: ChonkCache;
|
|
82
|
+
protected sessionManager: SessionManager | undefined;
|
|
83
|
+
|
|
84
|
+
private readonly config: ProverNodeOptions;
|
|
85
|
+
private readonly jobMetrics: ProverNodeJobMetrics;
|
|
86
|
+
private readonly rewardsMetrics: ProverNodeRewardsMetrics;
|
|
87
|
+
|
|
88
|
+
/** In-memory store for the L2BlockStream's local data provider. */
|
|
89
|
+
private tipsStore: L2TipsMemoryStore;
|
|
90
|
+
/** Block stream for checkpoint and reorg detection. */
|
|
91
|
+
private blockStream: EventDrivenL2BlockStream | undefined;
|
|
92
|
+
/**
|
|
93
|
+
* Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
|
|
94
|
+
* Seeded from the last fully-proven epoch at start(); advanced on every block-stream
|
|
95
|
+
* event by comparing the archiver's latest synced L2 slot against each epoch's
|
|
96
|
+
* submission deadline. Protected so tests can verify the start() seeding.
|
|
97
|
+
*/
|
|
98
|
+
protected lastExpiredEpoch: EpochNumber | undefined;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
|
|
102
|
+
* The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
|
|
103
|
+
* checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
|
|
104
|
+
* than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
|
|
105
|
+
*/
|
|
106
|
+
protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
|
|
107
|
+
|
|
108
|
+
/** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
|
|
109
|
+
private expiryTicker: RunningPromise | undefined;
|
|
64
110
|
|
|
65
111
|
public readonly tracer: Tracer;
|
|
66
112
|
|
|
67
|
-
protected
|
|
113
|
+
protected publishingService: ProofPublishingService | undefined;
|
|
68
114
|
|
|
69
115
|
constructor(
|
|
70
|
-
protected readonly prover: EpochProverManager,
|
|
116
|
+
protected readonly prover: EpochProverManager & EpochProverFactory,
|
|
71
117
|
protected readonly publisherFactory: ProverPublisherFactory,
|
|
72
118
|
protected readonly l2BlockSource: L2BlockSource & Partial<Service>,
|
|
73
119
|
protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
|
|
74
120
|
protected readonly contractDataSource: ContractDataSource,
|
|
75
121
|
protected readonly worldState: WorldStateSynchronizer,
|
|
76
|
-
protected readonly p2pClient:
|
|
77
|
-
protected readonly epochsMonitor: EpochMonitor,
|
|
122
|
+
protected readonly p2pClient: { getTxProvider(): ITxProvider } & Partial<Service>,
|
|
78
123
|
protected readonly rollupContract: RollupContract,
|
|
79
124
|
protected readonly l1Metrics: L1Metrics,
|
|
125
|
+
private readonly avmSimulator: AvmSimulator,
|
|
80
126
|
config: Partial<ProverNodeOptions> = {},
|
|
81
127
|
protected readonly telemetryClient: TelemetryClient = getTelemetryClient(),
|
|
128
|
+
private delayer?: Delayer,
|
|
129
|
+
private readonly dateProvider: DateProvider = new DateProvider(),
|
|
82
130
|
) {
|
|
83
131
|
this.config = {
|
|
84
132
|
proverNodePollingIntervalMs: 1_000,
|
|
85
133
|
proverNodeMaxPendingJobs: 100,
|
|
86
|
-
proverNodeMaxParallelBlocksPerEpoch:
|
|
134
|
+
proverNodeMaxParallelBlocksPerEpoch: 0,
|
|
87
135
|
txGatheringIntervalMs: 1_000,
|
|
88
136
|
txGatheringBatchSize: 10,
|
|
89
137
|
txGatheringMaxParallelRequestsPerNode: 100,
|
|
@@ -99,8 +147,36 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
99
147
|
this.tracer = telemetryClient.getTracer('ProverNode');
|
|
100
148
|
|
|
101
149
|
this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
|
|
102
|
-
|
|
103
150
|
this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
|
|
151
|
+
|
|
152
|
+
this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
|
|
153
|
+
|
|
154
|
+
this.chonkCache = new ChonkCache(this.log.getBindings());
|
|
155
|
+
this.checkpointStore = new CheckpointStore(
|
|
156
|
+
this.l2BlockSource,
|
|
157
|
+
{
|
|
158
|
+
proverFactory: this.prover,
|
|
159
|
+
chonkCache: this.chonkCache,
|
|
160
|
+
publicProcessorFactory: new PublicProcessorFactory(
|
|
161
|
+
this.contractDataSource,
|
|
162
|
+
this.avmSimulator,
|
|
163
|
+
this.dateProvider,
|
|
164
|
+
this.telemetryClient,
|
|
165
|
+
this.log.getBindings(),
|
|
166
|
+
),
|
|
167
|
+
dbProvider: this.worldState,
|
|
168
|
+
txProvider: this.p2pClient.getTxProvider(),
|
|
169
|
+
dateProvider: this.dateProvider,
|
|
170
|
+
proverId: this.prover.getProverId(),
|
|
171
|
+
metrics: this.jobMetrics,
|
|
172
|
+
txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
|
|
173
|
+
deadline: undefined,
|
|
174
|
+
// A checkpoint prover that fails (a sub-tree fault or a prune-induced fork fault) uploads a
|
|
175
|
+
// post-mortem for its own checkpoint, independently of any session. Fire-and-forget.
|
|
176
|
+
onFailed: prover => void this.tryUploadCheckpointFailure(prover),
|
|
177
|
+
},
|
|
178
|
+
this.log.getBindings(),
|
|
179
|
+
);
|
|
104
180
|
}
|
|
105
181
|
|
|
106
182
|
public getProverId() {
|
|
@@ -111,287 +187,598 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
111
187
|
return this.p2pClient;
|
|
112
188
|
}
|
|
113
189
|
|
|
190
|
+
/** Test-only: the shared L1 tx delayer, if enabled. */
|
|
191
|
+
public getDelayer(): Delayer | undefined {
|
|
192
|
+
return this.delayer;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Observability summary for the ProverNodeApi. */
|
|
196
|
+
public getJobs(): Promise<{ uuid: string; status: EpochProvingJobState; epochNumber: EpochNumber }[]> {
|
|
197
|
+
return Promise.resolve(this.sessionManager?.getJobs() ?? []);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Tests inspect this when validating reconcile behaviour. */
|
|
201
|
+
public getCheckpointStore(): CheckpointStore {
|
|
202
|
+
return this.checkpointStore;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Tests inspect this to verify chonk-cache release semantics. */
|
|
206
|
+
public getChonkCache(): ChonkCache {
|
|
207
|
+
return this.chonkCache;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Tests inspect this when looking up live sessions. */
|
|
211
|
+
public getSessionManager(): SessionManager {
|
|
212
|
+
if (!this.sessionManager) {
|
|
213
|
+
throw new Error('SessionManager not yet constructed — start() must be called first.');
|
|
214
|
+
}
|
|
215
|
+
return this.sessionManager;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Returns the underlying prover instance. */
|
|
219
|
+
public getProver() {
|
|
220
|
+
return this.prover;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ---------------- L2BlockStream handler ----------------
|
|
224
|
+
|
|
225
|
+
public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
|
|
226
|
+
switch (event.type) {
|
|
227
|
+
case 'chain-checkpointed':
|
|
228
|
+
await this.processCheckpointJump(event.checkpoint.number);
|
|
229
|
+
break;
|
|
230
|
+
case 'chain-pruned':
|
|
231
|
+
await this.handlePruneEvent(event.block);
|
|
232
|
+
break;
|
|
233
|
+
case 'chain-proven':
|
|
234
|
+
this.publishingService?.onChainProven(BlockNumber(event.block.number));
|
|
235
|
+
break;
|
|
236
|
+
// The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
|
|
237
|
+
// tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
|
|
238
|
+
// `chain-finalized` carries nothing the prover-node acts on.
|
|
239
|
+
case 'chain-proposed':
|
|
240
|
+
case 'chain-finalized':
|
|
241
|
+
case 'blocks-added':
|
|
242
|
+
break;
|
|
243
|
+
default: {
|
|
244
|
+
const _: never = event;
|
|
245
|
+
break;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// Advance the local tips store only after the proving-side handling (registration / prune) has
|
|
249
|
+
// succeeded. Any failure above propagates to the L2BlockStream (which logs and stops this poll
|
|
250
|
+
// pass) and skips this update, so the event is re-emitted on the next poll rather than skipped
|
|
251
|
+
await this.tipsStore.handleBlockStreamEvent(event);
|
|
252
|
+
}
|
|
253
|
+
|
|
114
254
|
/**
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
255
|
+
* Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
|
|
256
|
+
* each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
|
|
257
|
+
* `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
|
|
258
|
+
* catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
|
|
259
|
+
* `getCheckpoints` fetch only for checkpoints in provable epochs.
|
|
260
|
+
*
|
|
261
|
+
* The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
|
|
262
|
+
* fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
|
|
263
|
+
* behind so the next pass retries from the first checkpoint that did not complete.
|
|
118
264
|
*/
|
|
119
|
-
async
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
265
|
+
private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
|
|
266
|
+
if (targetCheckpoint <= this.lastProcessedCheckpoint) {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const l1Constants = await this.getL1Constants();
|
|
270
|
+
|
|
271
|
+
// Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
|
|
272
|
+
// When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
|
|
273
|
+
// load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
|
|
274
|
+
// its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
|
|
275
|
+
const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
|
|
276
|
+
let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
|
|
277
|
+
if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
|
|
278
|
+
const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
|
|
279
|
+
this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
|
|
280
|
+
from,
|
|
281
|
+
cappedFrom,
|
|
282
|
+
targetCheckpoint,
|
|
283
|
+
maxCheckpoints,
|
|
123
284
|
});
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
285
|
+
// Advance the cursor past the skipped checkpoints so they are never retried.
|
|
286
|
+
this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
|
|
287
|
+
from = cappedFrom;
|
|
288
|
+
}
|
|
289
|
+
const limit = Number(targetCheckpoint - from) + 1;
|
|
290
|
+
const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
|
|
291
|
+
|
|
292
|
+
// Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
|
|
293
|
+
// only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
|
|
294
|
+
// never drop an individual checkpoint inside an epoch we will prove.
|
|
295
|
+
const epochSkippable = new Map<EpochNumber, boolean>();
|
|
296
|
+
for (const metadata of metadatas) {
|
|
297
|
+
const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
|
|
298
|
+
let skippable = epochSkippable.get(epochNumber);
|
|
299
|
+
if (skippable === undefined) {
|
|
300
|
+
skippable =
|
|
301
|
+
(await this.isEpochFullyProven(epochNumber, l1Constants)) ||
|
|
302
|
+
(await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
|
|
303
|
+
epochSkippable.set(epochNumber, skippable);
|
|
130
304
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
} catch (err) {
|
|
134
|
-
if (err instanceof EmptyEpochError) {
|
|
135
|
-
this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
|
|
305
|
+
if (skippable) {
|
|
306
|
+
this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
|
|
136
307
|
} else {
|
|
137
|
-
this.
|
|
308
|
+
await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
|
|
138
309
|
}
|
|
310
|
+
// Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
|
|
311
|
+
// throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
|
|
312
|
+
this.lastProcessedCheckpoint = metadata.checkpointNumber;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
|
|
317
|
+
private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
|
|
318
|
+
const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
|
|
319
|
+
if (!published) {
|
|
320
|
+
throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
|
|
321
|
+
}
|
|
322
|
+
const checkpoint = published.checkpoint;
|
|
323
|
+
this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
|
|
324
|
+
checkpointNumber: checkpoint.number,
|
|
325
|
+
epochNumber,
|
|
326
|
+
slotNumber: checkpoint.header.slotNumber,
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
const registerData = await this.collectRegisterData(checkpoint, published.attestations);
|
|
330
|
+
await this.checkpointStore.addOrUpdate(checkpoint, registerData);
|
|
331
|
+
await this.sessionManager?.onCheckpointAdded(epochNumber);
|
|
332
|
+
|
|
333
|
+
// Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
|
|
334
|
+
// range then prunes at the true divergence instead of the nearest sparse tip anchor.
|
|
335
|
+
await this.tipsStore.recordBlockHashes(
|
|
336
|
+
await Promise.all(
|
|
337
|
+
checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
|
|
338
|
+
),
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
|
|
344
|
+
* and the archive sibling path.
|
|
345
|
+
*/
|
|
346
|
+
private async collectRegisterData(
|
|
347
|
+
checkpoint: Checkpoint,
|
|
348
|
+
attestations: PublishedCheckpoint['attestations'],
|
|
349
|
+
): Promise<RegisterCheckpointData> {
|
|
350
|
+
const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
|
|
351
|
+
const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
|
|
352
|
+
const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
|
|
353
|
+
const lastBlock = checkpoint.blocks.at(-1)!;
|
|
354
|
+
const lastBlockHash = await lastBlock.header.hash();
|
|
355
|
+
await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
|
|
356
|
+
const previousArchiveSiblingPath = await getLastSiblingPath(
|
|
357
|
+
MerkleTreeId.ARCHIVE,
|
|
358
|
+
this.worldState.getSnapshot(previousBlockNumber),
|
|
359
|
+
);
|
|
360
|
+
return {
|
|
361
|
+
attestations,
|
|
362
|
+
previousBlockHeader,
|
|
363
|
+
l1ToL2Messages,
|
|
364
|
+
previousArchiveSiblingPath,
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
|
|
370
|
+
* checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
|
|
371
|
+
* rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
|
|
372
|
+
* orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
|
|
373
|
+
* fails and the prune is retried next iteration.
|
|
374
|
+
*/
|
|
375
|
+
private async handlePruneEvent(prunedToBlock: L2BlockId) {
|
|
376
|
+
this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
|
|
377
|
+
|
|
378
|
+
// Resolve the cursor floor BEFORE removing provers: cancelAndRemoveAboveBlock returns only the provers it removed,
|
|
379
|
+
// so a throw after removing would leave a retry pass with nothing to act on. Resolving first means a throw leaves
|
|
380
|
+
// everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
|
|
381
|
+
let cursorFloor: CheckpointNumber;
|
|
382
|
+
if (prunedToBlock.number === 0) {
|
|
383
|
+
cursorFloor = CheckpointNumber.ZERO;
|
|
384
|
+
} else {
|
|
385
|
+
const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
|
|
386
|
+
if (targetData === undefined) {
|
|
387
|
+
throw new Error(
|
|
388
|
+
`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
// Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
|
|
392
|
+
// reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
|
|
393
|
+
// would permanently skip a rebuilt same-number checkpoint.
|
|
394
|
+
cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
const affected = this.checkpointStore.cancelAndRemoveAboveBlock(prunedToBlock.number);
|
|
398
|
+
|
|
399
|
+
if (this.lastProcessedCheckpoint > cursorFloor) {
|
|
400
|
+
this.lastProcessedCheckpoint = cursorFloor;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
if (affected.length === 0) {
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
const l1Constants = await this.getL1Constants();
|
|
407
|
+
const affectedEpochs = Array.from(
|
|
408
|
+
new Set(affected.map(p => Number(getEpochAtSlot(p.slotNumber, l1Constants)))),
|
|
409
|
+
).map(n => EpochNumber(n));
|
|
410
|
+
// The session manager cancels every affected session, which in turn calls
|
|
411
|
+
// publishingService.withdraw(uuid) for each candidate; no separate notification to the
|
|
412
|
+
// publishing service is needed.
|
|
413
|
+
await this.sessionManager?.onPrune(affectedEpochs);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Returns true once the chain has advanced past the given epoch's proof-submission window.
|
|
418
|
+
* Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
|
|
419
|
+
* archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
|
|
420
|
+
* against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
|
|
421
|
+
* be read yet.
|
|
422
|
+
*/
|
|
423
|
+
private async isEpochPastProofSubmissionWindow(
|
|
424
|
+
epochNumber: EpochNumber,
|
|
425
|
+
l1Constants: L1RollupConstants,
|
|
426
|
+
): Promise<boolean> {
|
|
427
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
428
|
+
if (latestSlot === undefined) {
|
|
139
429
|
return false;
|
|
140
430
|
}
|
|
431
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
432
|
+
return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
|
|
437
|
+
* newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
|
|
438
|
+
* CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
|
|
439
|
+
* of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
|
|
440
|
+
* the last check or the archiver's slot can't be read.
|
|
441
|
+
*/
|
|
442
|
+
private async checkEpochExpiry(): Promise<void> {
|
|
443
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
444
|
+
if (latestSlot === undefined) {
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
const l1Constants = await this.getL1Constants();
|
|
448
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
449
|
+
const offset = l1Constants.proofSubmissionEpochs + 1;
|
|
450
|
+
if (latestEpoch < offset) {
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
|
|
454
|
+
const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
|
|
455
|
+
if (newlyExpiredUpTo < from) {
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
for (let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)) {
|
|
459
|
+
await this.expireEpoch(e);
|
|
460
|
+
}
|
|
461
|
+
this.lastExpiredEpoch = newlyExpiredUpTo;
|
|
141
462
|
}
|
|
142
463
|
|
|
143
464
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
465
|
+
* Releases chonk-cache entries for every block in the supplied epoch (best-effort) and reaps every
|
|
466
|
+
* CheckpointProver in the store whose epoch is at or below it.
|
|
146
467
|
*/
|
|
468
|
+
private async expireEpoch(epoch: EpochNumber): Promise<void> {
|
|
469
|
+
try {
|
|
470
|
+
const blocks = await this.l2BlockSource.getBlocks({ epoch, onlyCheckpointed: true });
|
|
471
|
+
if (blocks.length > 0) {
|
|
472
|
+
this.chonkCache.releaseForBlocks(blocks);
|
|
473
|
+
}
|
|
474
|
+
} catch (err) {
|
|
475
|
+
this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
|
|
476
|
+
}
|
|
477
|
+
this.checkpointStore.reapExpired(epoch);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// ---------------- public API ----------------
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Schedules proving for the given epoch and returns the job id without waiting for completion.
|
|
484
|
+
*/
|
|
485
|
+
public async startProof(epochNumber: EpochNumber): Promise<string> {
|
|
486
|
+
if (!this.sessionManager) {
|
|
487
|
+
throw new Error('ProverNode not started');
|
|
488
|
+
}
|
|
489
|
+
return await this.sessionManager.startProof(epochNumber);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// ---------------- Service lifecycle ----------------
|
|
493
|
+
|
|
147
494
|
async start() {
|
|
148
|
-
this.
|
|
495
|
+
await this.checkpointStore.start();
|
|
496
|
+
|
|
149
497
|
await this.publisherFactory.start();
|
|
150
|
-
this.
|
|
498
|
+
this.publishingService = new ProofPublishingService({
|
|
499
|
+
publisherFactory: this.publisherFactory,
|
|
500
|
+
l2BlockSource: this.l2BlockSource,
|
|
501
|
+
dateProvider: this.dateProvider,
|
|
502
|
+
config: { skipSubmitProof: !!this.config.proverNodeDisableProofPublish },
|
|
503
|
+
bindings: this.log.getBindings(),
|
|
504
|
+
});
|
|
505
|
+
this.sessionManager = this.createSessionManager(this.publishingService);
|
|
506
|
+
// SessionManager owns its own periodic tick; start it here so it begins picking up
|
|
507
|
+
// epochs that become complete by time (no fresh checkpoint event) and advances once
|
|
508
|
+
// the previous epoch is proven on L1.
|
|
509
|
+
this.sessionManager.start();
|
|
510
|
+
// Now that the store + manager exist, arm the live-state observable gauges.
|
|
511
|
+
this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
|
|
512
|
+
|
|
513
|
+
const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
|
|
514
|
+
this.lastExpiredEpoch = lastFullyProvenEpoch;
|
|
515
|
+
this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
|
|
516
|
+
this.blockStream = new EventDrivenL2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
|
|
517
|
+
pollIntervalMS: this.config.proverNodePollingIntervalMs,
|
|
518
|
+
tipsOnly: true,
|
|
519
|
+
});
|
|
520
|
+
this.blockStream.start();
|
|
521
|
+
|
|
522
|
+
// The periodic ticker is the sole driver of the expiry sweep: it fires every poll interval whether
|
|
523
|
+
// or not block-stream events arrive, and RunningPromise never overlaps its own runs, so the sweep's
|
|
524
|
+
// `lastExpiredEpoch` high-water mark advances — and each epoch's post-mortem uploads — exactly once.
|
|
525
|
+
this.expiryTicker = new RunningPromise(
|
|
526
|
+
() => this.checkEpochExpiry(),
|
|
527
|
+
this.log,
|
|
528
|
+
this.config.proverNodePollingIntervalMs,
|
|
529
|
+
);
|
|
530
|
+
this.expiryTicker.start();
|
|
531
|
+
|
|
151
532
|
await this.rewardsMetrics.start();
|
|
152
533
|
this.l1Metrics.start();
|
|
153
534
|
this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
|
|
154
535
|
}
|
|
155
536
|
|
|
156
|
-
/**
|
|
157
|
-
* Stops the prover node and all its dependencies.
|
|
158
|
-
*/
|
|
159
537
|
async stop() {
|
|
160
538
|
this.log.info('Stopping ProverNode');
|
|
161
|
-
|
|
539
|
+
this.jobMetrics.stopObservingState();
|
|
540
|
+
await this.blockStream?.stop();
|
|
541
|
+
await this.expiryTicker?.stop();
|
|
542
|
+
if (this.sessionManager) {
|
|
543
|
+
await this.sessionManager.stop();
|
|
544
|
+
}
|
|
545
|
+
if (this.publishingService) {
|
|
546
|
+
// Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
|
|
547
|
+
// settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
|
|
548
|
+
// the tx may still mine, but shutdown must not hang on it.
|
|
549
|
+
const publishingService = this.publishingService;
|
|
550
|
+
await executeTimeout(
|
|
551
|
+
() => publishingService.stop(),
|
|
552
|
+
PUBLISHING_SERVICE_STOP_TIMEOUT_MS,
|
|
553
|
+
'prover-node publishing-service stop',
|
|
554
|
+
).catch(err => this.log.warn(`Timed out stopping proof publishing service`, err));
|
|
555
|
+
}
|
|
556
|
+
await this.checkpointStore.stop();
|
|
557
|
+
this.chonkCache.stop();
|
|
162
558
|
await this.prover.stop();
|
|
163
|
-
await tryStop(this.p2pClient);
|
|
164
|
-
await tryStop(this.l2BlockSource);
|
|
165
559
|
await tryStop(this.publisherFactory);
|
|
166
|
-
this.publisher?.interrupt();
|
|
167
|
-
await Promise.all(Array.from(this.jobs.values()).map(job => job.stop()));
|
|
168
|
-
await this.worldState.stop();
|
|
169
560
|
this.rewardsMetrics.stop();
|
|
170
561
|
this.l1Metrics.stop();
|
|
171
562
|
await this.telemetryClient.stop();
|
|
172
563
|
this.log.info('Stopped ProverNode');
|
|
173
564
|
}
|
|
174
565
|
|
|
175
|
-
/**
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
566
|
+
/**
|
|
567
|
+
* Constructs the session manager. Extracted so subclasses (test harness) can swap the
|
|
568
|
+
* implementation. Wired to upload a post-mortem when a full session ends in its own genuine failure
|
|
569
|
+
* (`EpochSession.hasFailed()` — top-tree/submit failed with every prover healthy, so definitively not
|
|
570
|
+
* a prune). A `stopped` session (a prover under it failed) is not uploaded; it recovers on re-add.
|
|
571
|
+
*/
|
|
572
|
+
protected createSessionManager(publishingService: ProofPublishingService): SessionManager {
|
|
573
|
+
return new SessionManager({
|
|
574
|
+
checkpointStore: this.checkpointStore,
|
|
575
|
+
l2BlockSource: this.l2BlockSource,
|
|
576
|
+
proverFactory: this.prover,
|
|
577
|
+
proverId: this.prover.getProverId(),
|
|
578
|
+
publishingService,
|
|
579
|
+
metrics: this.jobMetrics,
|
|
580
|
+
dateProvider: this.dateProvider,
|
|
581
|
+
config: {
|
|
582
|
+
maxPendingJobs: this.config.proverNodeMaxPendingJobs,
|
|
583
|
+
tickIntervalMs: this.config.proverNodePollingIntervalMs,
|
|
584
|
+
finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs,
|
|
585
|
+
},
|
|
586
|
+
onSessionFailed: async session => {
|
|
587
|
+
await this.tryUploadEpochFailure(session.getId(), session.getCheckpoints());
|
|
588
|
+
},
|
|
589
|
+
bindings: this.log.getBindings(),
|
|
590
|
+
});
|
|
179
591
|
}
|
|
180
592
|
|
|
181
|
-
/**
|
|
182
|
-
|
|
183
|
-
|
|
593
|
+
/**
|
|
594
|
+
* Installs session hooks for the e2e harness to interpose around top-tree proving
|
|
595
|
+
* (gate, override, or observe it) without monkey-patching the orchestrator factory.
|
|
596
|
+
* Applies to every session constructed after this call.
|
|
597
|
+
*/
|
|
598
|
+
public setSessionHooks(hooks: EpochSessionHooks): void {
|
|
599
|
+
if (!this.sessionManager) {
|
|
600
|
+
throw new Error('ProverNode not started; call start() before setting session hooks.');
|
|
601
|
+
}
|
|
602
|
+
this.sessionManager.setSessionHooks(hooks);
|
|
184
603
|
}
|
|
185
604
|
|
|
186
605
|
/**
|
|
187
|
-
*
|
|
606
|
+
* Installs checkpoint-prover test hooks (e.g. forcing a sub-tree failure) applied to every
|
|
607
|
+
* CheckpointProver constructed after this call. For the e2e harness only.
|
|
188
608
|
*/
|
|
189
|
-
public
|
|
190
|
-
|
|
191
|
-
void this.runJob(job);
|
|
609
|
+
public setCheckpointHooks(hooks: CheckpointProverTestHooks): void {
|
|
610
|
+
this.checkpointStore.setTestHooks(hooks);
|
|
192
611
|
}
|
|
193
612
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
} else if (state === 'failed') {
|
|
207
|
-
this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
208
|
-
await this.tryUploadEpochFailure(job);
|
|
209
|
-
} else {
|
|
210
|
-
this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
211
|
-
}
|
|
212
|
-
} catch (err) {
|
|
213
|
-
this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
|
|
214
|
-
} finally {
|
|
215
|
-
this.jobs.delete(job.getId());
|
|
613
|
+
/**
|
|
614
|
+
* Uploads a post-mortem snapshot for an epoch whose full session failed to prove, built from that
|
|
615
|
+
* session's checkpoint provers. Fired from the session manager's `onSessionFailed` callback (a
|
|
616
|
+
* genuine, race-free failure). Exposed as a method so tests can spy on it. No-ops if no failed-epoch
|
|
617
|
+
* store is configured or the checkpoint set is empty.
|
|
618
|
+
*/
|
|
619
|
+
public async tryUploadEpochFailure(
|
|
620
|
+
id: string,
|
|
621
|
+
checkpoints: readonly CheckpointProver[],
|
|
622
|
+
): Promise<string | undefined> {
|
|
623
|
+
if (!this.config.proverNodeFailedEpochStore || checkpoints.length === 0) {
|
|
624
|
+
return undefined;
|
|
216
625
|
}
|
|
626
|
+
const data = await SessionManager.buildProvingData(checkpoints);
|
|
627
|
+
return await uploadEpochProofFailure(
|
|
628
|
+
this.config.proverNodeFailedEpochStore,
|
|
629
|
+
// The session's own id; `uploadEpochProofFailure` already prefixes the path with the epoch number.
|
|
630
|
+
id,
|
|
631
|
+
data,
|
|
632
|
+
this.l2BlockSource as Archiver,
|
|
633
|
+
this.worldState,
|
|
634
|
+
assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
|
|
635
|
+
this.log,
|
|
636
|
+
);
|
|
217
637
|
}
|
|
218
638
|
|
|
219
|
-
|
|
220
|
-
|
|
639
|
+
/**
|
|
640
|
+
* Uploads a post-mortem for a single failed checkpoint prover, built from just that checkpoint's
|
|
641
|
+
* proving data. Fired (fire-and-forget) from the store's `onFailed` callback for any non-cancel
|
|
642
|
+
* block-proof failure — a genuine sub-tree fault or a prune-induced fork fault alike. No-ops if no
|
|
643
|
+
* failed-epoch store is configured, or if the checkpoint is no longer canonical (a prune left nothing
|
|
644
|
+
* to diagnose). Swallows its own errors so a fire-and-forget caller can't leak.
|
|
645
|
+
*/
|
|
646
|
+
public async tryUploadCheckpointFailure(prover: CheckpointProver): Promise<string | undefined> {
|
|
647
|
+
if (!this.config.proverNodeFailedEpochStore) {
|
|
648
|
+
return undefined;
|
|
649
|
+
}
|
|
650
|
+
try {
|
|
651
|
+
// A prune-induced fork fault and a genuine sub-tree failure are indistinguishable at the moment the
|
|
652
|
+
// prover rejects (no control-plane cancel has landed yet). But the archiver is the authoritative
|
|
653
|
+
// committed chain: if this checkpoint was pruned out, its last block is no longer canonical there.
|
|
654
|
+
// Only upload for a checkpoint that still exists on-chain — a prune leaves nothing to diagnose, and
|
|
655
|
+
// the snapshot (full world-state + archiver) is expensive to produce and store.
|
|
656
|
+
if (!(await this.isCheckpointCanonical(prover.checkpoint))) {
|
|
657
|
+
this.log.debug(`Skipping checkpoint-failure upload for ${prover.id}: no longer canonical (pruned)`, {
|
|
658
|
+
checkpointNumber: prover.checkpoint.number,
|
|
659
|
+
});
|
|
660
|
+
return undefined;
|
|
661
|
+
}
|
|
662
|
+
const data = await SessionManager.buildProvingData([prover]);
|
|
221
663
|
return await uploadEpochProofFailure(
|
|
222
664
|
this.config.proverNodeFailedEpochStore,
|
|
223
|
-
|
|
224
|
-
|
|
665
|
+
// The prover's content-addressed id; the epoch number is already in the upload path.
|
|
666
|
+
prover.id,
|
|
667
|
+
data,
|
|
225
668
|
this.l2BlockSource as Archiver,
|
|
226
669
|
this.worldState,
|
|
227
670
|
assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')),
|
|
228
671
|
this.log,
|
|
229
672
|
);
|
|
673
|
+
} catch (err) {
|
|
674
|
+
this.log.error(`Error uploading checkpoint failure for ${prover.id}`, err);
|
|
675
|
+
return undefined;
|
|
230
676
|
}
|
|
231
677
|
}
|
|
232
678
|
|
|
233
|
-
|
|
234
|
-
* Returns the prover instance.
|
|
235
|
-
*/
|
|
236
|
-
public getProver() {
|
|
237
|
-
return this.prover;
|
|
238
|
-
}
|
|
679
|
+
// ---------------- helpers ----------------
|
|
239
680
|
|
|
240
681
|
/**
|
|
241
|
-
*
|
|
682
|
+
* True if the checkpoint still exists on the canonical chain: the archiver holds a block at its last
|
|
683
|
+
* block's height whose archive root matches. A prune (fork fault) leaves the block missing or replaced,
|
|
684
|
+
* so this returns false. Protected for direct unit-test access.
|
|
242
685
|
*/
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
status: job.getState(),
|
|
248
|
-
epochNumber: job.getEpochNumber(),
|
|
249
|
-
})),
|
|
250
|
-
);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
protected async getActiveJobsForEpoch(
|
|
254
|
-
epochNumber: EpochNumber,
|
|
255
|
-
): Promise<{ uuid: string; status: EpochProvingJobState }[]> {
|
|
256
|
-
const jobs = await this.getJobs();
|
|
257
|
-
return jobs.filter(job => job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
private checkMaximumPendingJobs() {
|
|
261
|
-
const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
|
|
262
|
-
if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
|
|
263
|
-
throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
|
|
686
|
+
protected async isCheckpointCanonical(checkpoint: Checkpoint): Promise<boolean> {
|
|
687
|
+
const lastBlock = checkpoint.blocks.at(-1);
|
|
688
|
+
if (!lastBlock) {
|
|
689
|
+
return false;
|
|
264
690
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
@trackSpan('ProverNode.createProvingJob', epochNumber => ({ [Attributes.EPOCH_NUMBER]: epochNumber }))
|
|
268
|
-
private async createProvingJob(epochNumber: EpochNumber, opts: { skipEpochCheck?: boolean } = {}) {
|
|
269
|
-
this.checkMaximumPendingJobs();
|
|
270
|
-
|
|
271
|
-
this.publisher = await this.publisherFactory.create();
|
|
272
|
-
|
|
273
|
-
// Gather all data for this epoch
|
|
274
|
-
const epochData = await this.gatherEpochData(epochNumber);
|
|
275
|
-
const fromCheckpoint = epochData.checkpoints[0].number;
|
|
276
|
-
const toCheckpoint = epochData.checkpoints.at(-1)!.number;
|
|
277
|
-
const fromBlock = epochData.checkpoints[0].blocks[0].number;
|
|
278
|
-
const toBlock = epochData.checkpoints.at(-1)!.blocks.at(-1)!.number;
|
|
279
|
-
this.log.verbose(
|
|
280
|
-
`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`,
|
|
281
|
-
);
|
|
282
|
-
|
|
283
|
-
// Fast forward world state to right before the target block and get a fork
|
|
284
|
-
await this.worldState.syncImmediate(toBlock);
|
|
285
|
-
|
|
286
|
-
// Create a processor factory
|
|
287
|
-
const publicProcessorFactory = new PublicProcessorFactory(
|
|
288
|
-
this.contractDataSource,
|
|
289
|
-
this.dateProvider,
|
|
290
|
-
this.telemetryClient,
|
|
291
|
-
this.log.getBindings(),
|
|
292
|
-
);
|
|
293
|
-
|
|
294
|
-
// Set deadline for this job to run. It will abort if it takes too long.
|
|
295
|
-
const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
|
|
296
|
-
const deadline = new Date(Number(deadlineTs) * 1000);
|
|
297
|
-
const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
|
|
298
|
-
this.jobs.set(job.getId(), job);
|
|
299
|
-
return job;
|
|
691
|
+
const onChain = await this.l2BlockSource.getBlock({ number: lastBlock.number });
|
|
692
|
+
return !!onChain && onChain.archive.root.equals(checkpoint.archive.root);
|
|
300
693
|
}
|
|
301
694
|
|
|
302
695
|
@memoize
|
|
303
|
-
private getL1Constants() {
|
|
696
|
+
private getL1Constants(): Promise<L1RollupConstants> {
|
|
304
697
|
return this.l2BlockSource.getL1Constants();
|
|
305
698
|
}
|
|
306
699
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
const
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
return { checkpoints, txs, l1ToL2Messages, epochNumber, previousBlockHeader, attestations };
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
private async gatherCheckpoints(epochNumber: EpochNumber) {
|
|
322
|
-
const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
|
|
323
|
-
if (checkpoints.length === 0) {
|
|
324
|
-
throw new EmptyEpochError(epochNumber);
|
|
700
|
+
/**
|
|
701
|
+
* Returns true if every block in the given epoch is proven on L1. An epoch is only
|
|
702
|
+
* fully proven when its *last* block is proven. Protected for direct unit-test access.
|
|
703
|
+
*/
|
|
704
|
+
protected async isEpochFullyProven(
|
|
705
|
+
epochNumber: EpochNumber,
|
|
706
|
+
l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
|
|
707
|
+
): Promise<boolean> {
|
|
708
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
|
|
709
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
710
|
+
return false;
|
|
325
711
|
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
private async gatherTxs(epochNumber: EpochNumber, checkpoints: Checkpoint[]) {
|
|
330
|
-
const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
|
|
331
|
-
const txProvider = this.p2pClient.getTxProvider();
|
|
332
|
-
const blocks = checkpoints.flatMap(checkpoint => checkpoint.blocks);
|
|
333
|
-
const txsByBlock = await Promise.all(blocks.map(block => txProvider.getTxsForBlock(block, { deadline })));
|
|
334
|
-
const txs = txsByBlock.map(({ txs }) => txs).flat();
|
|
335
|
-
const missingTxs = txsByBlock.map(({ missingTxs }) => missingTxs).flat();
|
|
336
|
-
|
|
337
|
-
if (missingTxs.length === 0) {
|
|
338
|
-
this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, { epochNumber });
|
|
339
|
-
return txs;
|
|
712
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
|
|
713
|
+
if (!provenHeader) {
|
|
714
|
+
return false;
|
|
340
715
|
}
|
|
341
|
-
|
|
342
|
-
|
|
716
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
717
|
+
if (epochNumber < provenEpoch) {
|
|
718
|
+
return true;
|
|
719
|
+
}
|
|
720
|
+
if (epochNumber > provenEpoch) {
|
|
721
|
+
return false;
|
|
722
|
+
}
|
|
723
|
+
return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
|
|
343
724
|
}
|
|
344
725
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
726
|
+
/** Protected for direct unit-test access. */
|
|
727
|
+
protected async isProvenBlockLastOfItsEpoch(
|
|
728
|
+
provenBlockNumber: BlockNumber,
|
|
729
|
+
provenEpoch: EpochNumber,
|
|
730
|
+
l1Constants: Pick<L1RollupConstants, 'epochDuration'>,
|
|
731
|
+
): Promise<boolean> {
|
|
732
|
+
const nextHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber + 1) }))?.header;
|
|
733
|
+
if (nextHeader) {
|
|
734
|
+
return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
|
|
352
735
|
}
|
|
353
|
-
return
|
|
736
|
+
return this.l2BlockSource.isEpochComplete(provenEpoch);
|
|
354
737
|
}
|
|
355
738
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
739
|
+
/**
|
|
740
|
+
* Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
|
|
741
|
+
* `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
|
|
742
|
+
* proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
|
|
743
|
+
* proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
|
|
744
|
+
*/
|
|
745
|
+
protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
|
|
746
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
|
|
747
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
748
|
+
return { lastFullyProvenEpoch: undefined };
|
|
363
749
|
}
|
|
364
|
-
|
|
365
|
-
this.
|
|
366
|
-
|
|
750
|
+
const l1Constants = await this.getL1Constants();
|
|
751
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
|
|
752
|
+
if (!provenHeader) {
|
|
753
|
+
return { lastFullyProvenEpoch: undefined };
|
|
754
|
+
}
|
|
755
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
756
|
+
if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
|
|
757
|
+
return { lastFullyProvenEpoch: provenEpoch };
|
|
758
|
+
}
|
|
759
|
+
const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
|
|
760
|
+
return { lastFullyProvenEpoch };
|
|
367
761
|
}
|
|
368
762
|
|
|
369
|
-
/**
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
this.worldState,
|
|
381
|
-
this.prover.createEpochProver(),
|
|
382
|
-
publicProcessorFactory,
|
|
383
|
-
publisher,
|
|
384
|
-
this.l2BlockSource,
|
|
385
|
-
this.jobMetrics,
|
|
386
|
-
deadline,
|
|
387
|
-
{ parallelBlockLimit, skipSubmitProof: proverNodeDisableProofPublish, ...opts },
|
|
388
|
-
this.log.getBindings(),
|
|
389
|
-
);
|
|
763
|
+
/**
|
|
764
|
+
* Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
|
|
765
|
+
* from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
|
|
766
|
+
* partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
|
|
767
|
+
*/
|
|
768
|
+
protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
|
|
769
|
+
if (lastFullyProvenEpoch === undefined) {
|
|
770
|
+
return CheckpointNumber.ZERO;
|
|
771
|
+
}
|
|
772
|
+
const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
|
|
773
|
+
return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
|
|
390
774
|
}
|
|
391
775
|
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
776
|
+
private async gatherPreviousBlockHeader(previousBlockNumber: number) {
|
|
777
|
+
const data = await this.l2BlockSource.getBlockData({ number: BlockNumber(previousBlockNumber) });
|
|
778
|
+
if (!data?.header) {
|
|
779
|
+
throw new Error(`Previous block header ${previousBlockNumber} not found`);
|
|
780
|
+
}
|
|
781
|
+
return data.header;
|
|
395
782
|
}
|
|
396
783
|
|
|
397
784
|
private validateConfig() {
|
|
@@ -410,9 +797,5 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
|
|
|
410
797
|
}
|
|
411
798
|
}
|
|
412
799
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
super(`No blocks found for epoch ${epochNumber}`);
|
|
416
|
-
this.name = 'EmptyEpochError';
|
|
417
|
-
}
|
|
418
|
-
}
|
|
800
|
+
// Re-export so handlers can compare states externally.
|
|
801
|
+
export { EpochProvingJobTerminalState };
|