@aztec/prover-node 5.0.0-rc.1 → 5.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -8
- package/dest/actions/rerun-epoch-proving-job.d.ts +2 -2
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/checkpoint-store.d.ts +10 -6
- package/dest/checkpoint-store.d.ts.map +1 -1
- package/dest/checkpoint-store.js +10 -5
- package/dest/job/checkpoint-prover.d.ts +1 -1
- package/dest/job/checkpoint-prover.d.ts.map +1 -1
- package/dest/job/checkpoint-prover.js +4 -1
- package/dest/job/epoch-session.d.ts +2 -2
- package/dest/job/epoch-session.d.ts.map +1 -1
- package/dest/job/epoch-session.js +20 -9
- package/dest/metrics.d.ts +7 -4
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +13 -5
- package/dest/proof-publishing-service.d.ts +4 -2
- package/dest/proof-publishing-service.d.ts.map +1 -1
- package/dest/proof-publishing-service.js +1 -0
- package/dest/prover-node-publisher.d.ts +5 -5
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +2 -3
- package/dest/prover-node.d.ts +25 -15
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +150 -47
- package/package.json +23 -23
- package/src/checkpoint-store.ts +11 -6
- package/src/job/checkpoint-prover.ts +4 -1
- package/src/job/epoch-session.ts +21 -9
- package/src/metrics.ts +14 -11
- package/src/proof-publishing-service.ts +4 -1
- package/src/prover-node-publisher.ts +11 -10
- package/src/prover-node.ts +167 -56
package/src/job/epoch-session.ts
CHANGED
|
@@ -92,7 +92,7 @@ export type EpochSessionDeps = {
|
|
|
92
92
|
*
|
|
93
93
|
* Lifecycle (happy path):
|
|
94
94
|
*
|
|
95
|
-
* initialized → awaiting-checkpoints → completed
|
|
95
|
+
* initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
|
|
96
96
|
*
|
|
97
97
|
* Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
|
|
98
98
|
* `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
|
|
@@ -320,6 +320,12 @@ export class EpochSession implements Traceable {
|
|
|
320
320
|
0,
|
|
321
321
|
);
|
|
322
322
|
|
|
323
|
+
// Reflect the publish phase. Guard against a terminal state set concurrently by cancel() — the
|
|
324
|
+
// post-submit isTerminal() check below relies on cancel still winning.
|
|
325
|
+
if (!this.isTerminal()) {
|
|
326
|
+
this.state = 'publishing-proof';
|
|
327
|
+
}
|
|
328
|
+
|
|
323
329
|
const outcome = await this.deps.publishingService.submit({
|
|
324
330
|
id: this.uuid,
|
|
325
331
|
epoch: this.spec.epochNumber,
|
|
@@ -333,6 +339,7 @@ export class EpochSession implements Traceable {
|
|
|
333
339
|
proof: proof.proof,
|
|
334
340
|
batchedBlobInputs: proof.batchedBlobInputs,
|
|
335
341
|
attestations,
|
|
342
|
+
headers: this.checkpoints.map(c => c.checkpoint.header),
|
|
336
343
|
});
|
|
337
344
|
|
|
338
345
|
if (this.isTerminal()) {
|
|
@@ -347,7 +354,7 @@ export class EpochSession implements Traceable {
|
|
|
347
354
|
{ uuid: this.uuid, ...this.spec },
|
|
348
355
|
);
|
|
349
356
|
this.state = 'completed';
|
|
350
|
-
this.deps.metrics.recordProvingJob(timer.ms(),
|
|
357
|
+
this.deps.metrics.recordProvingJob(timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
|
|
351
358
|
return;
|
|
352
359
|
case 'superseded':
|
|
353
360
|
this.log.info(`EpochSession ${this.uuid} superseded by a longer candidate`, {
|
|
@@ -410,15 +417,20 @@ export class EpochSession implements Traceable {
|
|
|
410
417
|
}
|
|
411
418
|
}
|
|
412
419
|
|
|
413
|
-
private toTopTreeHooks(): TopTreeJobHooks
|
|
420
|
+
private toTopTreeHooks(): TopTreeJobHooks {
|
|
414
421
|
const hooks = this.deps.hooks;
|
|
415
|
-
if (!hooks?.beforeTopTreeProve && !hooks?.afterTopTreeProve && !hooks?.topTreeProveOverride) {
|
|
416
|
-
return undefined;
|
|
417
|
-
}
|
|
418
422
|
return {
|
|
419
|
-
beforeProve
|
|
420
|
-
|
|
421
|
-
|
|
423
|
+
// `beforeProve` fires once the sub-tree (checkpoint block) proofs are ready and the root prove is
|
|
424
|
+
// about to start — the boundary between `awaiting-checkpoints` and proving the top tree. Don't
|
|
425
|
+
// clobber a terminal state set concurrently by cancel().
|
|
426
|
+
beforeProve: async () => {
|
|
427
|
+
if (!this.isTerminal()) {
|
|
428
|
+
this.state = 'awaiting-root';
|
|
429
|
+
}
|
|
430
|
+
await hooks?.beforeTopTreeProve?.();
|
|
431
|
+
},
|
|
432
|
+
afterProve: hooks?.afterTopTreeProve,
|
|
433
|
+
proveOverride: hooks?.topTreeProveOverride,
|
|
422
434
|
};
|
|
423
435
|
}
|
|
424
436
|
}
|
package/src/metrics.ts
CHANGED
|
@@ -22,7 +22,6 @@ import type { CheckpointStore } from './checkpoint-store.js';
|
|
|
22
22
|
import type { SessionManager } from './session-manager.js';
|
|
23
23
|
|
|
24
24
|
export class ProverNodeJobMetrics {
|
|
25
|
-
proverEpochExecutionDuration: Histogram;
|
|
26
25
|
provingJobDuration: Histogram;
|
|
27
26
|
provingJobCheckpoints: Gauge;
|
|
28
27
|
provingJobBlocks: Gauge;
|
|
@@ -31,6 +30,9 @@ export class ProverNodeJobMetrics {
|
|
|
31
30
|
private blobProcessingDuration: Gauge;
|
|
32
31
|
private blockProcessingDuration: Histogram;
|
|
33
32
|
private checkpointProcessingDuration: Histogram;
|
|
33
|
+
private checkpointProvingDuration: Histogram;
|
|
34
|
+
private checkpointBlocks: Histogram;
|
|
35
|
+
private checkpointTransactions: Histogram;
|
|
34
36
|
|
|
35
37
|
/** Observable gauges for live state. Registered via `observeState(...)` once the
|
|
36
38
|
* CheckpointStore and SessionManager are available. */
|
|
@@ -44,7 +46,6 @@ export class ProverNodeJobMetrics {
|
|
|
44
46
|
public readonly tracer: Tracer,
|
|
45
47
|
private logger = createLogger('prover-node:publisher:metrics'),
|
|
46
48
|
) {
|
|
47
|
-
this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION);
|
|
48
49
|
this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION);
|
|
49
50
|
this.provingJobCheckpoints = this.meter.createGauge(Metrics.PROVER_NODE_JOB_CHECKPOINTS);
|
|
50
51
|
this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS);
|
|
@@ -53,16 +54,12 @@ export class ProverNodeJobMetrics {
|
|
|
53
54
|
this.blobProcessingDuration = this.meter.createGauge(Metrics.PROVER_NODE_BLOB_PROCESSING_LAST_DURATION);
|
|
54
55
|
this.blockProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_BLOCK_PROCESSING_DURATION);
|
|
55
56
|
this.checkpointProcessingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROCESSING_DURATION);
|
|
57
|
+
this.checkpointProvingDuration = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_PROVING_DURATION);
|
|
58
|
+
this.checkpointBlocks = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_BLOCKS);
|
|
59
|
+
this.checkpointTransactions = this.meter.createHistogram(Metrics.PROVER_NODE_CHECKPOINT_TRANSACTIONS);
|
|
56
60
|
}
|
|
57
61
|
|
|
58
|
-
public recordProvingJob(
|
|
59
|
-
executionTimeMs: number,
|
|
60
|
-
totalTimeMs: number,
|
|
61
|
-
numCheckpoints: number,
|
|
62
|
-
numBlocks: number,
|
|
63
|
-
numTxs: number,
|
|
64
|
-
) {
|
|
65
|
-
this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
|
|
62
|
+
public recordProvingJob(totalTimeMs: number, numCheckpoints: number, numBlocks: number, numTxs: number) {
|
|
66
63
|
this.provingJobDuration.record(totalTimeMs / 1000);
|
|
67
64
|
this.provingJobCheckpoints.record(Math.floor(numCheckpoints));
|
|
68
65
|
this.provingJobBlocks.record(Math.floor(numBlocks));
|
|
@@ -77,8 +74,14 @@ export class ProverNodeJobMetrics {
|
|
|
77
74
|
this.blockProcessingDuration.record(Math.ceil(durationMs));
|
|
78
75
|
}
|
|
79
76
|
|
|
80
|
-
public recordCheckpointProcessing(durationMs: number) {
|
|
77
|
+
public recordCheckpointProcessing(durationMs: number, numBlocks: number, numTxs: number) {
|
|
81
78
|
this.checkpointProcessingDuration.record(Math.ceil(durationMs));
|
|
79
|
+
this.checkpointBlocks.record(Math.floor(numBlocks));
|
|
80
|
+
this.checkpointTransactions.record(Math.floor(numTxs));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
public recordCheckpointProving(durationMs: number) {
|
|
84
|
+
this.checkpointProvingDuration.record(Math.ceil(durationMs));
|
|
82
85
|
}
|
|
83
86
|
|
|
84
87
|
/**
|
|
@@ -7,7 +7,7 @@ import { SerialQueue } from '@aztec/foundation/queue';
|
|
|
7
7
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
8
8
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
9
9
|
import type { Proof } from '@aztec/stdlib/proofs';
|
|
10
|
-
import type { RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
10
|
+
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
11
11
|
|
|
12
12
|
import type { ProverNodePublisher } from './prover-node-publisher.js';
|
|
13
13
|
import type { ProverPublisherFactory } from './prover-publisher-factory.js';
|
|
@@ -45,6 +45,8 @@ export type PublishCandidate = {
|
|
|
45
45
|
proof: Proof;
|
|
46
46
|
batchedBlobInputs: BatchedBlob;
|
|
47
47
|
attestations: ViemCommitteeAttestation[];
|
|
48
|
+
/** Committee-attested checkpoint headers for the range, supplying the L1-verified fee recipient/value. */
|
|
49
|
+
headers: CheckpointHeader[];
|
|
48
50
|
};
|
|
49
51
|
|
|
50
52
|
/** Terminal outcome for a candidate. The promise from `submit()` resolves with one of these. */
|
|
@@ -333,6 +335,7 @@ export class ProofPublishingService {
|
|
|
333
335
|
proof: candidate.proof,
|
|
334
336
|
batchedBlobInputs: candidate.batchedBlobInputs,
|
|
335
337
|
attestations: candidate.attestations,
|
|
338
|
+
headers: candidate.headers,
|
|
336
339
|
// Stop the L1 tx retrying past the candidate's submission-window deadline.
|
|
337
340
|
deadline: candidate.deadline,
|
|
338
341
|
};
|
|
@@ -2,19 +2,17 @@ import { BatchedBlob, getEthBlobEvaluationInputs } from '@aztec/blob-lib';
|
|
|
2
2
|
import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants';
|
|
3
3
|
import type { RollupContract, ViemCommitteeAttestation } from '@aztec/ethereum/contracts';
|
|
4
4
|
import type { L1TxUtils } from '@aztec/ethereum/l1-tx-utils';
|
|
5
|
-
import { makeTuple } from '@aztec/foundation/array';
|
|
6
5
|
import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
7
6
|
import { areArraysEqual } from '@aztec/foundation/collection';
|
|
8
7
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
9
8
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
9
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
11
|
-
import type { Tuple } from '@aztec/foundation/serialize';
|
|
12
10
|
import { Timer } from '@aztec/foundation/timer';
|
|
13
11
|
import { RollupAbi } from '@aztec/l1-artifacts';
|
|
14
12
|
import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
|
|
15
13
|
import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
|
|
16
14
|
import type { Proof } from '@aztec/stdlib/proofs';
|
|
17
|
-
import type {
|
|
15
|
+
import type { CheckpointHeader, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
|
|
18
16
|
import type { L1PublishProofStats } from '@aztec/stdlib/stats';
|
|
19
17
|
import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
|
|
20
18
|
|
|
@@ -31,7 +29,7 @@ export type L1SubmitEpochProofArgs = {
|
|
|
31
29
|
endTimestamp: Fr;
|
|
32
30
|
outHash: Fr;
|
|
33
31
|
proverId: Fr;
|
|
34
|
-
|
|
32
|
+
headers: CheckpointHeader[];
|
|
35
33
|
proof: Proof;
|
|
36
34
|
};
|
|
37
35
|
|
|
@@ -78,6 +76,7 @@ export class ProverNodePublisher {
|
|
|
78
76
|
proof: Proof;
|
|
79
77
|
batchedBlobInputs: BatchedBlob;
|
|
80
78
|
attestations: ViemCommitteeAttestation[];
|
|
79
|
+
headers: CheckpointHeader[];
|
|
81
80
|
/** Wall-clock deadline (proof-submission window end) past which the L1 tx should stop retrying. */
|
|
82
81
|
deadline?: Date;
|
|
83
82
|
}): Promise<boolean> {
|
|
@@ -134,6 +133,7 @@ export class ProverNodePublisher {
|
|
|
134
133
|
proof: Proof;
|
|
135
134
|
batchedBlobInputs: BatchedBlob;
|
|
136
135
|
attestations: ViemCommitteeAttestation[];
|
|
136
|
+
headers: CheckpointHeader[];
|
|
137
137
|
}) {
|
|
138
138
|
const { fromCheckpoint, toCheckpoint, publicInputs, batchedBlobInputs } = args;
|
|
139
139
|
|
|
@@ -207,6 +207,7 @@ export class ProverNodePublisher {
|
|
|
207
207
|
proof: Proof;
|
|
208
208
|
batchedBlobInputs: BatchedBlob;
|
|
209
209
|
attestations: ViemCommitteeAttestation[];
|
|
210
|
+
headers: CheckpointHeader[];
|
|
210
211
|
}): Promise<void> {
|
|
211
212
|
const { epochNumber, fromCheckpoint, toCheckpoint } = args;
|
|
212
213
|
|
|
@@ -254,6 +255,7 @@ export class ProverNodePublisher {
|
|
|
254
255
|
proof: Proof;
|
|
255
256
|
batchedBlobInputs: BatchedBlob;
|
|
256
257
|
attestations: ViemCommitteeAttestation[];
|
|
258
|
+
headers: CheckpointHeader[];
|
|
257
259
|
}): Hex {
|
|
258
260
|
return encodeFunctionData({
|
|
259
261
|
abi: RollupAbi,
|
|
@@ -270,6 +272,7 @@ export class ProverNodePublisher {
|
|
|
270
272
|
proof: Proof;
|
|
271
273
|
batchedBlobInputs: BatchedBlob;
|
|
272
274
|
attestations: ViemCommitteeAttestation[];
|
|
275
|
+
headers: CheckpointHeader[];
|
|
273
276
|
}): Promise<TransactionReceipt | undefined> {
|
|
274
277
|
const txArgs = [this.getSubmitEpochProofArgs(args)] as const;
|
|
275
278
|
|
|
@@ -316,6 +319,7 @@ export class ProverNodePublisher {
|
|
|
316
319
|
publicInputs: RootRollupPublicInputs;
|
|
317
320
|
batchedBlobInputs: BatchedBlob;
|
|
318
321
|
attestations: ViemCommitteeAttestation[];
|
|
322
|
+
headers: CheckpointHeader[];
|
|
319
323
|
}) {
|
|
320
324
|
// Returns arguments for EpochProofLib.sol -> getEpochProofPublicInputs()
|
|
321
325
|
return [
|
|
@@ -327,11 +331,7 @@ export class ProverNodePublisher {
|
|
|
327
331
|
outHash: args.publicInputs.outHash.toString(),
|
|
328
332
|
proverId: EthAddress.fromField(args.publicInputs.constants.proverId).toString(),
|
|
329
333
|
} /*_args*/,
|
|
330
|
-
|
|
331
|
-
i % 2 === 0
|
|
332
|
-
? args.publicInputs.fees[i / 2].recipient.toField().toString()
|
|
333
|
-
: args.publicInputs.fees[(i - 1) / 2].value.toString(),
|
|
334
|
-
) /*_fees*/,
|
|
334
|
+
args.headers.map(header => header.toViem()) /*_headers*/,
|
|
335
335
|
getEthBlobEvaluationInputs(args.batchedBlobInputs) /*_blobPublicInputs*/,
|
|
336
336
|
] as const;
|
|
337
337
|
}
|
|
@@ -343,6 +343,7 @@ export class ProverNodePublisher {
|
|
|
343
343
|
proof: Proof;
|
|
344
344
|
batchedBlobInputs: BatchedBlob;
|
|
345
345
|
attestations: ViemCommitteeAttestation[];
|
|
346
|
+
headers: CheckpointHeader[];
|
|
346
347
|
}) {
|
|
347
348
|
// Returns arguments for EpochProofLib.sol -> submitEpochRootProof()
|
|
348
349
|
const proofHex: Hex = `0x${args.proof.withoutPublicInputs().toString('hex')}`;
|
|
@@ -351,7 +352,7 @@ export class ProverNodePublisher {
|
|
|
351
352
|
start: argsArray[0],
|
|
352
353
|
end: argsArray[1],
|
|
353
354
|
args: argsArray[2],
|
|
354
|
-
|
|
355
|
+
headers: argsArray[3],
|
|
355
356
|
attestations: CommitteeAttestationsAndSigners.packAttestations(
|
|
356
357
|
args.attestations.map(a => CommitteeAttestation.fromViem(a)),
|
|
357
358
|
),
|
package/src/prover-node.ts
CHANGED
|
@@ -5,12 +5,14 @@ import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/br
|
|
|
5
5
|
import { assertRequired, compact, pick } from '@aztec/foundation/collection';
|
|
6
6
|
import { memoize } from '@aztec/foundation/decorators';
|
|
7
7
|
import { createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
8
9
|
import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
|
|
9
10
|
import type { EpochProverFactory } from '@aztec/prover-client';
|
|
10
11
|
import { getLastSiblingPath } from '@aztec/prover-client/helpers';
|
|
11
12
|
import { ChonkCache } from '@aztec/prover-client/orchestrator';
|
|
12
13
|
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
13
14
|
import {
|
|
15
|
+
type L2BlockId,
|
|
14
16
|
type L2BlockSource,
|
|
15
17
|
L2BlockStream,
|
|
16
18
|
type L2BlockStreamEvent,
|
|
@@ -28,7 +30,6 @@ import {
|
|
|
28
30
|
type ITxProvider,
|
|
29
31
|
type ProverNodeApi,
|
|
30
32
|
type Service,
|
|
31
|
-
type WorldStateSyncStatus,
|
|
32
33
|
type WorldStateSynchronizer,
|
|
33
34
|
tryStop,
|
|
34
35
|
} from '@aztec/stdlib/interfaces/server';
|
|
@@ -95,6 +96,17 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
95
96
|
*/
|
|
96
97
|
protected lastExpiredEpoch: EpochNumber | undefined;
|
|
97
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Highest checkpoint number whose proving-side handling has completed (or that was legitimately skipped).
|
|
101
|
+
* The catch-up loop walks from here to each `chain-checkpointed` tip event. Seeded at start() from the last
|
|
102
|
+
* checkpoint of the last fully-proven epoch (or 0), so a restart reprocesses the partially-proven epoch rather
|
|
103
|
+
* than trusting a checkpointed tip that may sit ahead of unproven checkpoints. Clamped down on a prune.
|
|
104
|
+
*/
|
|
105
|
+
protected lastProcessedCheckpoint: CheckpointNumber = CheckpointNumber.ZERO;
|
|
106
|
+
|
|
107
|
+
/** Periodic tick that runs the epoch-expiry sweep during idle periods when no block-stream events arrive. */
|
|
108
|
+
private expiryTicker: RunningPromise | undefined;
|
|
109
|
+
|
|
98
110
|
public readonly tracer: Tracer;
|
|
99
111
|
|
|
100
112
|
protected publishingService: ProofPublishingService | undefined;
|
|
@@ -198,17 +210,6 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
198
210
|
return this.sessionManager;
|
|
199
211
|
}
|
|
200
212
|
|
|
201
|
-
/** Returns world state status. */
|
|
202
|
-
public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
|
|
203
|
-
const { syncSummary } = await this.worldState.status();
|
|
204
|
-
return syncSummary;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/** Returns archiver status. */
|
|
208
|
-
public getL2Tips() {
|
|
209
|
-
return this.l2BlockSource.getL2Tips();
|
|
210
|
-
}
|
|
211
|
-
|
|
212
213
|
/** Returns the underlying prover instance. */
|
|
213
214
|
public getProver() {
|
|
214
215
|
return this.prover;
|
|
@@ -219,17 +220,25 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
219
220
|
public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
|
|
220
221
|
switch (event.type) {
|
|
221
222
|
case 'chain-checkpointed':
|
|
222
|
-
await this.
|
|
223
|
+
await this.processCheckpointJump(event.checkpoint.number);
|
|
223
224
|
break;
|
|
224
225
|
case 'chain-pruned':
|
|
225
|
-
await this.handlePruneEvent(event.
|
|
226
|
+
await this.handlePruneEvent(event.block);
|
|
226
227
|
break;
|
|
227
228
|
case 'chain-proven':
|
|
228
229
|
this.publishingService?.onChainProven(BlockNumber(event.block.number));
|
|
229
230
|
break;
|
|
231
|
+
// The proposed tip drives only the tips store's walk-back history (recorded below); the prover-node
|
|
232
|
+
// tracks checkpoints, not proposed blocks. `blocks-added` is never emitted in tips-only mode, and
|
|
233
|
+
// `chain-finalized` carries nothing the prover-node acts on.
|
|
234
|
+
case 'chain-proposed':
|
|
230
235
|
case 'chain-finalized':
|
|
231
236
|
case 'blocks-added':
|
|
232
237
|
break;
|
|
238
|
+
default: {
|
|
239
|
+
const _: never = event;
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
233
242
|
}
|
|
234
243
|
// Expiry is driven by the archiver's latest synced L2 slot
|
|
235
244
|
await this.checkEpochExpiry();
|
|
@@ -239,34 +248,92 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
239
248
|
await this.tipsStore.handleBlockStreamEvent(event);
|
|
240
249
|
}
|
|
241
250
|
|
|
242
|
-
/**
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
+
/**
|
|
252
|
+
* Walks every checkpoint between the local cursor and the newly-reported checkpointed tip, registering
|
|
253
|
+
* each one that belongs to an epoch that can still be proven. The block stream now delivers a single thin
|
|
254
|
+
* `chain-checkpointed` tip event per pass rather than one fat event per checkpoint, so this drives the
|
|
255
|
+
* catch-up itself: light metadata first (`getCheckpointsData`) to decide relevance per epoch, then a heavy
|
|
256
|
+
* `getCheckpoints` fetch only for checkpoints in provable epochs.
|
|
257
|
+
*
|
|
258
|
+
* The cursor advances one checkpoint at a time and only after that checkpoint's proving-side handling has
|
|
259
|
+
* fully succeeded, preserving the A-1041 at-least-once semantics: a mid-jump failure leaves the cursor
|
|
260
|
+
* behind so the next pass retries from the first checkpoint that did not complete.
|
|
261
|
+
*/
|
|
262
|
+
private async processCheckpointJump(targetCheckpoint: CheckpointNumber): Promise<void> {
|
|
263
|
+
if (targetCheckpoint <= this.lastProcessedCheckpoint) {
|
|
251
264
|
return;
|
|
252
265
|
}
|
|
266
|
+
const l1Constants = await this.getL1Constants();
|
|
253
267
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
268
|
+
// Cap the catch-up at the `(proofSubmissionEpochs + 1) * epochDuration` most recent checkpoints.
|
|
269
|
+
// When the cursor is much further behind (e.g. resyncing after a long time offline), fetching the whole gap could
|
|
270
|
+
// load thousands of checkpoints we cannot act on: anything older than the last two epochs is already past
|
|
271
|
+
// its proof-submission window, so we skip it and jump the cursor forward to the start of the capped range.
|
|
272
|
+
const maxCheckpoints = (l1Constants.proofSubmissionEpochs + 1) * l1Constants.epochDuration;
|
|
273
|
+
let from = CheckpointNumber(this.lastProcessedCheckpoint + 1);
|
|
274
|
+
if (Number(targetCheckpoint - from) + 1 > maxCheckpoints) {
|
|
275
|
+
const cappedFrom = CheckpointNumber(targetCheckpoint - maxCheckpoints + 1);
|
|
276
|
+
this.log.warn(`Skipping unprovable checkpoints during catch-up; the prover node is far behind`, {
|
|
277
|
+
from,
|
|
278
|
+
cappedFrom,
|
|
279
|
+
targetCheckpoint,
|
|
280
|
+
maxCheckpoints,
|
|
281
|
+
});
|
|
282
|
+
// Advance the cursor past the skipped checkpoints so they are never retried.
|
|
283
|
+
this.lastProcessedCheckpoint = CheckpointNumber(cappedFrom - 1);
|
|
284
|
+
from = cappedFrom;
|
|
259
285
|
}
|
|
286
|
+
const limit = Number(targetCheckpoint - from) + 1;
|
|
287
|
+
const metadatas = await this.l2BlockSource.getCheckpointsData({ from, limit });
|
|
288
|
+
|
|
289
|
+
// Per-epoch relevance is cached so a multi-checkpoint epoch resolves it once. Skipping is whole-epoch
|
|
290
|
+
// only: the SessionManager requires an epoch's checkpoints fully covered before it opens a session, so we
|
|
291
|
+
// never drop an individual checkpoint inside an epoch we will prove.
|
|
292
|
+
const epochSkippable = new Map<EpochNumber, boolean>();
|
|
293
|
+
for (const metadata of metadatas) {
|
|
294
|
+
const epochNumber = getEpochAtSlot(metadata.header.slotNumber, l1Constants);
|
|
295
|
+
let skippable = epochSkippable.get(epochNumber);
|
|
296
|
+
if (skippable === undefined) {
|
|
297
|
+
skippable =
|
|
298
|
+
(await this.isEpochFullyProven(epochNumber, l1Constants)) ||
|
|
299
|
+
(await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants));
|
|
300
|
+
epochSkippable.set(epochNumber, skippable);
|
|
301
|
+
}
|
|
302
|
+
if (skippable) {
|
|
303
|
+
this.log.debug(`Skipping checkpoint ${metadata.checkpointNumber} for unprovable epoch ${epochNumber}`);
|
|
304
|
+
} else {
|
|
305
|
+
await this.registerCheckpoint(metadata.checkpointNumber, epochNumber);
|
|
306
|
+
}
|
|
307
|
+
// Advance only after the checkpoint's handling succeeded (or it was legitimately skipped). registerCheckpoint
|
|
308
|
+
// throws on failure, which leaves the cursor here for the next pass to retry (A-1041).
|
|
309
|
+
this.lastProcessedCheckpoint = metadata.checkpointNumber;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
260
312
|
|
|
313
|
+
/** Heavy-fetch a single checkpoint, register it with the store, and notify the session manager. */
|
|
314
|
+
private async registerCheckpoint(checkpointNumber: CheckpointNumber, epochNumber: EpochNumber): Promise<void> {
|
|
315
|
+
const published = await this.l2BlockSource.getCheckpoint({ number: checkpointNumber });
|
|
316
|
+
if (!published) {
|
|
317
|
+
throw new Error(`Checkpoint ${checkpointNumber} not found in block source during catch-up`);
|
|
318
|
+
}
|
|
319
|
+
const checkpoint = published.checkpoint;
|
|
261
320
|
this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
|
|
262
321
|
checkpointNumber: checkpoint.number,
|
|
263
322
|
epochNumber,
|
|
264
|
-
slotNumber,
|
|
323
|
+
slotNumber: checkpoint.header.slotNumber,
|
|
265
324
|
});
|
|
266
325
|
|
|
267
|
-
const registerData = await this.collectRegisterData(checkpoint,
|
|
326
|
+
const registerData = await this.collectRegisterData(checkpoint, published.attestations);
|
|
268
327
|
await this.checkpointStore.addOrUpdate(checkpoint, registerData);
|
|
269
328
|
await this.sessionManager?.onCheckpointAdded(epochNumber);
|
|
329
|
+
|
|
330
|
+
// Tips-only mode delivers no blocks, so record one witness per checkpointed block: a reorg into the checkpoint's
|
|
331
|
+
// range then prunes at the true divergence instead of the nearest sparse tip anchor.
|
|
332
|
+
await this.tipsStore.recordBlockHashes(
|
|
333
|
+
await Promise.all(
|
|
334
|
+
checkpoint.blocks.map(async block => ({ number: block.number, hash: (await block.header.hash()).toString() })),
|
|
335
|
+
),
|
|
336
|
+
);
|
|
270
337
|
}
|
|
271
338
|
|
|
272
339
|
/**
|
|
@@ -295,10 +362,41 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
295
362
|
};
|
|
296
363
|
}
|
|
297
364
|
|
|
298
|
-
/**
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
365
|
+
/**
|
|
366
|
+
* Marks every prover orphaned by the prune as pruned, clamps the catch-up cursor below the prune target's
|
|
367
|
+
* checkpoint, and notifies the session manager. Keyed off the prune target block (the highest surviving block)
|
|
368
|
+
* rather than the source's checkpointed tip, which can sit above the target after a re-checkpoint and would leave
|
|
369
|
+
* orphaned provers canonical. Throws (rather than warning) if the cursor floor cannot be resolved, so the pass
|
|
370
|
+
* fails and the prune is retried next iteration.
|
|
371
|
+
*/
|
|
372
|
+
private async handlePruneEvent(prunedToBlock: L2BlockId) {
|
|
373
|
+
this.log.warn(`Chain pruned to block ${prunedToBlock.number}`, { prunedToBlock });
|
|
374
|
+
|
|
375
|
+
// Resolve the cursor floor BEFORE marking provers: markPrunedAboveBlock returns only newly-marked provers, so a
|
|
376
|
+
// throw after marking would leave a retry pass with nothing to act on. Resolving first means a throw leaves
|
|
377
|
+
// everything untouched and the next pass retries the whole handler (the tips cursor only advances on success).
|
|
378
|
+
let cursorFloor: CheckpointNumber;
|
|
379
|
+
if (prunedToBlock.number === 0) {
|
|
380
|
+
cursorFloor = CheckpointNumber.ZERO;
|
|
381
|
+
} else {
|
|
382
|
+
const targetData = await this.l2BlockSource.getBlockData({ number: prunedToBlock.number });
|
|
383
|
+
if (targetData === undefined) {
|
|
384
|
+
throw new Error(
|
|
385
|
+
`No block data found for prune target block ${prunedToBlock.number}; cannot clamp checkpoint cursor`,
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
// Clamp to `cpAtTarget - 1`: a mid-checkpoint target leaves that checkpoint partially orphaned and it must be
|
|
389
|
+
// reprocessed. Over-clamping merely re-registers a checkpoint (at-least-once by design — A-1041); under-clamping
|
|
390
|
+
// would permanently skip a rebuilt same-number checkpoint.
|
|
391
|
+
cursorFloor = CheckpointNumber(Math.max(0, Number(targetData.checkpointNumber) - 1));
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const affected = this.checkpointStore.markPrunedAboveBlock(prunedToBlock.number);
|
|
395
|
+
|
|
396
|
+
if (this.lastProcessedCheckpoint > cursorFloor) {
|
|
397
|
+
this.lastProcessedCheckpoint = cursorFloor;
|
|
398
|
+
}
|
|
399
|
+
|
|
302
400
|
if (affected.length === 0) {
|
|
303
401
|
return;
|
|
304
402
|
}
|
|
@@ -409,14 +507,24 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
409
507
|
// Now that the store + manager exist, arm the live-state observable gauges.
|
|
410
508
|
this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
|
|
411
509
|
|
|
412
|
-
const {
|
|
510
|
+
const { lastFullyProvenEpoch } = await this.resolveLastFullyProvenEpoch();
|
|
413
511
|
this.lastExpiredEpoch = lastFullyProvenEpoch;
|
|
512
|
+
this.lastProcessedCheckpoint = await this.computeStartingCheckpoint(lastFullyProvenEpoch);
|
|
414
513
|
this.blockStream = new L2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
|
|
415
514
|
pollIntervalMS: this.config.proverNodePollingIntervalMs,
|
|
416
|
-
|
|
515
|
+
tipsOnly: true,
|
|
417
516
|
});
|
|
418
517
|
this.blockStream.start();
|
|
419
518
|
|
|
519
|
+
// With thin once-per-pass tip events, the expiry sweep no longer fires once per checkpoint; drive it
|
|
520
|
+
// from a periodic tick so epochs still expire during idle/no-event periods.
|
|
521
|
+
this.expiryTicker = new RunningPromise(
|
|
522
|
+
() => this.checkEpochExpiry(),
|
|
523
|
+
this.log,
|
|
524
|
+
this.config.proverNodePollingIntervalMs,
|
|
525
|
+
);
|
|
526
|
+
this.expiryTicker.start();
|
|
527
|
+
|
|
420
528
|
await this.rewardsMetrics.start();
|
|
421
529
|
this.l1Metrics.start();
|
|
422
530
|
this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
|
|
@@ -426,6 +534,7 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
426
534
|
this.log.info('Stopping ProverNode');
|
|
427
535
|
this.jobMetrics.stopObservingState();
|
|
428
536
|
await this.blockStream?.stop();
|
|
537
|
+
await this.expiryTicker?.stop();
|
|
429
538
|
if (this.sessionManager) {
|
|
430
539
|
await this.sessionManager.stop();
|
|
431
540
|
}
|
|
@@ -552,38 +661,40 @@ export class ProverNode implements L2BlockStreamEventHandler, ProverNodeApi, Tra
|
|
|
552
661
|
}
|
|
553
662
|
|
|
554
663
|
/**
|
|
555
|
-
* Resolves the
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
*
|
|
559
|
-
* `provenEpoch - 1`, or `undefined` if no block is proven yet.
|
|
664
|
+
* Resolves the last fully-proven epoch from L1 proven state, used to seed the catch-up cursor (via
|
|
665
|
+
* `computeStartingCheckpoint`) and `lastExpiredEpoch`. The fully-proven epoch is `provenEpoch` when the
|
|
666
|
+
* proven tip is the last block of its epoch, otherwise `provenEpoch - 1`, or `undefined` if no block is
|
|
667
|
+
* proven yet (so a restart reprocesses the partially-proven epoch rather than trusting a stale tip).
|
|
560
668
|
*/
|
|
561
|
-
protected async
|
|
562
|
-
startingBlock: BlockNumber;
|
|
563
|
-
lastFullyProvenEpoch: EpochNumber | undefined;
|
|
564
|
-
}> {
|
|
669
|
+
protected async resolveLastFullyProvenEpoch(): Promise<{ lastFullyProvenEpoch: EpochNumber | undefined }> {
|
|
565
670
|
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({ tag: 'proven' });
|
|
566
671
|
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
567
|
-
return {
|
|
672
|
+
return { lastFullyProvenEpoch: undefined };
|
|
568
673
|
}
|
|
569
674
|
const l1Constants = await this.getL1Constants();
|
|
570
675
|
const provenHeader = (await this.l2BlockSource.getBlockData({ number: BlockNumber(provenBlockNumber) }))?.header;
|
|
571
676
|
if (!provenHeader) {
|
|
572
|
-
return {
|
|
677
|
+
return { lastFullyProvenEpoch: undefined };
|
|
573
678
|
}
|
|
574
679
|
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
575
680
|
if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
|
|
576
|
-
return {
|
|
681
|
+
return { lastFullyProvenEpoch: provenEpoch };
|
|
577
682
|
}
|
|
578
|
-
const epochCheckpoints = await this.l2BlockSource.getCheckpointsData({ epoch: provenEpoch });
|
|
579
|
-
const firstBlockOfEpoch =
|
|
580
|
-
epochCheckpoints.length > 0 ? epochCheckpoints[0].startBlock : BlockNumber(provenBlockNumber);
|
|
581
|
-
this.log.info(
|
|
582
|
-
`Starting L2BlockStream at block ${firstBlockOfEpoch} (start of partially-proven epoch ${provenEpoch})`,
|
|
583
|
-
{ provenBlockNumber, provenEpoch, firstBlockOfEpoch },
|
|
584
|
-
);
|
|
585
683
|
const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
|
|
586
|
-
return {
|
|
684
|
+
return { lastFullyProvenEpoch };
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Resolves the catch-up cursor seed: the last checkpoint of the last fully-proven epoch, or 0 if none. Seeding
|
|
689
|
+
* from a checkpoint (rather than a checkpointed tip) guarantees a restart reprocesses every checkpoint of the
|
|
690
|
+
* partially-proven epoch, since the checkpointed tip can sit ahead of the last fully-proven checkpoint.
|
|
691
|
+
*/
|
|
692
|
+
protected async computeStartingCheckpoint(lastFullyProvenEpoch: EpochNumber | undefined): Promise<CheckpointNumber> {
|
|
693
|
+
if (lastFullyProvenEpoch === undefined) {
|
|
694
|
+
return CheckpointNumber.ZERO;
|
|
695
|
+
}
|
|
696
|
+
const checkpoints = await this.l2BlockSource.getCheckpointsData({ epoch: lastFullyProvenEpoch });
|
|
697
|
+
return checkpoints.at(-1)?.checkpointNumber ?? CheckpointNumber.ZERO;
|
|
587
698
|
}
|
|
588
699
|
|
|
589
700
|
private async gatherPreviousBlockHeader(previousBlockNumber: number) {
|