@aztec/prover-node 5.0.0-private.20260319 → 5.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +506 -0
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +103 -21
- package/dest/bin/run-failed-epoch.js +1 -3
- package/dest/checkpoint-store.d.ts +83 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +181 -0
- package/dest/config.d.ts +1 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +1 -1
- package/dest/factory.d.ts +1 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +22 -8
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +134 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +350 -0
- package/dest/job/epoch-session.d.ts +146 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/epoch-session.js +709 -0
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +29 -5
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +73 -9
- package/dest/monitors/epoch-monitor.js +6 -2
- package/dest/proof-publishing-service.d.ts +159 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +334 -0
- package/dest/prover-node-publisher.d.ts +18 -11
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +195 -57
- package/dest/prover-node.d.ts +96 -68
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +382 -227
- package/dest/prover-publisher-factory.d.ts +2 -2
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +3 -3
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +452 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +23 -23
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +114 -28
- package/src/bin/run-failed-epoch.ts +1 -2
- package/src/checkpoint-store.ts +213 -0
- package/src/config.ts +2 -1
- package/src/factory.ts +18 -10
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +465 -0
- package/src/job/epoch-session.ts +424 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +88 -12
- package/src/monitors/epoch-monitor.ts +2 -2
- package/src/proof-publishing-service.ts +424 -0
- package/src/prover-node-publisher.ts +220 -67
- package/src/prover-node.ts +439 -249
- package/src/prover-publisher-factory.ts +3 -3
- package/src/session-manager.ts +552 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -63
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/dest/job/epoch-proving-job.js +0 -762
- package/src/job/epoch-proving-job.ts +0 -465
package/dest/prover-node.js
CHANGED
|
@@ -370,28 +370,39 @@ function applyDecs2203RFactory() {
|
|
|
370
370
|
function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
|
|
371
371
|
return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
|
|
372
372
|
}
|
|
373
|
-
var
|
|
374
|
-
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
375
|
-
import { assertRequired, compact, pick
|
|
373
|
+
var _initProto;
|
|
374
|
+
import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
|
|
375
|
+
import { assertRequired, compact, pick } from '@aztec/foundation/collection';
|
|
376
376
|
import { memoize } from '@aztec/foundation/decorators';
|
|
377
377
|
import { createLogger } from '@aztec/foundation/log';
|
|
378
|
-
import { DateProvider } from '@aztec/foundation/timer';
|
|
378
|
+
import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
|
|
379
|
+
import { getLastSiblingPath } from '@aztec/prover-client/helpers';
|
|
380
|
+
import { ChonkCache } from '@aztec/prover-client/orchestrator';
|
|
379
381
|
import { PublicProcessorFactory } from '@aztec/simulator/server';
|
|
380
|
-
import {
|
|
382
|
+
import { L2BlockStream, L2TipsMemoryStore } from '@aztec/stdlib/block';
|
|
383
|
+
import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
|
|
381
384
|
import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
|
|
382
|
-
import {
|
|
385
|
+
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
386
|
+
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
383
387
|
import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
|
|
384
|
-
import {
|
|
388
|
+
import { CheckpointStore } from './checkpoint-store.js';
|
|
385
389
|
import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
})), _dec1 = trackSpan('ProverNode.gatherEpochData', (epochNumber)=>({
|
|
389
|
-
[Attributes.EPOCH_NUMBER]: epochNumber
|
|
390
|
-
}));
|
|
390
|
+
import { ProofPublishingService } from './proof-publishing-service.js';
|
|
391
|
+
import { SessionManager } from './session-manager.js';
|
|
391
392
|
/**
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
*
|
|
393
|
+
* Grace period for the proof-publishing service to settle during shutdown. The service waits for
|
|
394
|
+
* any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
|
|
395
|
+
* the wait rather than letting `stop()` hang indefinitely.
|
|
396
|
+
*/ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
|
|
397
|
+
/**
|
|
398
|
+
* An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
|
|
399
|
+
* starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
|
|
400
|
+
* complete.
|
|
401
|
+
*
|
|
402
|
+
* The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
|
|
403
|
+
* `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
|
|
404
|
+
* manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
|
|
405
|
+
* the `SessionManager`. Each chain event is translated here into a single method call on it.
|
|
395
406
|
*/ export class ProverNode {
|
|
396
407
|
prover;
|
|
397
408
|
publisherFactory;
|
|
@@ -400,7 +411,6 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
400
411
|
contractDataSource;
|
|
401
412
|
worldState;
|
|
402
413
|
p2pClient;
|
|
403
|
-
epochsMonitor;
|
|
404
414
|
rollupContract;
|
|
405
415
|
l1Metrics;
|
|
406
416
|
telemetryClient;
|
|
@@ -408,31 +418,31 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
408
418
|
dateProvider;
|
|
409
419
|
static{
|
|
410
420
|
({ e: [_initProto] } = _apply_decs_2203_r(this, [
|
|
411
|
-
[
|
|
412
|
-
_dec,
|
|
413
|
-
2,
|
|
414
|
-
"createProvingJob"
|
|
415
|
-
],
|
|
416
421
|
[
|
|
417
422
|
memoize,
|
|
418
423
|
2,
|
|
419
424
|
"getL1Constants"
|
|
420
|
-
],
|
|
421
|
-
[
|
|
422
|
-
_dec1,
|
|
423
|
-
2,
|
|
424
|
-
"gatherEpochData"
|
|
425
425
|
]
|
|
426
426
|
], []));
|
|
427
427
|
}
|
|
428
428
|
log;
|
|
429
|
-
|
|
429
|
+
checkpointStore;
|
|
430
|
+
chonkCache;
|
|
431
|
+
sessionManager;
|
|
430
432
|
config;
|
|
431
433
|
jobMetrics;
|
|
432
434
|
rewardsMetrics;
|
|
435
|
+
/** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
|
|
436
|
+
/** Block stream for checkpoint and reorg detection. */ blockStream;
|
|
437
|
+
/**
|
|
438
|
+
* Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
|
|
439
|
+
* Seeded from the last fully-proven epoch at start(); advanced on every block-stream
|
|
440
|
+
* event by comparing the archiver's latest synced L2 slot against each epoch's
|
|
441
|
+
* submission deadline. Protected so tests can verify the start() seeding.
|
|
442
|
+
*/ lastExpiredEpoch;
|
|
433
443
|
tracer;
|
|
434
|
-
|
|
435
|
-
constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient,
|
|
444
|
+
publishingService;
|
|
445
|
+
constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
|
|
436
446
|
this.prover = prover;
|
|
437
447
|
this.publisherFactory = publisherFactory;
|
|
438
448
|
this.l2BlockSource = l2BlockSource;
|
|
@@ -440,14 +450,12 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
440
450
|
this.contractDataSource = contractDataSource;
|
|
441
451
|
this.worldState = worldState;
|
|
442
452
|
this.p2pClient = p2pClient;
|
|
443
|
-
this.epochsMonitor = epochsMonitor;
|
|
444
453
|
this.rollupContract = rollupContract;
|
|
445
454
|
this.l1Metrics = l1Metrics;
|
|
446
455
|
this.telemetryClient = telemetryClient;
|
|
447
456
|
this.delayer = delayer;
|
|
448
457
|
this.dateProvider = dateProvider;
|
|
449
458
|
this.log = (_initProto(this), createLogger('prover-node'));
|
|
450
|
-
this.jobs = new Map();
|
|
451
459
|
this.config = {
|
|
452
460
|
proverNodePollingIntervalMs: 1_000,
|
|
453
461
|
proverNodeMaxPendingJobs: 100,
|
|
@@ -465,6 +473,22 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
465
473
|
this.tracer = telemetryClient.getTracer('ProverNode');
|
|
466
474
|
this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
|
|
467
475
|
this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
|
|
476
|
+
this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
|
|
477
|
+
this.chonkCache = new ChonkCache(this.log.getBindings());
|
|
478
|
+
this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
|
|
479
|
+
proverFactory: this.prover,
|
|
480
|
+
chonkCache: this.chonkCache,
|
|
481
|
+
publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings()),
|
|
482
|
+
dbProvider: this.worldState,
|
|
483
|
+
txProvider: this.p2pClient.getTxProvider(),
|
|
484
|
+
dateProvider: this.dateProvider,
|
|
485
|
+
proverId: this.prover.getProverId(),
|
|
486
|
+
metrics: this.jobMetrics,
|
|
487
|
+
txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
|
|
488
|
+
deadline: undefined
|
|
489
|
+
}, {
|
|
490
|
+
slotWatcherPollIntervalMs: this.config.proverNodePollingIntervalMs
|
|
491
|
+
}, this.log.getBindings());
|
|
468
492
|
}
|
|
469
493
|
getProverId() {
|
|
470
494
|
return this.prover.getProverId();
|
|
@@ -472,233 +496,368 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
472
496
|
getP2P() {
|
|
473
497
|
return this.p2pClient;
|
|
474
498
|
}
|
|
475
|
-
/**
|
|
499
|
+
/** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
|
|
476
500
|
return this.delayer;
|
|
477
501
|
}
|
|
502
|
+
/** Observability summary for the ProverNodeApi. */ getJobs() {
|
|
503
|
+
return Promise.resolve(this.sessionManager?.getJobs() ?? []);
|
|
504
|
+
}
|
|
505
|
+
/** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
|
|
506
|
+
return this.checkpointStore;
|
|
507
|
+
}
|
|
508
|
+
/** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
|
|
509
|
+
return this.chonkCache;
|
|
510
|
+
}
|
|
511
|
+
/** Tests inspect this when looking up live sessions. */ getSessionManager() {
|
|
512
|
+
if (!this.sessionManager) {
|
|
513
|
+
throw new Error('SessionManager not yet constructed — start() must be called first.');
|
|
514
|
+
}
|
|
515
|
+
return this.sessionManager;
|
|
516
|
+
}
|
|
517
|
+
/** Returns world state status. */ async getWorldStateSyncStatus() {
|
|
518
|
+
const { syncSummary } = await this.worldState.status();
|
|
519
|
+
return syncSummary;
|
|
520
|
+
}
|
|
521
|
+
/** Returns archiver status. */ getL2Tips() {
|
|
522
|
+
return this.l2BlockSource.getL2Tips();
|
|
523
|
+
}
|
|
524
|
+
/** Returns the underlying prover instance. */ getProver() {
|
|
525
|
+
return this.prover;
|
|
526
|
+
}
|
|
527
|
+
// ---------------- L2BlockStream handler ----------------
|
|
528
|
+
async handleBlockStreamEvent(event) {
|
|
529
|
+
switch(event.type){
|
|
530
|
+
case 'chain-checkpointed':
|
|
531
|
+
await this.handleCheckpointEvent(event.checkpoint);
|
|
532
|
+
break;
|
|
533
|
+
case 'chain-pruned':
|
|
534
|
+
await this.handlePruneEvent(event.checkpointed.checkpoint);
|
|
535
|
+
break;
|
|
536
|
+
case 'chain-proven':
|
|
537
|
+
this.publishingService?.onChainProven(BlockNumber(event.block.number));
|
|
538
|
+
break;
|
|
539
|
+
case 'chain-finalized':
|
|
540
|
+
case 'blocks-added':
|
|
541
|
+
break;
|
|
542
|
+
}
|
|
543
|
+
// Expiry is driven by the archiver's latest synced L2 slot
|
|
544
|
+
await this.checkEpochExpiry();
|
|
545
|
+
// Advance the local tips store only after the proving-side handling has succeeded. Any
|
|
546
|
+
// failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
|
|
547
|
+
// skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
|
|
548
|
+
await this.tipsStore.handleBlockStreamEvent(event);
|
|
549
|
+
}
|
|
550
|
+
/** Register a new checkpoint with the store and notify the session manager. */ async handleCheckpointEvent(publishedCheckpoint) {
|
|
551
|
+
const checkpoint = publishedCheckpoint.checkpoint;
|
|
552
|
+
const slotNumber = checkpoint.header.slotNumber;
|
|
553
|
+
const l1Constants = await this.getL1Constants();
|
|
554
|
+
const epochNumber = getEpochAtSlot(slotNumber, l1Constants);
|
|
555
|
+
if (await this.isEpochFullyProven(epochNumber, l1Constants)) {
|
|
556
|
+
this.log.debug(`Skipping checkpoint ${checkpoint.number} for already-proven epoch ${epochNumber}`);
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
if (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants)) {
|
|
560
|
+
this.log.debug(`Skipping checkpoint ${checkpoint.number} for epoch ${epochNumber} past its proof-submission window`);
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
|
|
564
|
+
checkpointNumber: checkpoint.number,
|
|
565
|
+
epochNumber,
|
|
566
|
+
slotNumber
|
|
567
|
+
});
|
|
568
|
+
const registerData = await this.collectRegisterData(checkpoint, publishedCheckpoint.attestations);
|
|
569
|
+
await this.checkpointStore.addOrUpdate(checkpoint, registerData);
|
|
570
|
+
await this.sessionManager?.onCheckpointAdded(epochNumber);
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
|
|
574
|
+
* and the archive sibling path.
|
|
575
|
+
*/ async collectRegisterData(checkpoint, attestations) {
|
|
576
|
+
const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
|
|
577
|
+
const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
|
|
578
|
+
const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
|
|
579
|
+
const lastBlock = checkpoint.blocks.at(-1);
|
|
580
|
+
const lastBlockHash = await lastBlock.header.hash();
|
|
581
|
+
await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
|
|
582
|
+
const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
|
|
583
|
+
return {
|
|
584
|
+
attestations,
|
|
585
|
+
previousBlockHeader,
|
|
586
|
+
l1ToL2Messages,
|
|
587
|
+
previousArchiveSiblingPath
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
/** Mark every prover above the prune threshold as pruned and notify the session manager. */ async handlePruneEvent(prunedCheckpoint) {
|
|
591
|
+
this.log.warn(`Chain pruned to checkpoint ${prunedCheckpoint.number}`, {
|
|
592
|
+
prunedCheckpoint
|
|
593
|
+
});
|
|
594
|
+
const affected = this.checkpointStore.markPrunedAfter(prunedCheckpoint.number);
|
|
595
|
+
if (affected.length === 0) {
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
const l1Constants = await this.getL1Constants();
|
|
599
|
+
const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
|
|
600
|
+
// The session manager cancels every affected session, which in turn calls
|
|
601
|
+
// publishingService.withdraw(uuid) for each candidate; no separate notification to the
|
|
602
|
+
// publishing service is needed.
|
|
603
|
+
await this.sessionManager?.onPrune(affectedEpochs);
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Returns true once the chain has advanced past the given epoch's proof-submission window.
|
|
607
|
+
* Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
|
|
608
|
+
* archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
|
|
609
|
+
* against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
|
|
610
|
+
* be read yet.
|
|
611
|
+
*/ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
|
|
612
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
613
|
+
if (latestSlot === undefined) {
|
|
614
|
+
return false;
|
|
615
|
+
}
|
|
616
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
617
|
+
return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
|
|
618
|
+
}
|
|
478
619
|
/**
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
*
|
|
482
|
-
|
|
620
|
+
* Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
|
|
621
|
+
* newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
|
|
622
|
+
* CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
|
|
623
|
+
* of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
|
|
624
|
+
* the last check or the archiver's slot can't be read.
|
|
625
|
+
*/ async checkEpochExpiry() {
|
|
626
|
+
const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
627
|
+
if (latestSlot === undefined) {
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
const l1Constants = await this.getL1Constants();
|
|
631
|
+
const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
|
|
632
|
+
const offset = l1Constants.proofSubmissionEpochs + 1;
|
|
633
|
+
if (latestEpoch < offset) {
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
|
|
637
|
+
const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
|
|
638
|
+
if (newlyExpiredUpTo < from) {
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
|
|
642
|
+
await this.expireEpoch(e);
|
|
643
|
+
}
|
|
644
|
+
this.lastExpiredEpoch = newlyExpiredUpTo;
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
|
|
648
|
+
* reaps every CheckpointProver in the store whose epoch number matches.
|
|
649
|
+
*/ async expireEpoch(epoch) {
|
|
483
650
|
try {
|
|
484
|
-
this.
|
|
485
|
-
|
|
651
|
+
const blocks = await this.l2BlockSource.getBlocks({
|
|
652
|
+
epoch,
|
|
653
|
+
onlyCheckpointed: true
|
|
486
654
|
});
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
|
|
490
|
-
activeJobs: activeJobs.map((job)=>job.uuid)
|
|
491
|
-
});
|
|
492
|
-
return true;
|
|
655
|
+
if (blocks.length > 0) {
|
|
656
|
+
this.chonkCache.releaseForBlocks(blocks);
|
|
493
657
|
}
|
|
494
|
-
await this.startProof(epochNumber);
|
|
495
|
-
return true;
|
|
496
658
|
} catch (err) {
|
|
497
|
-
|
|
498
|
-
this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
|
|
499
|
-
} else {
|
|
500
|
-
this.log.error(`Error handling epoch completed`, err);
|
|
501
|
-
}
|
|
502
|
-
return false;
|
|
659
|
+
this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
|
|
503
660
|
}
|
|
661
|
+
this.checkpointStore.reapExpired(epoch);
|
|
504
662
|
}
|
|
663
|
+
// ---------------- public API ----------------
|
|
505
664
|
/**
|
|
506
|
-
*
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
665
|
+
* Schedules proving for the given epoch and returns the job id without waiting for completion.
|
|
666
|
+
*/ async startProof(epochNumber) {
|
|
667
|
+
if (!this.sessionManager) {
|
|
668
|
+
throw new Error('ProverNode not started');
|
|
669
|
+
}
|
|
670
|
+
return await this.sessionManager.startProof(epochNumber);
|
|
671
|
+
}
|
|
672
|
+
// ---------------- Service lifecycle ----------------
|
|
673
|
+
async start() {
|
|
674
|
+
await this.checkpointStore.start();
|
|
510
675
|
await this.publisherFactory.start();
|
|
511
|
-
this.
|
|
676
|
+
this.publishingService = new ProofPublishingService({
|
|
677
|
+
publisherFactory: this.publisherFactory,
|
|
678
|
+
l2BlockSource: this.l2BlockSource,
|
|
679
|
+
dateProvider: this.dateProvider,
|
|
680
|
+
config: {
|
|
681
|
+
skipSubmitProof: !!this.config.proverNodeDisableProofPublish
|
|
682
|
+
},
|
|
683
|
+
bindings: this.log.getBindings()
|
|
684
|
+
});
|
|
685
|
+
this.sessionManager = this.createSessionManager(this.publishingService);
|
|
686
|
+
// SessionManager owns its own periodic tick; start it here so it begins picking up
|
|
687
|
+
// epochs that become complete by time (no fresh checkpoint event) and advances once
|
|
688
|
+
// the previous epoch is proven on L1.
|
|
689
|
+
this.sessionManager.start();
|
|
690
|
+
// Now that the store + manager exist, arm the live-state observable gauges.
|
|
691
|
+
this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
|
|
692
|
+
const { startingBlock, lastFullyProvenEpoch } = await this.computeStartupState();
|
|
693
|
+
this.lastExpiredEpoch = lastFullyProvenEpoch;
|
|
694
|
+
this.blockStream = new L2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
|
|
695
|
+
pollIntervalMS: this.config.proverNodePollingIntervalMs,
|
|
696
|
+
startingBlock
|
|
697
|
+
});
|
|
698
|
+
this.blockStream.start();
|
|
512
699
|
await this.rewardsMetrics.start();
|
|
513
700
|
this.l1Metrics.start();
|
|
514
701
|
this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
|
|
515
702
|
}
|
|
516
|
-
|
|
517
|
-
* Stops the prover node and all its dependencies.
|
|
518
|
-
* Resources not owned by this node (shared with the parent aztec-node) are skipped.
|
|
519
|
-
*/ async stop() {
|
|
703
|
+
async stop() {
|
|
520
704
|
this.log.info('Stopping ProverNode');
|
|
521
|
-
|
|
705
|
+
this.jobMetrics.stopObservingState();
|
|
706
|
+
await this.blockStream?.stop();
|
|
707
|
+
if (this.sessionManager) {
|
|
708
|
+
await this.sessionManager.stop();
|
|
709
|
+
}
|
|
710
|
+
if (this.publishingService) {
|
|
711
|
+
// Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
|
|
712
|
+
// settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
|
|
713
|
+
// the tx may still mine, but shutdown must not hang on it.
|
|
714
|
+
const publishingService = this.publishingService;
|
|
715
|
+
await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
|
|
716
|
+
}
|
|
717
|
+
await this.checkpointStore.stop();
|
|
718
|
+
this.chonkCache.stop();
|
|
522
719
|
await this.prover.stop();
|
|
523
720
|
await tryStop(this.publisherFactory);
|
|
524
|
-
this.publisher?.interrupt();
|
|
525
|
-
await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
|
|
526
721
|
this.rewardsMetrics.stop();
|
|
527
722
|
this.l1Metrics.stop();
|
|
528
723
|
await this.telemetryClient.stop();
|
|
529
724
|
this.log.info('Stopped ProverNode');
|
|
530
725
|
}
|
|
531
|
-
/** Returns world state status. */ async getWorldStateSyncStatus() {
|
|
532
|
-
const { syncSummary } = await this.worldState.status();
|
|
533
|
-
return syncSummary;
|
|
534
|
-
}
|
|
535
|
-
/** Returns archiver status. */ getL2Tips() {
|
|
536
|
-
return this.l2BlockSource.getL2Tips();
|
|
537
|
-
}
|
|
538
726
|
/**
|
|
539
|
-
*
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
727
|
+
* Constructs the session manager. Extracted so subclasses (test harness) can swap
|
|
728
|
+
* the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
|
|
729
|
+
* their proving data uploaded.
|
|
730
|
+
*/ createSessionManager(publishingService) {
|
|
731
|
+
return new SessionManager({
|
|
732
|
+
checkpointStore: this.checkpointStore,
|
|
733
|
+
l2BlockSource: this.l2BlockSource,
|
|
734
|
+
proverFactory: this.prover,
|
|
735
|
+
proverId: this.prover.getProverId(),
|
|
736
|
+
publishingService,
|
|
737
|
+
metrics: this.jobMetrics,
|
|
738
|
+
dateProvider: this.dateProvider,
|
|
739
|
+
config: {
|
|
740
|
+
maxPendingJobs: this.config.proverNodeMaxPendingJobs,
|
|
741
|
+
tickIntervalMs: this.config.proverNodePollingIntervalMs,
|
|
742
|
+
finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
|
|
743
|
+
},
|
|
744
|
+
onSessionFailed: async (session)=>{
|
|
745
|
+
await this.tryUploadSessionFailure(session);
|
|
746
|
+
},
|
|
747
|
+
bindings: this.log.getBindings()
|
|
543
748
|
});
|
|
544
|
-
void this.runJob(job);
|
|
545
749
|
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
try {
|
|
554
|
-
await job.run();
|
|
555
|
-
const state = job.getState();
|
|
556
|
-
ctx.state = state;
|
|
557
|
-
if (state === 'reorg') {
|
|
558
|
-
this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
|
|
559
|
-
await this.createProvingJob(epochNumber);
|
|
560
|
-
} else if (state === 'failed') {
|
|
561
|
-
this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
562
|
-
await this.tryUploadEpochFailure(job);
|
|
563
|
-
} else {
|
|
564
|
-
this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
|
|
565
|
-
}
|
|
566
|
-
} catch (err) {
|
|
567
|
-
this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
|
|
568
|
-
} finally{
|
|
569
|
-
this.jobs.delete(job.getId());
|
|
750
|
+
/**
|
|
751
|
+
* Installs session hooks for the e2e harness to interpose around top-tree proving
|
|
752
|
+
* (gate, override, or observe it) without monkey-patching the orchestrator factory.
|
|
753
|
+
* Applies to every session constructed after this call.
|
|
754
|
+
*/ setSessionHooks(hooks) {
|
|
755
|
+
if (!this.sessionManager) {
|
|
756
|
+
throw new Error('ProverNode not started; call start() before setting session hooks.');
|
|
570
757
|
}
|
|
758
|
+
this.sessionManager.setSessionHooks(hooks);
|
|
571
759
|
}
|
|
572
|
-
async
|
|
573
|
-
if (this.config.proverNodeFailedEpochStore) {
|
|
574
|
-
return
|
|
760
|
+
/** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */ async tryUploadSessionFailure(session) {
|
|
761
|
+
if (!this.config.proverNodeFailedEpochStore) {
|
|
762
|
+
return undefined;
|
|
575
763
|
}
|
|
764
|
+
const data = SessionManager.buildSessionProvingData(session);
|
|
765
|
+
return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, session.getId(), data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
|
|
576
766
|
}
|
|
577
|
-
|
|
578
|
-
* Returns the prover instance.
|
|
579
|
-
*/ getProver() {
|
|
580
|
-
return this.prover;
|
|
581
|
-
}
|
|
582
|
-
/**
|
|
583
|
-
* Returns an array of jobs being processed.
|
|
584
|
-
*/ getJobs() {
|
|
585
|
-
return Promise.resolve(Array.from(this.jobs.entries()).map(([uuid, job])=>({
|
|
586
|
-
uuid,
|
|
587
|
-
status: job.getState(),
|
|
588
|
-
epochNumber: job.getEpochNumber()
|
|
589
|
-
})));
|
|
590
|
-
}
|
|
591
|
-
async getActiveJobsForEpoch(epochNumber) {
|
|
592
|
-
const jobs = await this.getJobs();
|
|
593
|
-
return jobs.filter((job)=>job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
|
|
594
|
-
}
|
|
595
|
-
checkMaximumPendingJobs() {
|
|
596
|
-
const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
|
|
597
|
-
if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
|
|
598
|
-
throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
|
|
599
|
-
}
|
|
600
|
-
}
|
|
601
|
-
async createProvingJob(epochNumber, opts = {}) {
|
|
602
|
-
this.checkMaximumPendingJobs();
|
|
603
|
-
this.publisher = await this.publisherFactory.create();
|
|
604
|
-
// Gather all data for this epoch
|
|
605
|
-
const epochData = await this.gatherEpochData(epochNumber);
|
|
606
|
-
const fromCheckpoint = epochData.checkpoints[0].number;
|
|
607
|
-
const toCheckpoint = epochData.checkpoints.at(-1).number;
|
|
608
|
-
const fromBlock = epochData.checkpoints[0].blocks[0].number;
|
|
609
|
-
const lastBlock = epochData.checkpoints.at(-1).blocks.at(-1);
|
|
610
|
-
const toBlock = lastBlock.number;
|
|
611
|
-
this.log.verbose(`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`);
|
|
612
|
-
// Fast forward world state to right before the target block and get a fork
|
|
613
|
-
const lastBlockHash = await lastBlock.header.hash();
|
|
614
|
-
await this.worldState.syncImmediate(toBlock, lastBlockHash);
|
|
615
|
-
// Create a processor factory
|
|
616
|
-
const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings());
|
|
617
|
-
// Set deadline for this job to run. It will abort if it takes too long.
|
|
618
|
-
const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
|
|
619
|
-
const deadline = new Date(Number(deadlineTs) * 1000);
|
|
620
|
-
const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
|
|
621
|
-
this.jobs.set(job.getId(), job);
|
|
622
|
-
return job;
|
|
623
|
-
}
|
|
767
|
+
// ---------------- helpers ----------------
|
|
624
768
|
getL1Constants() {
|
|
625
769
|
return this.l2BlockSource.getL1Constants();
|
|
626
770
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
const
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
l1ToL2Messages,
|
|
643
|
-
epochNumber,
|
|
644
|
-
previousBlockHeader,
|
|
645
|
-
attestations
|
|
646
|
-
};
|
|
647
|
-
}
|
|
648
|
-
async gatherCheckpoints(epochNumber) {
|
|
649
|
-
const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
|
|
650
|
-
if (checkpoints.length === 0) {
|
|
651
|
-
throw new EmptyEpochError(epochNumber);
|
|
652
|
-
}
|
|
653
|
-
return checkpoints;
|
|
654
|
-
}
|
|
655
|
-
async gatherTxs(epochNumber, checkpoints) {
|
|
656
|
-
const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
|
|
657
|
-
const txProvider = this.p2pClient.getTxProvider();
|
|
658
|
-
const blocks = checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
|
|
659
|
-
const txsByBlock = await Promise.all(blocks.map((block)=>txProvider.getTxsForBlock(block, {
|
|
660
|
-
deadline
|
|
661
|
-
})));
|
|
662
|
-
const txs = txsByBlock.map(({ txs })=>txs).flat();
|
|
663
|
-
const missingTxs = txsByBlock.map(({ missingTxs })=>missingTxs).flat();
|
|
664
|
-
if (missingTxs.length === 0) {
|
|
665
|
-
this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, {
|
|
666
|
-
epochNumber
|
|
667
|
-
});
|
|
668
|
-
return txs;
|
|
771
|
+
/**
|
|
772
|
+
* Returns true if every block in the given epoch is proven on L1. An epoch is only
|
|
773
|
+
* fully proven when its *last* block is proven. Protected for direct unit-test access.
|
|
774
|
+
*/ async isEpochFullyProven(epochNumber, l1Constants) {
|
|
775
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
|
|
776
|
+
tag: 'proven'
|
|
777
|
+
});
|
|
778
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
779
|
+
return false;
|
|
780
|
+
}
|
|
781
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({
|
|
782
|
+
number: BlockNumber(provenBlockNumber)
|
|
783
|
+
}))?.header;
|
|
784
|
+
if (!provenHeader) {
|
|
785
|
+
return false;
|
|
669
786
|
}
|
|
670
|
-
|
|
787
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
788
|
+
if (epochNumber < provenEpoch) {
|
|
789
|
+
return true;
|
|
790
|
+
}
|
|
791
|
+
if (epochNumber > provenEpoch) {
|
|
792
|
+
return false;
|
|
793
|
+
}
|
|
794
|
+
return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
|
|
795
|
+
}
|
|
796
|
+
/** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
|
|
797
|
+
const nextHeader = (await this.l2BlockSource.getBlockData({
|
|
798
|
+
number: BlockNumber(provenBlockNumber + 1)
|
|
799
|
+
}))?.header;
|
|
800
|
+
if (nextHeader) {
|
|
801
|
+
return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
|
|
802
|
+
}
|
|
803
|
+
return this.l2BlockSource.isEpochComplete(provenEpoch);
|
|
671
804
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
805
|
+
/**
|
|
806
|
+
* Resolves the L2BlockStream's starting block and the last fully-proven epoch in one
|
|
807
|
+
* pass. The starting block is the first block of the next unproven epoch (or the start
|
|
808
|
+
* of the partially-proven epoch if the proven tip falls mid-epoch). The fully-proven
|
|
809
|
+
* epoch is `provenEpoch` when the proven tip is the last block of its epoch, otherwise
|
|
810
|
+
* `provenEpoch - 1`, or `undefined` if no block is proven yet.
|
|
811
|
+
*/ async computeStartupState() {
|
|
812
|
+
const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
|
|
813
|
+
tag: 'proven'
|
|
677
814
|
});
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
const
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
815
|
+
if (!provenBlockNumber || provenBlockNumber <= 0) {
|
|
816
|
+
return {
|
|
817
|
+
startingBlock: BlockNumber(1),
|
|
818
|
+
lastFullyProvenEpoch: undefined
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
const l1Constants = await this.getL1Constants();
|
|
822
|
+
const provenHeader = (await this.l2BlockSource.getBlockData({
|
|
823
|
+
number: BlockNumber(provenBlockNumber)
|
|
824
|
+
}))?.header;
|
|
825
|
+
if (!provenHeader) {
|
|
826
|
+
return {
|
|
827
|
+
startingBlock: BlockNumber(provenBlockNumber + 1),
|
|
828
|
+
lastFullyProvenEpoch: undefined
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
|
|
832
|
+
if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
|
|
833
|
+
return {
|
|
834
|
+
startingBlock: BlockNumber(provenBlockNumber + 1),
|
|
835
|
+
lastFullyProvenEpoch: provenEpoch
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
const epochCheckpoints = await this.l2BlockSource.getCheckpointsData({
|
|
839
|
+
epoch: provenEpoch
|
|
840
|
+
});
|
|
841
|
+
const firstBlockOfEpoch = epochCheckpoints.length > 0 ? epochCheckpoints[0].startBlock : BlockNumber(provenBlockNumber);
|
|
842
|
+
this.log.info(`Starting L2BlockStream at block ${firstBlockOfEpoch} (start of partially-proven epoch ${provenEpoch})`, {
|
|
843
|
+
provenBlockNumber,
|
|
844
|
+
provenEpoch,
|
|
845
|
+
firstBlockOfEpoch
|
|
846
|
+
});
|
|
847
|
+
const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
|
|
848
|
+
return {
|
|
849
|
+
startingBlock: firstBlockOfEpoch,
|
|
850
|
+
lastFullyProvenEpoch
|
|
851
|
+
};
|
|
699
852
|
}
|
|
700
|
-
|
|
701
|
-
await this.
|
|
853
|
+
async gatherPreviousBlockHeader(previousBlockNumber) {
|
|
854
|
+
const data = await this.l2BlockSource.getBlockData({
|
|
855
|
+
number: BlockNumber(previousBlockNumber)
|
|
856
|
+
});
|
|
857
|
+
if (!data?.header) {
|
|
858
|
+
throw new Error(`Previous block header ${previousBlockNumber} not found`);
|
|
859
|
+
}
|
|
860
|
+
return data.header;
|
|
702
861
|
}
|
|
703
862
|
validateConfig() {
|
|
704
863
|
if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
|
|
@@ -707,9 +866,5 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
|
|
|
707
866
|
}
|
|
708
867
|
}
|
|
709
868
|
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
super(`No blocks found for epoch ${epochNumber}`);
|
|
713
|
-
this.name = 'EmptyEpochError';
|
|
714
|
-
}
|
|
715
|
-
}
|
|
869
|
+
// Re-export so handlers can compare states externally.
|
|
870
|
+
export { EpochProvingJobTerminalState };
|