@aztec/prover-node 5.0.0-private.20260318 → 5.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +506 -0
  2. package/dest/actions/download-epoch-proving-job.js +1 -1
  3. package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
  4. package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
  5. package/dest/actions/rerun-epoch-proving-job.js +103 -21
  6. package/dest/bin/run-failed-epoch.js +1 -3
  7. package/dest/checkpoint-store.d.ts +83 -0
  8. package/dest/checkpoint-store.d.ts.map +1 -0
  9. package/dest/checkpoint-store.js +181 -0
  10. package/dest/config.d.ts +1 -1
  11. package/dest/config.d.ts.map +1 -1
  12. package/dest/config.js +1 -1
  13. package/dest/factory.d.ts +1 -1
  14. package/dest/factory.d.ts.map +1 -1
  15. package/dest/factory.js +22 -8
  16. package/dest/index.d.ts +2 -1
  17. package/dest/index.d.ts.map +1 -1
  18. package/dest/index.js +1 -0
  19. package/dest/job/checkpoint-prover.d.ts +134 -0
  20. package/dest/job/checkpoint-prover.d.ts.map +1 -0
  21. package/dest/job/checkpoint-prover.js +350 -0
  22. package/dest/job/epoch-session.d.ts +146 -0
  23. package/dest/job/epoch-session.d.ts.map +1 -0
  24. package/dest/job/epoch-session.js +709 -0
  25. package/dest/job/top-tree-job.d.ts +82 -0
  26. package/dest/job/top-tree-job.d.ts.map +1 -0
  27. package/dest/job/top-tree-job.js +152 -0
  28. package/dest/metrics.d.ts +29 -5
  29. package/dest/metrics.d.ts.map +1 -1
  30. package/dest/metrics.js +73 -9
  31. package/dest/monitors/epoch-monitor.js +6 -2
  32. package/dest/proof-publishing-service.d.ts +159 -0
  33. package/dest/proof-publishing-service.d.ts.map +1 -0
  34. package/dest/proof-publishing-service.js +334 -0
  35. package/dest/prover-node-publisher.d.ts +18 -11
  36. package/dest/prover-node-publisher.d.ts.map +1 -1
  37. package/dest/prover-node-publisher.js +195 -57
  38. package/dest/prover-node.d.ts +96 -68
  39. package/dest/prover-node.d.ts.map +1 -1
  40. package/dest/prover-node.js +382 -227
  41. package/dest/prover-publisher-factory.d.ts +2 -2
  42. package/dest/prover-publisher-factory.d.ts.map +1 -1
  43. package/dest/prover-publisher-factory.js +3 -3
  44. package/dest/session-manager.d.ts +158 -0
  45. package/dest/session-manager.d.ts.map +1 -0
  46. package/dest/session-manager.js +452 -0
  47. package/dest/test/index.d.ts +7 -6
  48. package/dest/test/index.d.ts.map +1 -1
  49. package/package.json +23 -23
  50. package/src/actions/download-epoch-proving-job.ts +1 -1
  51. package/src/actions/rerun-epoch-proving-job.ts +114 -28
  52. package/src/bin/run-failed-epoch.ts +1 -2
  53. package/src/checkpoint-store.ts +213 -0
  54. package/src/config.ts +2 -1
  55. package/src/factory.ts +18 -10
  56. package/src/index.ts +1 -0
  57. package/src/job/checkpoint-prover.ts +465 -0
  58. package/src/job/epoch-session.ts +424 -0
  59. package/src/job/top-tree-job.ts +227 -0
  60. package/src/metrics.ts +88 -12
  61. package/src/monitors/epoch-monitor.ts +2 -2
  62. package/src/proof-publishing-service.ts +424 -0
  63. package/src/prover-node-publisher.ts +220 -67
  64. package/src/prover-node.ts +439 -249
  65. package/src/prover-publisher-factory.ts +3 -3
  66. package/src/session-manager.ts +552 -0
  67. package/src/test/index.ts +6 -6
  68. package/dest/job/epoch-proving-job.d.ts +0 -63
  69. package/dest/job/epoch-proving-job.d.ts.map +0 -1
  70. package/dest/job/epoch-proving-job.js +0 -762
  71. package/src/job/epoch-proving-job.ts +0 -465
@@ -370,28 +370,39 @@ function applyDecs2203RFactory() {
370
370
  function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
371
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
372
372
  }
373
- var _dec, _dec1, _initProto;
374
- import { BlockNumber } from '@aztec/foundation/branded-types';
375
- import { assertRequired, compact, pick, sum } from '@aztec/foundation/collection';
373
+ var _initProto;
374
+ import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types';
375
+ import { assertRequired, compact, pick } from '@aztec/foundation/collection';
376
376
  import { memoize } from '@aztec/foundation/decorators';
377
377
  import { createLogger } from '@aztec/foundation/log';
378
- import { DateProvider } from '@aztec/foundation/timer';
378
+ import { DateProvider, executeTimeout } from '@aztec/foundation/timer';
379
+ import { getLastSiblingPath } from '@aztec/prover-client/helpers';
380
+ import { ChonkCache } from '@aztec/prover-client/orchestrator';
379
381
  import { PublicProcessorFactory } from '@aztec/simulator/server';
380
- import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
382
+ import { L2BlockStream, L2TipsMemoryStore } from '@aztec/stdlib/block';
383
+ import { getEpochAtSlot, getProofSubmissionDeadlineEpoch } from '@aztec/stdlib/epoch-helpers';
381
384
  import { EpochProvingJobTerminalState, tryStop } from '@aztec/stdlib/interfaces/server';
382
- import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
385
+ import { MerkleTreeId } from '@aztec/stdlib/trees';
386
+ import { getTelemetryClient } from '@aztec/telemetry-client';
383
387
  import { uploadEpochProofFailure } from './actions/upload-epoch-proof-failure.js';
384
- import { EpochProvingJob } from './job/epoch-proving-job.js';
388
+ import { CheckpointStore } from './checkpoint-store.js';
385
389
  import { ProverNodeJobMetrics, ProverNodeRewardsMetrics } from './metrics.js';
386
- _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
387
- [Attributes.EPOCH_NUMBER]: epochNumber
388
- })), _dec1 = trackSpan('ProverNode.gatherEpochData', (epochNumber)=>({
389
- [Attributes.EPOCH_NUMBER]: epochNumber
390
- }));
390
+ import { ProofPublishingService } from './proof-publishing-service.js';
391
+ import { SessionManager } from './session-manager.js';
391
392
  /**
392
- * An Aztec Prover Node is a standalone process that monitors the unfinalized chain on L1 for unproven epochs,
393
- * fetches their txs from the p2p network or external nodes, re-executes their public functions, creates a rollup
394
- * proof for the epoch, and submits it to L1.
393
+ * Grace period for the proof-publishing service to settle during shutdown. The service waits for
394
+ * any in-flight L1 proof-submission tx to finish; that tx can take a long time to mine, so we cap
395
+ * the wait rather than letting `stop()` hang indefinitely.
396
+ */ const PUBLISHING_SERVICE_STOP_TIMEOUT_MS = 30_000;
397
+ /**
398
+ * An Aztec Prover Node is a standalone process that monitors the chain for new checkpoints,
399
+ * starts proving them optimistically as they arrive, and submits epoch proofs to L1 once
400
+ * complete.
401
+ *
402
+ * The class is intentionally thin: it owns the long-lived collections (`CheckpointStore`,
403
+ * `ChonkCache`, `SessionManager`), the L2BlockStream, and a periodic ticker that nudges the
404
+ * manager to pick up newly-complete epochs. Every session lifecycle decision is delegated to
405
+ * the `SessionManager`. Each chain event is translated here into a single method call on it.
395
406
  */ export class ProverNode {
396
407
  prover;
397
408
  publisherFactory;
@@ -400,7 +411,6 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
400
411
  contractDataSource;
401
412
  worldState;
402
413
  p2pClient;
403
- epochsMonitor;
404
414
  rollupContract;
405
415
  l1Metrics;
406
416
  telemetryClient;
@@ -408,31 +418,31 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
408
418
  dateProvider;
409
419
  static{
410
420
  ({ e: [_initProto] } = _apply_decs_2203_r(this, [
411
- [
412
- _dec,
413
- 2,
414
- "createProvingJob"
415
- ],
416
421
  [
417
422
  memoize,
418
423
  2,
419
424
  "getL1Constants"
420
- ],
421
- [
422
- _dec1,
423
- 2,
424
- "gatherEpochData"
425
425
  ]
426
426
  ], []));
427
427
  }
428
428
  log;
429
- jobs;
429
+ checkpointStore;
430
+ chonkCache;
431
+ sessionManager;
430
432
  config;
431
433
  jobMetrics;
432
434
  rewardsMetrics;
435
+ /** In-memory store for the L2BlockStream's local data provider. */ tipsStore;
436
+ /** Block stream for checkpoint and reorg detection. */ blockStream;
437
+ /**
438
+ * Highest epoch whose proof-submission window has passed. Monotonic high-water mark.
439
+ * Seeded from the last fully-proven epoch at start(); advanced on every block-stream
440
+ * event by comparing the archiver's latest synced L2 slot against each epoch's
441
+ * submission deadline. Protected so tests can verify the start() seeding.
442
+ */ lastExpiredEpoch;
433
443
  tracer;
434
- publisher;
435
- constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, epochsMonitor, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
444
+ publishingService;
445
+ constructor(prover, publisherFactory, l2BlockSource, l1ToL2MessageSource, contractDataSource, worldState, p2pClient, rollupContract, l1Metrics, config = {}, telemetryClient = getTelemetryClient(), delayer, dateProvider = new DateProvider()){
436
446
  this.prover = prover;
437
447
  this.publisherFactory = publisherFactory;
438
448
  this.l2BlockSource = l2BlockSource;
@@ -440,14 +450,12 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
440
450
  this.contractDataSource = contractDataSource;
441
451
  this.worldState = worldState;
442
452
  this.p2pClient = p2pClient;
443
- this.epochsMonitor = epochsMonitor;
444
453
  this.rollupContract = rollupContract;
445
454
  this.l1Metrics = l1Metrics;
446
455
  this.telemetryClient = telemetryClient;
447
456
  this.delayer = delayer;
448
457
  this.dateProvider = dateProvider;
449
458
  this.log = (_initProto(this), createLogger('prover-node'));
450
- this.jobs = new Map();
451
459
  this.config = {
452
460
  proverNodePollingIntervalMs: 1_000,
453
461
  proverNodeMaxPendingJobs: 100,
@@ -465,6 +473,22 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
465
473
  this.tracer = telemetryClient.getTracer('ProverNode');
466
474
  this.jobMetrics = new ProverNodeJobMetrics(meter, telemetryClient.getTracer('EpochProvingJob'));
467
475
  this.rewardsMetrics = new ProverNodeRewardsMetrics(meter, this.prover.getProverId(), rollupContract);
476
+ this.tipsStore = new L2TipsMemoryStore(this.l2BlockSource.getGenesisBlockHash());
477
+ this.chonkCache = new ChonkCache(this.log.getBindings());
478
+ this.checkpointStore = new CheckpointStore(this.l2BlockSource, {
479
+ proverFactory: this.prover,
480
+ chonkCache: this.chonkCache,
481
+ publicProcessorFactory: new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings()),
482
+ dbProvider: this.worldState,
483
+ txProvider: this.p2pClient.getTxProvider(),
484
+ dateProvider: this.dateProvider,
485
+ proverId: this.prover.getProverId(),
486
+ metrics: this.jobMetrics,
487
+ txGatheringTimeoutMs: this.config.txGatheringTimeoutMs,
488
+ deadline: undefined
489
+ }, {
490
+ slotWatcherPollIntervalMs: this.config.proverNodePollingIntervalMs
491
+ }, this.log.getBindings());
468
492
  }
469
493
  getProverId() {
470
494
  return this.prover.getProverId();
@@ -472,233 +496,368 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
472
496
  getP2P() {
473
497
  return this.p2pClient;
474
498
  }
475
- /** Returns the shared tx delayer for prover L1 txs, if enabled. Test-only. */ getDelayer() {
499
+ /** Test-only: the shared L1 tx delayer, if enabled. */ getDelayer() {
476
500
  return this.delayer;
477
501
  }
502
+ /** Observability summary for the ProverNodeApi. */ getJobs() {
503
+ return Promise.resolve(this.sessionManager?.getJobs() ?? []);
504
+ }
505
+ /** Tests inspect this when validating reconcile behaviour. */ getCheckpointStore() {
506
+ return this.checkpointStore;
507
+ }
508
+ /** Tests inspect this to verify chonk-cache release semantics. */ getChonkCache() {
509
+ return this.chonkCache;
510
+ }
511
+ /** Tests inspect this when looking up live sessions. */ getSessionManager() {
512
+ if (!this.sessionManager) {
513
+ throw new Error('SessionManager not yet constructed — start() must be called first.');
514
+ }
515
+ return this.sessionManager;
516
+ }
517
+ /** Returns world state status. */ async getWorldStateSyncStatus() {
518
+ const { syncSummary } = await this.worldState.status();
519
+ return syncSummary;
520
+ }
521
+ /** Returns archiver status. */ getL2Tips() {
522
+ return this.l2BlockSource.getL2Tips();
523
+ }
524
+ /** Returns the underlying prover instance. */ getProver() {
525
+ return this.prover;
526
+ }
527
+ // ---------------- L2BlockStream handler ----------------
528
+ async handleBlockStreamEvent(event) {
529
+ switch(event.type){
530
+ case 'chain-checkpointed':
531
+ await this.handleCheckpointEvent(event.checkpoint);
532
+ break;
533
+ case 'chain-pruned':
534
+ await this.handlePruneEvent(event.checkpointed.checkpoint);
535
+ break;
536
+ case 'chain-proven':
537
+ this.publishingService?.onChainProven(BlockNumber(event.block.number));
538
+ break;
539
+ case 'chain-finalized':
540
+ case 'blocks-added':
541
+ break;
542
+ }
543
+ // Expiry is driven by the archiver's latest synced L2 slot
544
+ await this.checkEpochExpiry();
545
+ // Advance the local tips store only after the proving-side handling has succeeded. Any
546
+ // failure above propagates to the L2BlockStream (which logs and stops this poll pass) and
547
+ // skips this update, so the event is re-emitted on the next poll rather than skipped (A-1041).
548
+ await this.tipsStore.handleBlockStreamEvent(event);
549
+ }
550
+ /** Register a new checkpoint with the store and notify the session manager. */ async handleCheckpointEvent(publishedCheckpoint) {
551
+ const checkpoint = publishedCheckpoint.checkpoint;
552
+ const slotNumber = checkpoint.header.slotNumber;
553
+ const l1Constants = await this.getL1Constants();
554
+ const epochNumber = getEpochAtSlot(slotNumber, l1Constants);
555
+ if (await this.isEpochFullyProven(epochNumber, l1Constants)) {
556
+ this.log.debug(`Skipping checkpoint ${checkpoint.number} for already-proven epoch ${epochNumber}`);
557
+ return;
558
+ }
559
+ if (await this.isEpochPastProofSubmissionWindow(epochNumber, l1Constants)) {
560
+ this.log.debug(`Skipping checkpoint ${checkpoint.number} for epoch ${epochNumber} past its proof-submission window`);
561
+ return;
562
+ }
563
+ this.log.info(`New checkpoint ${checkpoint.number} for epoch ${epochNumber}`, {
564
+ checkpointNumber: checkpoint.number,
565
+ epochNumber,
566
+ slotNumber
567
+ });
568
+ const registerData = await this.collectRegisterData(checkpoint, publishedCheckpoint.attestations);
569
+ await this.checkpointStore.addOrUpdate(checkpoint, registerData);
570
+ await this.sessionManager?.onCheckpointAdded(epochNumber);
571
+ }
572
+ /**
573
+ * Gathers register-time data for a checkpoint: previous block header, L1-to-L2 messages,
574
+ * and the archive sibling path.
575
+ */ async collectRegisterData(checkpoint, attestations) {
576
+ const previousBlockNumber = BlockNumber(checkpoint.blocks[0].number - 1);
577
+ const previousBlockHeader = await this.gatherPreviousBlockHeader(previousBlockNumber);
578
+ const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpoint.number);
579
+ const lastBlock = checkpoint.blocks.at(-1);
580
+ const lastBlockHash = await lastBlock.header.hash();
581
+ await this.worldState.syncImmediate(lastBlock.number, lastBlockHash);
582
+ const previousArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, this.worldState.getSnapshot(previousBlockNumber));
583
+ return {
584
+ attestations,
585
+ previousBlockHeader,
586
+ l1ToL2Messages,
587
+ previousArchiveSiblingPath
588
+ };
589
+ }
590
+ /** Mark every prover above the prune threshold as pruned and notify the session manager. */ async handlePruneEvent(prunedCheckpoint) {
591
+ this.log.warn(`Chain pruned to checkpoint ${prunedCheckpoint.number}`, {
592
+ prunedCheckpoint
593
+ });
594
+ const affected = this.checkpointStore.markPrunedAfter(prunedCheckpoint.number);
595
+ if (affected.length === 0) {
596
+ return;
597
+ }
598
+ const l1Constants = await this.getL1Constants();
599
+ const affectedEpochs = Array.from(new Set(affected.map((p)=>Number(getEpochAtSlot(p.slotNumber, l1Constants))))).map((n)=>EpochNumber(n));
600
+ // The session manager cancels every affected session, which in turn calls
601
+ // publishingService.withdraw(uuid) for each candidate; no separate notification to the
602
+ // publishing service is needed.
603
+ await this.sessionManager?.onPrune(affectedEpochs);
604
+ }
605
+ /**
606
+ * Returns true once the chain has advanced past the given epoch's proof-submission window.
607
+ * Used to ignore checkpoints whose epoch can no longer be proven in time — chiefly while the
608
+ * archiver replays old blocks after a restart. Compares the archiver's latest synced L2 slot
609
+ * against the epoch's submission-deadline epoch; conservatively returns false if the slot can't
610
+ * be read yet.
611
+ */ async isEpochPastProofSubmissionWindow(epochNumber, l1Constants) {
612
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
613
+ if (latestSlot === undefined) {
614
+ return false;
615
+ }
616
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
617
+ return latestEpoch >= getProofSubmissionDeadlineEpoch(epochNumber, l1Constants);
618
+ }
478
619
  /**
479
- * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
480
- * @param epochNumber - The epoch number that was just completed.
481
- * @returns false if there is an error, true otherwise
482
- */ async handleEpochReadyToProve(epochNumber) {
620
+ * Compares the archiver's latest synced L2 slot against `lastExpiredEpoch` and, for each
621
+ * newly-expired epoch, releases the chonk-cache entries for its blocks and reaps any
622
+ * CheckpointProvers in the store. An epoch E is expired once the chain reaches the start
623
+ * of epoch `E + proofSubmissionEpochs + 1`. Silently no-ops if nothing has expired since
624
+ * the last check or the archiver's slot can't be read.
625
+ */ async checkEpochExpiry() {
626
+ const latestSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
627
+ if (latestSlot === undefined) {
628
+ return;
629
+ }
630
+ const l1Constants = await this.getL1Constants();
631
+ const latestEpoch = getEpochAtSlot(latestSlot, l1Constants);
632
+ const offset = l1Constants.proofSubmissionEpochs + 1;
633
+ if (latestEpoch < offset) {
634
+ return;
635
+ }
636
+ const newlyExpiredUpTo = EpochNumber(latestEpoch - offset);
637
+ const from = this.lastExpiredEpoch === undefined ? EpochNumber(0) : EpochNumber(this.lastExpiredEpoch + 1);
638
+ if (newlyExpiredUpTo < from) {
639
+ return;
640
+ }
641
+ for(let e = from; e <= newlyExpiredUpTo; e = EpochNumber(e + 1)){
642
+ await this.expireEpoch(e);
643
+ }
644
+ this.lastExpiredEpoch = newlyExpiredUpTo;
645
+ }
646
+ /**
647
+ * Releases chonk-cache entries for every block in the supplied epoch (best-effort) and
648
+ * reaps every CheckpointProver in the store whose epoch number matches.
649
+ */ async expireEpoch(epoch) {
483
650
  try {
484
- this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
485
- jobs: Array.from(this.jobs.values()).map((job)=>`${job.getEpochNumber()}:${job.getId()}`)
651
+ const blocks = await this.l2BlockSource.getBlocks({
652
+ epoch,
653
+ onlyCheckpointed: true
486
654
  });
487
- const activeJobs = await this.getActiveJobsForEpoch(epochNumber);
488
- if (activeJobs.length > 0) {
489
- this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
490
- activeJobs: activeJobs.map((job)=>job.uuid)
491
- });
492
- return true;
655
+ if (blocks.length > 0) {
656
+ this.chonkCache.releaseForBlocks(blocks);
493
657
  }
494
- await this.startProof(epochNumber);
495
- return true;
496
658
  } catch (err) {
497
- if (err instanceof EmptyEpochError) {
498
- this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
499
- } else {
500
- this.log.error(`Error handling epoch completed`, err);
501
- }
502
- return false;
659
+ this.log.warn(`Could not release chonk-cache entries for expired epoch ${epoch}`, err);
503
660
  }
661
+ this.checkpointStore.reapExpired(epoch);
504
662
  }
663
+ // ---------------- public API ----------------
505
664
  /**
506
- * Starts the prover node so it periodically checks for unproven epochs in the unfinalized chain from L1 and
507
- * starts proving jobs for them.
508
- */ async start() {
509
- this.epochsMonitor.start(this);
665
+ * Schedules proving for the given epoch and returns the job id without waiting for completion.
666
+ */ async startProof(epochNumber) {
667
+ if (!this.sessionManager) {
668
+ throw new Error('ProverNode not started');
669
+ }
670
+ return await this.sessionManager.startProof(epochNumber);
671
+ }
672
+ // ---------------- Service lifecycle ----------------
673
+ async start() {
674
+ await this.checkpointStore.start();
510
675
  await this.publisherFactory.start();
511
- this.publisher = await this.publisherFactory.create();
676
+ this.publishingService = new ProofPublishingService({
677
+ publisherFactory: this.publisherFactory,
678
+ l2BlockSource: this.l2BlockSource,
679
+ dateProvider: this.dateProvider,
680
+ config: {
681
+ skipSubmitProof: !!this.config.proverNodeDisableProofPublish
682
+ },
683
+ bindings: this.log.getBindings()
684
+ });
685
+ this.sessionManager = this.createSessionManager(this.publishingService);
686
+ // SessionManager owns its own periodic tick; start it here so it begins picking up
687
+ // epochs that become complete by time (no fresh checkpoint event) and advances once
688
+ // the previous epoch is proven on L1.
689
+ this.sessionManager.start();
690
+ // Now that the store + manager exist, arm the live-state observable gauges.
691
+ this.jobMetrics.observeState(this.checkpointStore, this.sessionManager);
692
+ const { startingBlock, lastFullyProvenEpoch } = await this.computeStartupState();
693
+ this.lastExpiredEpoch = lastFullyProvenEpoch;
694
+ this.blockStream = new L2BlockStream(this.l2BlockSource, this.tipsStore, this, this.log, {
695
+ pollIntervalMS: this.config.proverNodePollingIntervalMs,
696
+ startingBlock
697
+ });
698
+ this.blockStream.start();
512
699
  await this.rewardsMetrics.start();
513
700
  this.l1Metrics.start();
514
701
  this.log.info(`Started Prover Node with prover id ${this.prover.getProverId().toString()}`, this.config);
515
702
  }
516
- /**
517
- * Stops the prover node and all its dependencies.
518
- * Resources not owned by this node (shared with the parent aztec-node) are skipped.
519
- */ async stop() {
703
+ async stop() {
520
704
  this.log.info('Stopping ProverNode');
521
- await this.epochsMonitor.stop();
705
+ this.jobMetrics.stopObservingState();
706
+ await this.blockStream?.stop();
707
+ if (this.sessionManager) {
708
+ await this.sessionManager.stop();
709
+ }
710
+ if (this.publishingService) {
711
+ // Bound the wait: the publishing service blocks until any in-flight L1 proof-submission tx
712
+ // settles, which can outlast a reasonable shutdown window. On timeout we log and move on —
713
+ // the tx may still mine, but shutdown must not hang on it.
714
+ const publishingService = this.publishingService;
715
+ await executeTimeout(()=>publishingService.stop(), PUBLISHING_SERVICE_STOP_TIMEOUT_MS, 'prover-node publishing-service stop').catch((err)=>this.log.warn(`Timed out stopping proof publishing service`, err));
716
+ }
717
+ await this.checkpointStore.stop();
718
+ this.chonkCache.stop();
522
719
  await this.prover.stop();
523
720
  await tryStop(this.publisherFactory);
524
- this.publisher?.interrupt();
525
- await Promise.all(Array.from(this.jobs.values()).map((job)=>job.stop()));
526
721
  this.rewardsMetrics.stop();
527
722
  this.l1Metrics.stop();
528
723
  await this.telemetryClient.stop();
529
724
  this.log.info('Stopped ProverNode');
530
725
  }
531
- /** Returns world state status. */ async getWorldStateSyncStatus() {
532
- const { syncSummary } = await this.worldState.status();
533
- return syncSummary;
534
- }
535
- /** Returns archiver status. */ getL2Tips() {
536
- return this.l2BlockSource.getL2Tips();
537
- }
538
726
  /**
539
- * Starts a proving process and returns immediately.
540
- */ async startProof(epochNumber) {
541
- const job = await this.createProvingJob(epochNumber, {
542
- skipEpochCheck: true
727
+ * Constructs the session manager. Extracted so subclasses (test harness) can swap
728
+ * the implementation. Wired to `tryUploadSessionFailure` so failed sessions get
729
+ * their proving data uploaded.
730
+ */ createSessionManager(publishingService) {
731
+ return new SessionManager({
732
+ checkpointStore: this.checkpointStore,
733
+ l2BlockSource: this.l2BlockSource,
734
+ proverFactory: this.prover,
735
+ proverId: this.prover.getProverId(),
736
+ publishingService,
737
+ metrics: this.jobMetrics,
738
+ dateProvider: this.dateProvider,
739
+ config: {
740
+ maxPendingJobs: this.config.proverNodeMaxPendingJobs,
741
+ tickIntervalMs: this.config.proverNodePollingIntervalMs,
742
+ finalizationDelayMs: this.config.proverNodeEpochProvingDelayMs
743
+ },
744
+ onSessionFailed: async (session)=>{
745
+ await this.tryUploadSessionFailure(session);
746
+ },
747
+ bindings: this.log.getBindings()
543
748
  });
544
- void this.runJob(job);
545
749
  }
546
- async runJob(job) {
547
- const epochNumber = job.getEpochNumber();
548
- const ctx = {
549
- id: job.getId(),
550
- epochNumber,
551
- state: undefined
552
- };
553
- try {
554
- await job.run();
555
- const state = job.getState();
556
- ctx.state = state;
557
- if (state === 'reorg') {
558
- this.log.warn(`Running new job for epoch ${epochNumber} due to reorg`, ctx);
559
- await this.createProvingJob(epochNumber);
560
- } else if (state === 'failed') {
561
- this.log.error(`Job for ${epochNumber} exited with state ${state}`, ctx);
562
- await this.tryUploadEpochFailure(job);
563
- } else {
564
- this.log.verbose(`Job for ${epochNumber} exited with state ${state}`, ctx);
565
- }
566
- } catch (err) {
567
- this.log.error(`Error proving epoch ${epochNumber}`, err, ctx);
568
- } finally{
569
- this.jobs.delete(job.getId());
750
+ /**
751
+ * Installs session hooks for the e2e harness to interpose around top-tree proving
752
+ * (gate, override, or observe it) without monkey-patching the orchestrator factory.
753
+ * Applies to every session constructed after this call.
754
+ */ setSessionHooks(hooks) {
755
+ if (!this.sessionManager) {
756
+ throw new Error('ProverNode not started; call start() before setting session hooks.');
570
757
  }
758
+ this.sessionManager.setSessionHooks(hooks);
571
759
  }
572
- async tryUploadEpochFailure(job) {
573
- if (this.config.proverNodeFailedEpochStore) {
574
- return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, job.getId(), job.getProvingData(), this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
760
+ /** Uploads failure snapshots when sessions exit with `failed`. Exposed as a method so tests can spy on it. */ async tryUploadSessionFailure(session) {
761
+ if (!this.config.proverNodeFailedEpochStore) {
762
+ return undefined;
575
763
  }
764
+ const data = SessionManager.buildSessionProvingData(session);
765
+ return await uploadEpochProofFailure(this.config.proverNodeFailedEpochStore, session.getId(), data, this.l2BlockSource, this.worldState, assertRequired(pick(this.config, 'l1ChainId', 'rollupVersion', 'dataDirectory')), this.log);
576
766
  }
577
- /**
578
- * Returns the prover instance.
579
- */ getProver() {
580
- return this.prover;
581
- }
582
- /**
583
- * Returns an array of jobs being processed.
584
- */ getJobs() {
585
- return Promise.resolve(Array.from(this.jobs.entries()).map(([uuid, job])=>({
586
- uuid,
587
- status: job.getState(),
588
- epochNumber: job.getEpochNumber()
589
- })));
590
- }
591
- async getActiveJobsForEpoch(epochNumber) {
592
- const jobs = await this.getJobs();
593
- return jobs.filter((job)=>job.epochNumber === epochNumber && !EpochProvingJobTerminalState.includes(job.status));
594
- }
595
- checkMaximumPendingJobs() {
596
- const { proverNodeMaxPendingJobs: maxPendingJobs } = this.config;
597
- if (maxPendingJobs > 0 && this.jobs.size >= maxPendingJobs) {
598
- throw new Error(`Maximum pending proving jobs ${maxPendingJobs} reached. Cannot create new job.`);
599
- }
600
- }
601
- async createProvingJob(epochNumber, opts = {}) {
602
- this.checkMaximumPendingJobs();
603
- this.publisher = await this.publisherFactory.create();
604
- // Gather all data for this epoch
605
- const epochData = await this.gatherEpochData(epochNumber);
606
- const fromCheckpoint = epochData.checkpoints[0].number;
607
- const toCheckpoint = epochData.checkpoints.at(-1).number;
608
- const fromBlock = epochData.checkpoints[0].blocks[0].number;
609
- const lastBlock = epochData.checkpoints.at(-1).blocks.at(-1);
610
- const toBlock = lastBlock.number;
611
- this.log.verbose(`Creating proving job for epoch ${epochNumber} for checkpoint range ${fromCheckpoint} to ${toCheckpoint} and block range ${fromBlock} to ${toBlock}`);
612
- // Fast forward world state to right before the target block and get a fork
613
- const lastBlockHash = await lastBlock.header.hash();
614
- await this.worldState.syncImmediate(toBlock, lastBlockHash);
615
- // Create a processor factory
616
- const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.dateProvider, this.telemetryClient, this.log.getBindings());
617
- // Set deadline for this job to run. It will abort if it takes too long.
618
- const deadlineTs = getProofSubmissionDeadlineTimestamp(epochNumber, await this.getL1Constants());
619
- const deadline = new Date(Number(deadlineTs) * 1000);
620
- const job = this.doCreateEpochProvingJob(epochData, deadline, publicProcessorFactory, this.publisher, opts);
621
- this.jobs.set(job.getId(), job);
622
- return job;
623
- }
767
+ // ---------------- helpers ----------------
624
768
  getL1Constants() {
625
769
  return this.l2BlockSource.getL1Constants();
626
770
  }
627
- async gatherEpochData(epochNumber) {
628
- const checkpoints = await this.gatherCheckpoints(epochNumber);
629
- const txArray = await this.gatherTxs(epochNumber, checkpoints);
630
- const txs = new Map(txArray.map((tx)=>[
631
- tx.getTxHash().toString(),
632
- tx
633
- ]));
634
- const l1ToL2Messages = await this.gatherMessages(epochNumber, checkpoints);
635
- const [firstBlock] = checkpoints[0].blocks;
636
- const previousBlockHeader = await this.gatherPreviousBlockHeader(epochNumber, firstBlock.number - 1);
637
- const [lastPublishedCheckpoint] = await this.l2BlockSource.getCheckpoints(checkpoints.at(-1).number, 1);
638
- const attestations = lastPublishedCheckpoint?.attestations ?? [];
639
- return {
640
- checkpoints,
641
- txs,
642
- l1ToL2Messages,
643
- epochNumber,
644
- previousBlockHeader,
645
- attestations
646
- };
647
- }
648
- async gatherCheckpoints(epochNumber) {
649
- const checkpoints = await this.l2BlockSource.getCheckpointsForEpoch(epochNumber);
650
- if (checkpoints.length === 0) {
651
- throw new EmptyEpochError(epochNumber);
652
- }
653
- return checkpoints;
654
- }
655
- async gatherTxs(epochNumber, checkpoints) {
656
- const deadline = new Date(this.dateProvider.now() + this.config.txGatheringTimeoutMs);
657
- const txProvider = this.p2pClient.getTxProvider();
658
- const blocks = checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
659
- const txsByBlock = await Promise.all(blocks.map((block)=>txProvider.getTxsForBlock(block, {
660
- deadline
661
- })));
662
- const txs = txsByBlock.map(({ txs })=>txs).flat();
663
- const missingTxs = txsByBlock.map(({ missingTxs })=>missingTxs).flat();
664
- if (missingTxs.length === 0) {
665
- this.log.verbose(`Gathered all ${txs.length} txs for epoch ${epochNumber}`, {
666
- epochNumber
667
- });
668
- return txs;
771
+ /**
772
+ * Returns true if every block in the given epoch is proven on L1. An epoch is only
773
+ * fully proven when its *last* block is proven. Protected for direct unit-test access.
774
+ */ async isEpochFullyProven(epochNumber, l1Constants) {
775
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
776
+ tag: 'proven'
777
+ });
778
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
779
+ return false;
780
+ }
781
+ const provenHeader = (await this.l2BlockSource.getBlockData({
782
+ number: BlockNumber(provenBlockNumber)
783
+ }))?.header;
784
+ if (!provenHeader) {
785
+ return false;
669
786
  }
670
- throw new Error(`Txs not found for epoch ${epochNumber}: ${missingTxs.map((hash)=>hash.toString()).join(', ')}`);
787
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
788
+ if (epochNumber < provenEpoch) {
789
+ return true;
790
+ }
791
+ if (epochNumber > provenEpoch) {
792
+ return false;
793
+ }
794
+ return this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants);
795
+ }
796
+ /** Protected for direct unit-test access. */ async isProvenBlockLastOfItsEpoch(provenBlockNumber, provenEpoch, l1Constants) {
797
+ const nextHeader = (await this.l2BlockSource.getBlockData({
798
+ number: BlockNumber(provenBlockNumber + 1)
799
+ }))?.header;
800
+ if (nextHeader) {
801
+ return getEpochAtSlot(nextHeader.getSlot(), l1Constants) > provenEpoch;
802
+ }
803
+ return this.l2BlockSource.isEpochComplete(provenEpoch);
671
804
  }
672
- async gatherMessages(epochNumber, checkpoints) {
673
- const messages = await Promise.all(checkpoints.map((c)=>this.l1ToL2MessageSource.getL1ToL2Messages(c.number)));
674
- const messageCount = sum(messages.map((m)=>m.length));
675
- this.log.verbose(`Gathered all ${messageCount} messages for epoch ${epochNumber}`, {
676
- epochNumber
805
+ /**
806
+ * Resolves the L2BlockStream's starting block and the last fully-proven epoch in one
807
+ * pass. The starting block is the first block of the next unproven epoch (or the start
808
+ * of the partially-proven epoch if the proven tip falls mid-epoch). The fully-proven
809
+ * epoch is `provenEpoch` when the proven tip is the last block of its epoch, otherwise
810
+ * `provenEpoch - 1`, or `undefined` if no block is proven yet.
811
+ */ async computeStartupState() {
812
+ const provenBlockNumber = await this.l2BlockSource.getBlockNumber({
813
+ tag: 'proven'
677
814
  });
678
- const messagesByCheckpoint = {};
679
- for(let i = 0; i < checkpoints.length; i++){
680
- messagesByCheckpoint[checkpoints[i].number] = messages[i];
681
- }
682
- return messagesByCheckpoint;
683
- }
684
- async gatherPreviousBlockHeader(epochNumber, previousBlockNumber) {
685
- const header = await (previousBlockNumber === 0 ? this.worldState.getCommitted().getInitialHeader() : this.l2BlockSource.getBlockHeader(BlockNumber(previousBlockNumber)));
686
- if (!header) {
687
- throw new Error(`Previous block header ${previousBlockNumber} not found for proving epoch ${epochNumber}`);
688
- }
689
- this.log.verbose(`Gathered previous block header ${header.getBlockNumber()} for epoch ${epochNumber}`);
690
- return header;
691
- }
692
- /** Extracted for testing purposes. */ doCreateEpochProvingJob(data, deadline, publicProcessorFactory, publisher, opts = {}) {
693
- const { proverNodeMaxParallelBlocksPerEpoch: parallelBlockLimit, proverNodeDisableProofPublish } = this.config;
694
- return new EpochProvingJob(data, this.worldState, this.prover.createEpochProver(), publicProcessorFactory, publisher, this.l2BlockSource, this.jobMetrics, deadline, {
695
- parallelBlockLimit,
696
- skipSubmitProof: proverNodeDisableProofPublish,
697
- ...opts
698
- }, this.log.getBindings());
815
+ if (!provenBlockNumber || provenBlockNumber <= 0) {
816
+ return {
817
+ startingBlock: BlockNumber(1),
818
+ lastFullyProvenEpoch: undefined
819
+ };
820
+ }
821
+ const l1Constants = await this.getL1Constants();
822
+ const provenHeader = (await this.l2BlockSource.getBlockData({
823
+ number: BlockNumber(provenBlockNumber)
824
+ }))?.header;
825
+ if (!provenHeader) {
826
+ return {
827
+ startingBlock: BlockNumber(provenBlockNumber + 1),
828
+ lastFullyProvenEpoch: undefined
829
+ };
830
+ }
831
+ const provenEpoch = getEpochAtSlot(provenHeader.getSlot(), l1Constants);
832
+ if (await this.isProvenBlockLastOfItsEpoch(BlockNumber(provenBlockNumber), provenEpoch, l1Constants)) {
833
+ return {
834
+ startingBlock: BlockNumber(provenBlockNumber + 1),
835
+ lastFullyProvenEpoch: provenEpoch
836
+ };
837
+ }
838
+ const epochCheckpoints = await this.l2BlockSource.getCheckpointsData({
839
+ epoch: provenEpoch
840
+ });
841
+ const firstBlockOfEpoch = epochCheckpoints.length > 0 ? epochCheckpoints[0].startBlock : BlockNumber(provenBlockNumber);
842
+ this.log.info(`Starting L2BlockStream at block ${firstBlockOfEpoch} (start of partially-proven epoch ${provenEpoch})`, {
843
+ provenBlockNumber,
844
+ provenEpoch,
845
+ firstBlockOfEpoch
846
+ });
847
+ const lastFullyProvenEpoch = provenEpoch > 0 ? EpochNumber(provenEpoch - 1) : undefined;
848
+ return {
849
+ startingBlock: firstBlockOfEpoch,
850
+ lastFullyProvenEpoch
851
+ };
699
852
  }
700
- /** Extracted for testing purposes. */ async triggerMonitors() {
701
- await this.epochsMonitor.work();
853
+ async gatherPreviousBlockHeader(previousBlockNumber) {
854
+ const data = await this.l2BlockSource.getBlockData({
855
+ number: BlockNumber(previousBlockNumber)
856
+ });
857
+ if (!data?.header) {
858
+ throw new Error(`Previous block header ${previousBlockNumber} not found`);
859
+ }
860
+ return data.header;
702
861
  }
703
862
  validateConfig() {
704
863
  if (this.config.proverNodeFailedEpochStore && (!this.config.dataDirectory || !this.config.l1ChainId || this.config.rollupVersion === undefined)) {
@@ -707,9 +866,5 @@ _dec = trackSpan('ProverNode.createProvingJob', (epochNumber)=>({
707
866
  }
708
867
  }
709
868
  }
710
- class EmptyEpochError extends Error {
711
- constructor(epochNumber){
712
- super(`No blocks found for epoch ${epochNumber}`);
713
- this.name = 'EmptyEpochError';
714
- }
715
- }
869
+ // Re-export so handlers can compare states externally.
870
+ export { EpochProvingJobTerminalState };