@lodestar/beacon-node 1.47.0-dev.4546ac1229 → 1.47.0-dev.56250f4ab6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +39 -11
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/beacon/pool/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/pool/index.js +13 -1
  6. package/lib/api/impl/beacon/pool/index.js.map +1 -1
  7. package/lib/api/impl/events/index.d.ts.map +1 -1
  8. package/lib/api/impl/events/index.js +2 -0
  9. package/lib/api/impl/events/index.js.map +1 -1
  10. package/lib/api/impl/lodestar/index.js +1 -1
  11. package/lib/api/impl/lodestar/index.js.map +1 -1
  12. package/lib/api/impl/validator/index.d.ts.map +1 -1
  13. package/lib/api/impl/validator/index.js +232 -49
  14. package/lib/api/impl/validator/index.js.map +1 -1
  15. package/lib/api/impl/validator/utils.js +1 -1
  16. package/lib/api/impl/validator/utils.js.map +1 -1
  17. package/lib/api/rest/base.d.ts.map +1 -1
  18. package/lib/api/rest/base.js +7 -0
  19. package/lib/api/rest/base.js.map +1 -1
  20. package/lib/chain/archiveStore/archiveStore.js +1 -1
  21. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  22. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +2 -1
  23. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  24. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +6 -2
  25. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  26. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +14 -5
  27. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  28. package/lib/chain/blocks/payloadEnvelopeInput/types.js +5 -0
  29. package/lib/chain/blocks/payloadEnvelopeInput/types.js.map +1 -1
  30. package/lib/chain/chain.d.ts +6 -2
  31. package/lib/chain/chain.d.ts.map +1 -1
  32. package/lib/chain/chain.js +14 -3
  33. package/lib/chain/chain.js.map +1 -1
  34. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  35. package/lib/chain/forkChoice/index.js +4 -0
  36. package/lib/chain/forkChoice/index.js.map +1 -1
  37. package/lib/chain/interface.d.ts +4 -1
  38. package/lib/chain/interface.d.ts.map +1 -1
  39. package/lib/chain/interface.js.map +1 -1
  40. package/lib/chain/opPools/deferredVoluntaryExitPool.d.ts +16 -0
  41. package/lib/chain/opPools/deferredVoluntaryExitPool.d.ts.map +1 -0
  42. package/lib/chain/opPools/deferredVoluntaryExitPool.js +57 -0
  43. package/lib/chain/opPools/deferredVoluntaryExitPool.js.map +1 -0
  44. package/lib/chain/opPools/executionPayloadBidPool.d.ts +7 -2
  45. package/lib/chain/opPools/executionPayloadBidPool.d.ts.map +1 -1
  46. package/lib/chain/opPools/executionPayloadBidPool.js +4 -4
  47. package/lib/chain/opPools/executionPayloadBidPool.js.map +1 -1
  48. package/lib/chain/opPools/index.d.ts +2 -1
  49. package/lib/chain/opPools/index.d.ts.map +1 -1
  50. package/lib/chain/opPools/index.js +1 -0
  51. package/lib/chain/opPools/index.js.map +1 -1
  52. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  53. package/lib/chain/prepareNextSlot.js +23 -5
  54. package/lib/chain/prepareNextSlot.js.map +1 -1
  55. package/lib/chain/produceBlock/produceBlockBody.js +1 -1
  56. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  57. package/lib/chain/regen/interface.d.ts +4 -2
  58. package/lib/chain/regen/interface.d.ts.map +1 -1
  59. package/lib/chain/regen/interface.js +2 -0
  60. package/lib/chain/regen/interface.js.map +1 -1
  61. package/lib/chain/regen/queued.d.ts +2 -2
  62. package/lib/chain/regen/queued.d.ts.map +1 -1
  63. package/lib/chain/regen/queued.js +5 -5
  64. package/lib/chain/regen/queued.js.map +1 -1
  65. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +28 -1
  66. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  67. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +121 -13
  68. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  69. package/lib/chain/stateCache/fifoBlockStateCache.d.ts +5 -1
  70. package/lib/chain/stateCache/fifoBlockStateCache.d.ts.map +1 -1
  71. package/lib/chain/stateCache/fifoBlockStateCache.js +6 -0
  72. package/lib/chain/stateCache/fifoBlockStateCache.js.map +1 -1
  73. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +47 -5
  74. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  75. package/lib/chain/stateCache/persistentCheckpointsCache.js +117 -7
  76. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  77. package/lib/chain/stateCache/types.d.ts +3 -3
  78. package/lib/chain/stateCache/types.d.ts.map +1 -1
  79. package/lib/chain/validation/executionPayloadBid.js +2 -2
  80. package/lib/chain/validation/executionPayloadBid.js.map +1 -1
  81. package/lib/chain/validation/voluntaryExit.d.ts +9 -1
  82. package/lib/chain/validation/voluntaryExit.d.ts.map +1 -1
  83. package/lib/chain/validation/voluntaryExit.js +46 -1
  84. package/lib/chain/validation/voluntaryExit.js.map +1 -1
  85. package/lib/execution/builder/apiClient.d.ts +64 -0
  86. package/lib/execution/builder/apiClient.d.ts.map +1 -0
  87. package/lib/execution/builder/apiClient.js +184 -0
  88. package/lib/execution/builder/apiClient.js.map +1 -0
  89. package/lib/execution/builder/validateBid.d.ts +22 -0
  90. package/lib/execution/builder/validateBid.d.ts.map +1 -0
  91. package/lib/execution/builder/validateBid.js +111 -0
  92. package/lib/execution/builder/validateBid.js.map +1 -0
  93. package/lib/metrics/metrics/lodestar.d.ts +39 -5
  94. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  95. package/lib/metrics/metrics/lodestar.js +63 -2
  96. package/lib/metrics/metrics/lodestar.js.map +1 -1
  97. package/lib/network/processor/constants.d.ts +7 -0
  98. package/lib/network/processor/constants.d.ts.map +1 -0
  99. package/lib/network/processor/constants.js +7 -0
  100. package/lib/network/processor/constants.js.map +1 -0
  101. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  102. package/lib/network/processor/gossipHandlers.js +6 -2
  103. package/lib/network/processor/gossipHandlers.js.map +1 -1
  104. package/lib/network/processor/gossipQueues/indexed.d.ts.map +1 -1
  105. package/lib/network/processor/gossipQueues/indexed.js +0 -1
  106. package/lib/network/processor/gossipQueues/indexed.js.map +1 -1
  107. package/lib/network/processor/index.d.ts +3 -0
  108. package/lib/network/processor/index.d.ts.map +1 -1
  109. package/lib/network/processor/index.js +41 -17
  110. package/lib/network/processor/index.js.map +1 -1
  111. package/lib/node/deferredVoluntaryExitPublisher.d.ts +15 -0
  112. package/lib/node/deferredVoluntaryExitPublisher.d.ts.map +1 -0
  113. package/lib/node/deferredVoluntaryExitPublisher.js +38 -0
  114. package/lib/node/deferredVoluntaryExitPublisher.js.map +1 -0
  115. package/lib/node/nodejs.d.ts.map +1 -1
  116. package/lib/node/nodejs.js +7 -0
  117. package/lib/node/nodejs.js.map +1 -1
  118. package/lib/sync/types.d.ts +32 -0
  119. package/lib/sync/types.d.ts.map +1 -1
  120. package/lib/sync/types.js +33 -0
  121. package/lib/sync/types.js.map +1 -1
  122. package/lib/sync/unknownBlock.d.ts +6 -2
  123. package/lib/sync/unknownBlock.d.ts.map +1 -1
  124. package/lib/sync/unknownBlock.js +143 -68
  125. package/lib/sync/unknownBlock.js.map +1 -1
  126. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  127. package/lib/sync/utils/downloadByRange.js +16 -11
  128. package/lib/sync/utils/downloadByRange.js.map +1 -1
  129. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  130. package/lib/sync/utils/downloadByRoot.js +3 -1
  131. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  132. package/package.json +34 -34
  133. package/src/api/impl/beacon/blocks/index.ts +46 -14
  134. package/src/api/impl/beacon/pool/index.ts +15 -1
  135. package/src/api/impl/events/index.ts +3 -1
  136. package/src/api/impl/lodestar/index.ts +1 -1
  137. package/src/api/impl/validator/index.ts +285 -49
  138. package/src/api/impl/validator/utils.ts +1 -1
  139. package/src/api/rest/base.ts +7 -0
  140. package/src/chain/archiveStore/archiveStore.ts +1 -1
  141. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +8 -2
  142. package/src/chain/blocks/payloadEnvelopeInput/types.ts +17 -4
  143. package/src/chain/chain.ts +16 -1
  144. package/src/chain/forkChoice/index.ts +4 -0
  145. package/src/chain/interface.ts +4 -0
  146. package/src/chain/opPools/deferredVoluntaryExitPool.ts +65 -0
  147. package/src/chain/opPools/executionPayloadBidPool.ts +13 -7
  148. package/src/chain/opPools/index.ts +2 -1
  149. package/src/chain/prepareNextSlot.ts +23 -5
  150. package/src/chain/produceBlock/produceBlockBody.ts +1 -1
  151. package/src/chain/regen/interface.ts +4 -2
  152. package/src/chain/regen/queued.ts +5 -5
  153. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +143 -14
  154. package/src/chain/stateCache/fifoBlockStateCache.ts +8 -1
  155. package/src/chain/stateCache/persistentCheckpointsCache.ts +133 -8
  156. package/src/chain/stateCache/types.ts +3 -3
  157. package/src/chain/validation/executionPayloadBid.ts +2 -2
  158. package/src/chain/validation/voluntaryExit.ts +59 -2
  159. package/src/execution/builder/apiClient.ts +244 -0
  160. package/src/execution/builder/validateBid.ts +166 -0
  161. package/src/metrics/metrics/lodestar.ts +72 -7
  162. package/src/network/processor/constants.ts +6 -0
  163. package/src/network/processor/gossipHandlers.ts +6 -2
  164. package/src/network/processor/gossipQueues/indexed.ts +0 -1
  165. package/src/network/processor/index.ts +53 -21
  166. package/src/node/deferredVoluntaryExitPublisher.ts +54 -0
  167. package/src/node/nodejs.ts +8 -0
  168. package/src/sync/types.ts +33 -0
  169. package/src/sync/unknownBlock.ts +167 -73
  170. package/src/sync/utils/downloadByRange.ts +21 -12
  171. package/src/sync/utils/downloadByRoot.ts +3 -1
@@ -144,13 +144,20 @@ export class FIFOBlockStateCache implements BlockStateCache {
144
144
  return this.cache.size;
145
145
  }
146
146
 
147
+ /**
148
+ * On a checkpoint event, prune the cache while keeping the head state.
149
+ */
150
+ onCheckpoint(headStateRootHex: string): void {
151
+ this.prune(headStateRootHex);
152
+ }
153
+
147
154
  /**
148
155
  * Prune the cache from tail to keep the most recent n states consistently.
149
156
  * The tail of the list is the oldest state, in case regen adds back the same state,
150
157
  * it should stay next to head so that it won't be pruned right away.
151
158
  * The FIFO cache helps with this.
152
159
  */
153
- prune(lastAddedKey: string): void {
160
+ private prune(lastAddedKey: string): void {
154
161
  while (this.keyOrder.length > this.maxStates) {
155
162
  const key = this.keyOrder.last();
156
163
  // it does not make sense to prune the last added state
@@ -14,7 +14,11 @@ import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} fro
14
14
  export type PersistentCheckpointStateCacheOpts = {
15
15
  /** Keep max n state epochs in memory, persist the rest to disk */
16
16
  maxCPStateEpochsInMemory?: number;
17
- /** Keep max n state epochs on disk */
17
+ /**
18
+ * On-disk retention of persisted checkpoint states. Selects one of two modes:
19
+ * - `Infinity` (default) -> tiered retention mode
20
+ * - any finite `n` (>= 0) -> legacy flat retention (keep the last n dense epochs)
21
+ */
18
22
  maxCPStateEpochsOnDisk?: number;
19
23
  };
20
24
 
@@ -64,6 +68,18 @@ export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3;
64
68
  */
65
69
  export const DEFAULT_MAX_CP_STATE_ON_DISK = Infinity;
66
70
 
71
+ // CP_STATE_TIER_BASE = spacing multiplier per tier AND per-tier capacity. Deciding capacity == base
72
+ // so that a full tier[n] always holds exactly ONE epoch aligned to tier[n+1]
73
+ // given tier[n] has span = 16 ^ n and tier[n+1] has span = 16 ^ (n+1)
74
+ // why 16?
75
+ // - tier 0 is the dense hot-regen window; it must exceed regen's MAX_EPOCH_TO_PROCESS (=5).
76
+ // - with a full 4 tiers = 64 epochs, we can cover ~ a year of continuous non-finality, and it'll grow very slowly after that
77
+ // - tier0 spans 16 epochs ≈ 1.7h, this is more than enough to handle reorg
78
+ // - tier1 spans 256 epochs ≈ 1.1d, ie we store 16 epochs of the previous day
79
+ // - tier2 spans 4096 epochs ≈ 18d, ie we store 16 epochs of the last ~2.5 weeks
80
+ // - tier3 spans 65536ep ≈ 291d, ie we store 16 epochs of the last 0.8 year
81
+ export const CP_STATE_TIER_BASE = 16;
82
+
67
83
  // TODO GLOAS: re-evaluate this timing
68
84
  const PROCESS_CHECKPOINT_STATES_BPS = 6667;
69
85
 
@@ -112,6 +128,19 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
112
128
  private preComputedCheckpointHits: number | null = null;
113
129
  private readonly maxEpochsInMemory: number;
114
130
  private readonly maxEpochsOnDisk: number;
131
+ /**
132
+ * Persist checkpoint states (epoch + root) by tiers
133
+ *
134
+ * ╔═════════ tier 2 ══════════╦═══════ tier 1 ═══════╦═══ tier 0 (dense) ══╗
135
+ * finalizedEpoch -> ║ keep 1 epoch per 256 ║ keep 1 epoch per 16 ║ keep every epoch ║ -> head
136
+ * ║ x·······x·······x······ ║ x···x···x···x··· ║ xxxxxxxxxxxxxxxxx ║
137
+ * ╚═══════════════════════════╩══════════════════════╩═════════════════════╝
138
+ *
139
+ * starting from tier 1, all epochs must be aligned to the finalized epoch
140
+ */
141
+ private persistentTiers: Map<Epoch, Set<RootHex>>[] = [new Map()];
142
+ private finalizedEpoch: Epoch = 0;
143
+ private justifiedEpoch: Epoch = 0;
115
144
  private readonly datastore: CPStateDatastore;
116
145
  private readonly blockStateCache: BlockStateCache;
117
146
  private readonly bufferPool?: BufferPool;
@@ -152,6 +181,13 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
152
181
  metrics.cpStateCache.size.set({type: CacheItemType.inMemory}, inMemoryCount);
153
182
  metrics.cpStateCache.epochSize.set({type: CacheItemType.persisted}, persistentEpochs.size);
154
183
  metrics.cpStateCache.epochSize.set({type: CacheItemType.inMemory}, memoryEpochs.size);
184
+
185
+ for (let tier = 0; tier < this.persistentTiers.length; tier++) {
186
+ let states = 0;
187
+ for (const roots of this.persistentTiers[tier].values()) states += roots.size;
188
+ metrics.cpStateCache.persistentTierEpochs.set({tier}, this.persistentTiers[tier].size);
189
+ metrics.cpStateCache.persistentTierStates.set({tier}, states);
190
+ }
155
191
  });
156
192
  }
157
193
  this.logger = logger;
@@ -407,10 +443,77 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
407
443
  }
408
444
 
409
445
  /**
410
- * This is just to conform to the old implementation
446
+ * This is called on every checkpoint event to refresh the pinned finalized/justified epochs.
411
447
  */
412
- prune(): void {
413
- // do nothing
448
+ onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void {
449
+ this.finalizedEpoch = finalizedEpoch;
450
+ this.justifiedEpoch = justifiedEpoch;
451
+ }
452
+
453
+ /**
454
+ * This is called right after we persist a checkpoint state.
455
+ */
456
+ private async onPersist(epoch: Epoch, rootHex: RootHex): Promise<void> {
457
+ let roots = this.persistentTiers[0].get(epoch);
458
+ if (roots === undefined) {
459
+ roots = new Set<RootHex>();
460
+ this.persistentTiers[0].set(epoch, roots);
461
+ }
462
+ roots.add(rootHex);
463
+ await this.prunePersistentTier(0);
464
+ }
465
+
466
+ /**
467
+ * When tier `n` overflows (> CP_STATE_TIER_BASE epochs), evict its oldest epoch and either promote it to tier `n + 1` (if aligned there) or delete it.
468
+ *
469
+ * before: persistentTiers[n] = { ..., e_old }
470
+ * persistentTiers[n+1] = { ... }
471
+ *
472
+ * after, if e_old aligns to n+1: | after, if e_old does NOT align to n+1:
473
+ * persistentTiers[n] = { ... } | persistentTiers[n] = { ... }
474
+ * persistentTiers[n+1] = { ..., e_old } | e_old and all its roots deleted from disk
475
+ *
476
+ * Since pruning a tier[n] may push an item at tier[n+1], we must call the function recursively.
477
+ */
478
+ private async prunePersistentTier(tier: number): Promise<void> {
479
+ const tierMap = this.persistentTiers[tier];
480
+ while (tierMap.size > CP_STATE_TIER_BASE) {
481
+ // Evict the oldest epoch, but never the pinned justified epoch
482
+ const oldestEpoch = Math.min(...Array.from(tierMap.keys()).filter((epoch) => epoch !== this.justifiedEpoch));
483
+ const roots = tierMap.get(oldestEpoch) as Set<RootHex>;
484
+ tierMap.delete(oldestEpoch);
485
+
486
+ const rootHexes = Array.from(roots).join(",");
487
+ // `>= finalizedEpoch` guards the transient window where an epoch below the (fork-choice)
488
+ // finalized epoch has not yet been removed by pruneFinalized: its offset would be negative
489
+ // and JS `%` could spuriously report it as aligned. Such epochs are stale -> delete.
490
+ if (oldestEpoch >= this.finalizedEpoch && this.alignedForTier(oldestEpoch, tier + 1)) {
491
+ if (this.persistentTiers[tier + 1] === undefined) {
492
+ this.persistentTiers[tier + 1] = new Map<Epoch, Set<RootHex>>();
493
+ }
494
+ this.persistentTiers[tier + 1].set(oldestEpoch, roots);
495
+ this.logger.verbose("Migrated persistent checkpoint states to lower tier", {
496
+ epoch: oldestEpoch,
497
+ fromTier: tier,
498
+ toTier: tier + 1,
499
+ rootHexes,
500
+ });
501
+ await this.prunePersistentTier(tier + 1);
502
+ } else {
503
+ // the oldest epoch is not aligned to the next tier
504
+ await this.deleteAllEpochItems(oldestEpoch)
505
+ .then(() => this.logger.verbose("Pruned checkpoint states from disk", {epoch: oldestEpoch, tier, rootHexes}))
506
+ .catch((e) => this.logger.debug("Error delete all epoch items", {epoch: oldestEpoch, tier}, e as Error));
507
+ }
508
+ }
509
+ }
510
+
511
+ /**
512
+ * Starting from tier[1], it must be aligned to the finalized epoch
513
+ * ie given finalized epoch F, tier[n] only epochs of F + n*16, n = 1,2,3...
514
+ */
515
+ private alignedForTier(epoch: Epoch, tier: number): boolean {
516
+ return (epoch - this.finalizedEpoch) % CP_STATE_TIER_BASE ** tier === 0;
414
517
  }
415
518
 
416
519
  /**
@@ -598,9 +701,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
598
701
  return seedBlockState;
599
702
  }
600
703
 
601
- clear(): void {
602
- this.cache.clear();
603
- this.epochIndex.clear();
704
+ async clear(): Promise<void> {
705
+ // deleteAllEpochItems removes every root of an epoch from disk and clears the cache
706
+ await Promise.all(
707
+ Array.from(this.epochIndex.keys()).map((epoch) =>
708
+ this.deleteAllEpochItems(epoch).catch((e) =>
709
+ this.logger.debug("Error delete all epoch items on clear", {epoch}, e as Error)
710
+ )
711
+ )
712
+ );
713
+ this.persistentTiers = [new Map()];
604
714
  }
605
715
 
606
716
  /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
@@ -743,12 +853,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
743
853
  }
744
854
  // overwrite cpKey, this means the state is deleted from memory
745
855
  this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
856
+ if (this.isTieredMode) {
857
+ await this.onPersist(epoch, rootHex);
858
+ }
746
859
  } else {
747
860
  if (persistedKey) {
748
861
  // persisted file will be eventually deleted by the archive task
749
862
  // this also means the state is deleted from memory
750
863
  this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
751
864
  // do not update epochIndex
865
+ if (this.isTieredMode) {
866
+ await this.onPersist(epoch, rootHex);
867
+ }
752
868
  } else {
753
869
  // delete the state from memory
754
870
  this.cache.delete(cpKey);
@@ -790,6 +906,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
790
906
  this.cache.delete(key);
791
907
  }
792
908
  this.epochIndex.delete(epoch);
909
+ // keep the on-disk retention tiers in sync so they never drift from epochIndex
910
+ for (const tierMap of this.persistentTiers) {
911
+ tierMap.delete(epoch);
912
+ }
793
913
  this.logger.verbose("Pruned checkpoint states for epoch", {
794
914
  epoch,
795
915
  persistCount,
@@ -798,7 +918,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
798
918
  }
799
919
 
800
920
  /**
801
- * Prune persisted checkpoint states from disk.
921
+ * Legacy flat-window on-disk pruning, used only when a `maxEpochsOnDisk` is configured.
802
922
  * Note that this should handle all possible errors and not throw.
803
923
  */
804
924
  private prunePersistedStates(): void {
@@ -842,6 +962,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
842
962
 
843
963
  return null;
844
964
  }
965
+
966
+ /** Tiered pruning is active with the default `Infinity` on-disk cap; a finite cap uses the flat path */
967
+ private get isTieredMode(): boolean {
968
+ return this.maxEpochsOnDisk === Infinity;
969
+ }
845
970
  }
846
971
 
847
972
  export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex {
@@ -29,7 +29,7 @@ export interface BlockStateCache {
29
29
  getSeedState(): IBeaconStateView;
30
30
  clear(): void;
31
31
  size: number;
32
- prune(headStateRootHex: RootHex): void;
32
+ onCheckpoint(headStateRootHex: RootHex): void;
33
33
  dumpSummary(): routes.lodestar.StateCacheItem[];
34
34
  /** Expose beacon states stored in cache. Use with caution */
35
35
  getStates(): IterableIterator<IBeaconStateView>;
@@ -65,9 +65,9 @@ export interface CheckpointStateCache {
65
65
  getLatest(rootHex: RootHex, maxEpoch: Epoch): IBeaconStateView | null;
66
66
  getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<IBeaconStateView | null>;
67
67
  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
68
- prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
68
+ onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
69
69
  processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number>;
70
- clear(): void;
70
+ clear(): Promise<void>;
71
71
  dumpSummary(): routes.lodestar.StateCacheItem[];
72
72
  /** Expose beacon states stored in cache. Use with caution */
73
73
  getStates(): IterableIterator<IBeaconStateView>;
@@ -313,11 +313,11 @@ async function validateExecutionPayloadBid(
313
313
  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
314
314
  // from builders submitting numerous bids with minimal value increments.
315
315
  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
316
- if (bestBid !== null && bid.value < getMinBidValue(bestBid.message.value)) {
316
+ if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
317
317
  throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
318
318
  code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
319
319
  bidValue: bid.value,
320
- currentHighestBid: bestBid.message.value,
320
+ currentHighestBid: bestBid.signedBid.message.value,
321
321
  });
322
322
  }
323
323
  // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
@@ -9,12 +9,69 @@ import {
9
9
  import {IBeaconChain} from "../index.js";
10
10
  import {RegenCaller} from "../regen/index.js";
11
11
 
12
+ export type ApiVoluntaryExitResult = {status: "published"} | {status: "deferred"; validity: VoluntaryExitValidity};
13
+
14
+ // Variants that can become valid in a future epoch without user action.
15
+ // - earlyEpoch: exit.message.epoch is in the future; valid once current epoch catches up
16
+ // - shortTimeActive: validator active < SHARD_COMMITTEE_PERIOD; valid once enough time passes
17
+ // - pendingWithdrawals: Electra; valid once pending partial withdrawals drain
18
+ // Note: VoluntaryExitValidity.inactive is intentionally excluded. It conflates
19
+ // "validator does not exist" (permanent) with "validator not yet activated"
20
+ // (transient), and cleanly classifying it requires splitting the enum variant
21
+ // upstream. Left for a future follow-up.
22
+
23
+ export function isTransientExitValidity(v: VoluntaryExitValidity): boolean {
24
+ switch (v) {
25
+ case VoluntaryExitValidity.earlyEpoch:
26
+ case VoluntaryExitValidity.shortTimeActive:
27
+ case VoluntaryExitValidity.pendingWithdrawals:
28
+ return true;
29
+ case VoluntaryExitValidity.valid:
30
+ case VoluntaryExitValidity.inactive:
31
+ case VoluntaryExitValidity.alreadyExited:
32
+ case VoluntaryExitValidity.invalidSignature:
33
+ return false;
34
+ }
35
+ }
36
+
37
+ // Comments for each call are present inside `validateVoluntaryExit`.
12
38
  export async function validateApiVoluntaryExit(
13
39
  chain: IBeaconChain,
14
40
  voluntaryExit: phase0.SignedVoluntaryExit
15
- ): Promise<void> {
41
+ ): Promise<ApiVoluntaryExitResult> {
16
42
  const prioritizeBls = true;
17
- return validateVoluntaryExit(chain, voluntaryExit, prioritizeBls);
43
+
44
+ if (
45
+ chain.opPool.hasSeenVoluntaryExit(voluntaryExit.message.validatorIndex) ||
46
+ chain.deferredVoluntaryExitPool.has(voluntaryExit.message.validatorIndex)
47
+ ) {
48
+ throw new VoluntaryExitError(GossipAction.IGNORE, {
49
+ code: VoluntaryExitErrorCode.ALREADY_EXISTS,
50
+ });
51
+ }
52
+
53
+ const state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.validateApiVoluntaryExit);
54
+ const validity = state.getVoluntaryExitValidity(voluntaryExit, false);
55
+
56
+ if (validity !== VoluntaryExitValidity.valid && !isTransientExitValidity(validity)) {
57
+ throw new VoluntaryExitError(GossipAction.REJECT, {
58
+ code: voluntaryExitValidityToErrorCode(validity),
59
+ });
60
+ }
61
+
62
+ const signatureSet = getVoluntaryExitSignatureSet(chain.config, state, voluntaryExit);
63
+ if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) {
64
+ throw new VoluntaryExitError(GossipAction.REJECT, {
65
+ code: VoluntaryExitErrorCode.INVALID_SIGNATURE,
66
+ });
67
+ }
68
+
69
+ if (validity !== VoluntaryExitValidity.valid) {
70
+ // Transient failure — signature is good, defer
71
+ return {status: "deferred", validity};
72
+ }
73
+
74
+ return {status: "published"};
18
75
  }
19
76
 
20
77
  export async function validateGossipVoluntaryExit(
@@ -0,0 +1,244 @@
1
+ import {routes} from "@lodestar/api";
2
+ import {ApiClient as BuilderApi, getClient} from "@lodestar/api/builder";
3
+ import {ChainForkConfig} from "@lodestar/config";
4
+ import {Logger} from "@lodestar/logger";
5
+ import {DOMAIN_BUILDER_REQUEST_AUTH, ForkPostGloas} from "@lodestar/params";
6
+ import {
7
+ ZERO_HASH,
8
+ computeDomain,
9
+ computeSigningRoot,
10
+ createSingleSignatureSetFromComponents,
11
+ } from "@lodestar/state-transition";
12
+ import {BLSPubkey, Root, SignedBeaconBlock, Slot, WithOptionalBytes, gloas, ssz} from "@lodestar/types";
13
+ import {isValidAsciiHttpUrl, toHex, toPrintableUrl} from "@lodestar/utils";
14
+ import type {IBlsVerifier} from "../../chain/bls/index.js";
15
+ import {Metrics} from "../../metrics/metrics.js";
16
+ import {IClock} from "../../util/clock.js";
17
+
18
+ /**
19
+ * Additional duration to account for potential event loop lag which causes
20
+ * builder bids to be rejected even though the response was sent in time.
21
+ */
22
+ const EVENT_LOOP_LAG_BUFFER = 250;
23
+
24
+ /**
25
+ * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
26
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
27
+ * selected after the same duration, only considering bids received up to that point.
28
+ */
29
+ export const BUILDER_BID_DEADLINE_MS = 500;
30
+
31
+ /** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
32
+ export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
33
+
34
+ export type BuilderApiClientOpts = {
35
+ /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
36
+ timeout?: number;
37
+ // Add User-Agent header to all requests
38
+ userAgent?: string;
39
+ };
40
+
41
+ type BuilderUrl = string;
42
+
43
+ /** Decode the SSZ URL bytes without allowing replacement characters or unsafe header values. */
44
+ export function decodeBuilderUrl(value: Uint8Array): BuilderUrl {
45
+ let url: string;
46
+ try {
47
+ url = new TextDecoder("utf8", {fatal: true}).decode(value);
48
+ } catch {
49
+ throw Error("Builder url must be valid UTF-8");
50
+ }
51
+ if (!isValidAsciiHttpUrl(url)) {
52
+ throw Error("Invalid builder url");
53
+ }
54
+ return url;
55
+ }
56
+
57
+ export type BuilderApiBid = {
58
+ url: BuilderUrl;
59
+ entry: routes.validator.BuilderEntry;
60
+ signedBid: gloas.SignedExecutionPayloadBid;
61
+ /** Time in milliseconds from the slot start when the bid was received */
62
+ receivedMs: number;
63
+ };
64
+
65
+ /**
66
+ * External builder integration post-gloas
67
+ *
68
+ * The builder set is driven by the resolved `BuilderConfig` the validator client supplies on
69
+ * each block production request, clients are dialed on demand based on the entry `url`.
70
+ */
71
+ export class BuilderApiClient {
72
+ private readonly clients = new Map<BuilderUrl, BuilderApi>();
73
+
74
+ constructor(
75
+ private readonly opts: BuilderApiClientOpts,
76
+ private readonly config: ChainForkConfig,
77
+ private readonly clock: IClock,
78
+ private readonly bls: IBlsVerifier,
79
+ private readonly metrics: Metrics | null = null,
80
+ private readonly logger?: Logger
81
+ ) {}
82
+
83
+ /**
84
+ * Fan out bid requests to the builders named by the entries, one request per unique
85
+ * `(url, auth.message.data)` pair. Errors and empty responses (204) are logged and filtered
86
+ * out so a single unresponsive builder never fails block production.
87
+ */
88
+ async getExecutionPayloadBids(
89
+ entries: routes.validator.BuilderEntry[],
90
+ slot: Slot,
91
+ parentHash: Root,
92
+ parentRoot: Root,
93
+ proposerPubkey: BLSPubkey
94
+ ): Promise<BuilderApiBid[]> {
95
+ const seenRequests = new Set<string>();
96
+ const requests: {url: BuilderUrl; entry: routes.validator.BuilderEntry}[] = [];
97
+
98
+ for (const entry of entries) {
99
+ const urlForLog = Buffer.from(entry.url).toString("utf8");
100
+ let url: BuilderUrl;
101
+ try {
102
+ url = decodeBuilderUrl(entry.url);
103
+ } catch {
104
+ this.logger?.warn("Ignoring builder entry with invalid url", {slot, url: urlForLog});
105
+ continue;
106
+ }
107
+
108
+ const requestKey = `${url}-${toHex(entry.auth.message.data)}`;
109
+ if (seenRequests.has(requestKey)) {
110
+ continue;
111
+ }
112
+ seenRequests.add(requestKey);
113
+
114
+ // The builder rejects a mismatch, an entry naming a different slot is not used for a bid request
115
+ if (entry.auth.message.slot !== slot) {
116
+ this.logger?.warn("Ignoring builder entry with auth for different slot", {
117
+ slot,
118
+ authSlot: entry.auth.message.slot,
119
+ builder: toPrintableUrl(url),
120
+ });
121
+ continue;
122
+ }
123
+
124
+ requests.push({url, entry});
125
+ }
126
+
127
+ // Collected in arrival order so an earlier received bid wins ties during candidate ranking
128
+ const bids: BuilderApiBid[] = [];
129
+ await Promise.all(
130
+ requests.map(async ({url, entry}) => {
131
+ this.metrics?.builderApi.bidRequests.inc();
132
+ try {
133
+ const client = await this.getOrCreateClient(url, proposerPubkey, entry.auth);
134
+ const res = await client.getExecutionPayloadBid(
135
+ {
136
+ slot,
137
+ parentHash,
138
+ parentRoot,
139
+ proposerPubkey,
140
+ requestAuth: entry.auth,
141
+ dateMilliseconds: Date.now(),
142
+ timeoutMs: BUILDER_BID_DEADLINE_MS,
143
+ },
144
+ {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
145
+ );
146
+ const signedBid = res.value();
147
+ if (signedBid === undefined) {
148
+ this.logger?.debug("No bid received from builder", {slot, builder: toPrintableUrl(url)});
149
+ return;
150
+ }
151
+ this.metrics?.builderApi.bidsReceived.inc();
152
+ bids.push({url, entry, signedBid, receivedMs: this.clock.msFromSlot(slot)});
153
+ } catch (e) {
154
+ this.metrics?.builderApi.bidRequestErrors.inc();
155
+ this.logger?.warn("Failed to get bid from builder", {slot, builder: toPrintableUrl(url)}, e as Error);
156
+ }
157
+ })
158
+ );
159
+
160
+ return bids;
161
+ }
162
+
163
+ /** Forward a proposer's builder preferences to the builder at the given url */
164
+ async submitBuilderPreferences(
165
+ url: BuilderUrl,
166
+ proposerPubkey: BLSPubkey,
167
+ request: gloas.BuilderPreferencesRequest
168
+ ): Promise<void> {
169
+ try {
170
+ const client = await this.getOrCreateClient(url, proposerPubkey, request.auth);
171
+ (await client.submitBuilderPreferences({proposerPubkey, request})).assertOk();
172
+ this.metrics?.builderApi.preferencesForwarded.inc({status: "success"});
173
+ } catch (e) {
174
+ this.metrics?.builderApi.preferencesForwarded.inc({status: "error"});
175
+ throw e;
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Submit the signed beacon block to the builder whose bid was included. The builder is
181
+ * then responsible for constructing and broadcasting the corresponding
182
+ * `SignedExecutionPayloadEnvelope`, no further action is required by the proposer.
183
+ */
184
+ async submitSignedBeaconBlock(
185
+ url: BuilderUrl,
186
+ signedBlock: WithOptionalBytes<SignedBeaconBlock<ForkPostGloas>>
187
+ ): Promise<void> {
188
+ const client = this.clients.get(url);
189
+ if (client === undefined) {
190
+ this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
191
+ this.logger?.warn("Ignoring signed block submission to unauthenticated builder", {url});
192
+ return;
193
+ }
194
+
195
+ try {
196
+ (await client.submitSignedBeaconBlock({signedBlock}, {redirect: "manual"})).assertOk();
197
+ this.metrics?.builderApi.blockSubmissions.inc({status: "success"});
198
+ } catch (e) {
199
+ this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
200
+ throw e;
201
+ }
202
+ }
203
+
204
+ private async getOrCreateClient(
205
+ url: BuilderUrl,
206
+ proposerPubkey: BLSPubkey,
207
+ auth: gloas.SignedBuilderRequestAuth
208
+ ): Promise<BuilderApi> {
209
+ if (auth.message.data.length === 0) {
210
+ throw Error("Builder request auth data must not be empty");
211
+ }
212
+
213
+ let client = this.clients.get(url);
214
+ if (client === undefined) {
215
+ const domain = computeDomain(DOMAIN_BUILDER_REQUEST_AUTH, this.config.GENESIS_FORK_VERSION, ZERO_HASH);
216
+ const signingRoot = computeSigningRoot(ssz.gloas.BuilderRequestAuth, auth.message, domain);
217
+ const signatureSet = createSingleSignatureSetFromComponents(proposerPubkey, signingRoot, auth.signature);
218
+
219
+ let isValid = false;
220
+ try {
221
+ isValid = await this.bls.verifySignatureSets([signatureSet]);
222
+ } catch {
223
+ // Malformed signatures are invalid request authentication.
224
+ }
225
+ if (!isValid) {
226
+ throw Error("Invalid builder request auth");
227
+ }
228
+
229
+ client = getClient(
230
+ {
231
+ baseUrl: url,
232
+ globalInit: {
233
+ timeoutMs: this.opts.timeout,
234
+ headers: this.opts.userAgent ? {"User-Agent": this.opts.userAgent} : undefined,
235
+ },
236
+ },
237
+ {config: this.config, metrics: this.metrics?.builderHttpClient, logger: this.logger}
238
+ );
239
+ this.clients.set(url, client);
240
+ this.logger?.info("External builder registered", {url: toPrintableUrl(url)});
241
+ }
242
+ return client;
243
+ }
244
+ }