@lodestar/beacon-node 1.50.0-dev.c04a499311 → 1.50.0-dev.c24762bc60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/lib/api/impl/beacon/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/index.js +13 -0
  3. package/lib/api/impl/beacon/index.js.map +1 -1
  4. package/lib/api/impl/beacon/state/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/state/index.js +28 -6
  6. package/lib/api/impl/beacon/state/index.js.map +1 -1
  7. package/lib/api/impl/debug/index.d.ts.map +1 -1
  8. package/lib/api/impl/debug/index.js +26 -18
  9. package/lib/api/impl/debug/index.js.map +1 -1
  10. package/lib/api/impl/validator/index.d.ts.map +1 -1
  11. package/lib/api/impl/validator/index.js +32 -0
  12. package/lib/api/impl/validator/index.js.map +1 -1
  13. package/lib/chain/archiveStore/archiveStore.d.ts +5 -0
  14. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  15. package/lib/chain/archiveStore/archiveStore.js +13 -23
  16. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  17. package/lib/chain/archiveStore/utils/pruneHistory.d.ts +1 -1
  18. package/lib/chain/archiveStore/utils/pruneHistory.d.ts.map +1 -1
  19. package/lib/chain/archiveStore/utils/pruneHistory.js +2 -1
  20. package/lib/chain/archiveStore/utils/pruneHistory.js.map +1 -1
  21. package/lib/chain/beaconProposerCache.d.ts +6 -0
  22. package/lib/chain/beaconProposerCache.d.ts.map +1 -1
  23. package/lib/chain/beaconProposerCache.js +16 -0
  24. package/lib/chain/beaconProposerCache.js.map +1 -1
  25. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  26. package/lib/chain/blocks/importBlock.js +14 -8
  27. package/lib/chain/blocks/importBlock.js.map +1 -1
  28. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  29. package/lib/chain/blocks/verifyBlock.js +6 -6
  30. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  31. package/lib/chain/chain.d.ts +5 -1
  32. package/lib/chain/chain.d.ts.map +1 -1
  33. package/lib/chain/chain.js +73 -20
  34. package/lib/chain/chain.js.map +1 -1
  35. package/lib/chain/interface.d.ts +3 -0
  36. package/lib/chain/interface.d.ts.map +1 -1
  37. package/lib/chain/opPools/aggregatedAttestationPool.d.ts +4 -2
  38. package/lib/chain/opPools/aggregatedAttestationPool.d.ts.map +1 -1
  39. package/lib/chain/opPools/aggregatedAttestationPool.js +30 -8
  40. package/lib/chain/opPools/aggregatedAttestationPool.js.map +1 -1
  41. package/lib/chain/opPools/opPool.d.ts.map +1 -1
  42. package/lib/chain/opPools/opPool.js +1 -23
  43. package/lib/chain/opPools/opPool.js.map +1 -1
  44. package/lib/chain/opPools/proposerPreferencesPool.d.ts +8 -4
  45. package/lib/chain/opPools/proposerPreferencesPool.d.ts.map +1 -1
  46. package/lib/chain/opPools/proposerPreferencesPool.js +27 -12
  47. package/lib/chain/opPools/proposerPreferencesPool.js.map +1 -1
  48. package/lib/chain/opPools/utils.d.ts +11 -0
  49. package/lib/chain/opPools/utils.d.ts.map +1 -1
  50. package/lib/chain/opPools/utils.js +22 -0
  51. package/lib/chain/opPools/utils.js.map +1 -1
  52. package/lib/chain/options.d.ts +1 -0
  53. package/lib/chain/options.d.ts.map +1 -1
  54. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  55. package/lib/chain/prepareNextSlot.js +11 -3
  56. package/lib/chain/prepareNextSlot.js.map +1 -1
  57. package/lib/chain/produceBlock/produceBlockBody.d.ts +8 -0
  58. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  59. package/lib/chain/produceBlock/produceBlockBody.js +40 -28
  60. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  61. package/lib/chain/seenCache/seenGossipBlockInput.d.ts +7 -0
  62. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  63. package/lib/chain/seenCache/seenGossipBlockInput.js +21 -0
  64. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  65. package/lib/chain/shufflingCache.d.ts +18 -8
  66. package/lib/chain/shufflingCache.d.ts.map +1 -1
  67. package/lib/chain/shufflingCache.js +72 -18
  68. package/lib/chain/shufflingCache.js.map +1 -1
  69. package/lib/db/beacon.d.ts +4 -1
  70. package/lib/db/beacon.d.ts.map +1 -1
  71. package/lib/db/beacon.js +6 -1
  72. package/lib/db/beacon.js.map +1 -1
  73. package/lib/db/buckets.d.ts +3 -1
  74. package/lib/db/buckets.d.ts.map +1 -1
  75. package/lib/db/buckets.js +2 -0
  76. package/lib/db/buckets.js.map +1 -1
  77. package/lib/db/interface.d.ts +4 -1
  78. package/lib/db/interface.d.ts.map +1 -1
  79. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  80. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  81. package/lib/db/repositories/earliestAvailableSlot.d.ts +11 -0
  82. package/lib/db/repositories/earliestAvailableSlot.d.ts.map +1 -0
  83. package/lib/db/repositories/earliestAvailableSlot.js +19 -0
  84. package/lib/db/repositories/earliestAvailableSlot.js.map +1 -0
  85. package/lib/db/repositories/index.d.ts +1 -0
  86. package/lib/db/repositories/index.d.ts.map +1 -1
  87. package/lib/db/repositories/index.js +1 -0
  88. package/lib/db/repositories/index.js.map +1 -1
  89. package/lib/db/repositories/proposerPreferences.d.ts +14 -0
  90. package/lib/db/repositories/proposerPreferences.d.ts.map +1 -0
  91. package/lib/db/repositories/proposerPreferences.js +21 -0
  92. package/lib/db/repositories/proposerPreferences.js.map +1 -0
  93. package/lib/execution/engine/blobVersionedHashes.d.ts +9 -0
  94. package/lib/execution/engine/blobVersionedHashes.d.ts.map +1 -0
  95. package/lib/execution/engine/blobVersionedHashes.js +74 -0
  96. package/lib/execution/engine/blobVersionedHashes.js.map +1 -0
  97. package/lib/execution/engine/http.d.ts +45 -13
  98. package/lib/execution/engine/http.d.ts.map +1 -1
  99. package/lib/execution/engine/http.js +236 -198
  100. package/lib/execution/engine/http.js.map +1 -1
  101. package/lib/execution/engine/index.d.ts +3 -2
  102. package/lib/execution/engine/index.d.ts.map +1 -1
  103. package/lib/execution/engine/index.js +33 -8
  104. package/lib/execution/engine/index.js.map +1 -1
  105. package/lib/execution/engine/interface.d.ts +1 -1
  106. package/lib/execution/engine/interface.d.ts.map +1 -1
  107. package/lib/execution/engine/jsonRpcTransport.d.ts +25 -0
  108. package/lib/execution/engine/jsonRpcTransport.d.ts.map +1 -0
  109. package/lib/execution/engine/jsonRpcTransport.js +158 -0
  110. package/lib/execution/engine/jsonRpcTransport.js.map +1 -0
  111. package/lib/execution/engine/restHttpClient.d.ts +71 -0
  112. package/lib/execution/engine/restHttpClient.d.ts.map +1 -0
  113. package/lib/execution/engine/restHttpClient.js +201 -0
  114. package/lib/execution/engine/restHttpClient.js.map +1 -0
  115. package/lib/execution/engine/restTransport.d.ts +46 -0
  116. package/lib/execution/engine/restTransport.d.ts.map +1 -0
  117. package/lib/execution/engine/restTransport.js +399 -0
  118. package/lib/execution/engine/restTransport.js.map +1 -0
  119. package/lib/execution/engine/sszTypes.d.ts +1261 -0
  120. package/lib/execution/engine/sszTypes.d.ts.map +1 -0
  121. package/lib/execution/engine/sszTypes.js +216 -0
  122. package/lib/execution/engine/sszTypes.js.map +1 -0
  123. package/lib/execution/engine/transport.d.ts +39 -0
  124. package/lib/execution/engine/transport.d.ts.map +1 -0
  125. package/lib/execution/engine/transport.js +2 -0
  126. package/lib/execution/engine/transport.js.map +1 -0
  127. package/lib/execution/engine/types.d.ts +2 -0
  128. package/lib/execution/engine/types.d.ts.map +1 -1
  129. package/lib/execution/engine/types.js +24 -57
  130. package/lib/execution/engine/types.js.map +1 -1
  131. package/lib/execution/engine/utils.d.ts.map +1 -1
  132. package/lib/execution/engine/utils.js +9 -2
  133. package/lib/execution/engine/utils.js.map +1 -1
  134. package/lib/metrics/metrics/lodestar.d.ts +10 -0
  135. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  136. package/lib/metrics/metrics/lodestar.js +22 -0
  137. package/lib/metrics/metrics/lodestar.js.map +1 -1
  138. package/lib/network/core/events.d.ts +5 -1
  139. package/lib/network/core/events.d.ts.map +1 -1
  140. package/lib/network/core/events.js +8 -0
  141. package/lib/network/core/events.js.map +1 -1
  142. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  143. package/lib/network/processor/gossipHandlers.js +60 -64
  144. package/lib/network/processor/gossipHandlers.js.map +1 -1
  145. package/lib/node/nodejs.d.ts +3 -1
  146. package/lib/node/nodejs.d.ts.map +1 -1
  147. package/lib/node/nodejs.js +2 -1
  148. package/lib/node/nodejs.js.map +1 -1
  149. package/lib/node/options.d.ts +3 -2
  150. package/lib/node/options.d.ts.map +1 -1
  151. package/lib/node/options.js +2 -2
  152. package/lib/node/options.js.map +1 -1
  153. package/lib/sync/range/batch.d.ts +5 -5
  154. package/lib/sync/range/batch.d.ts.map +1 -1
  155. package/lib/sync/range/batch.js +11 -17
  156. package/lib/sync/range/batch.js.map +1 -1
  157. package/lib/sync/range/chain.d.ts +4 -4
  158. package/lib/sync/range/chain.d.ts.map +1 -1
  159. package/lib/sync/range/chain.js +9 -5
  160. package/lib/sync/range/chain.js.map +1 -1
  161. package/lib/sync/range/range.d.ts.map +1 -1
  162. package/lib/sync/range/range.js +25 -11
  163. package/lib/sync/range/range.js.map +1 -1
  164. package/lib/sync/unknownBlock.d.ts +5 -2
  165. package/lib/sync/unknownBlock.d.ts.map +1 -1
  166. package/lib/sync/unknownBlock.js +44 -20
  167. package/lib/sync/unknownBlock.js.map +1 -1
  168. package/lib/sync/utils/downloadByRange.d.ts +38 -14
  169. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  170. package/lib/sync/utils/downloadByRange.js +84 -39
  171. package/lib/sync/utils/downloadByRange.js.map +1 -1
  172. package/lib/util/asyncIterableToEvents.d.ts +10 -2
  173. package/lib/util/asyncIterableToEvents.d.ts.map +1 -1
  174. package/lib/util/asyncIterableToEvents.js +29 -3
  175. package/lib/util/asyncIterableToEvents.js.map +1 -1
  176. package/lib/util/metadata.d.ts.map +1 -1
  177. package/lib/util/metadata.js +2 -1
  178. package/lib/util/metadata.js.map +1 -1
  179. package/package.json +17 -16
  180. package/src/api/impl/beacon/index.ts +15 -0
  181. package/src/api/impl/beacon/state/index.ts +43 -6
  182. package/src/api/impl/debug/index.ts +28 -19
  183. package/src/api/impl/validator/index.ts +45 -0
  184. package/src/chain/archiveStore/archiveStore.ts +18 -24
  185. package/src/chain/archiveStore/utils/pruneHistory.ts +4 -1
  186. package/src/chain/beaconProposerCache.ts +19 -1
  187. package/src/chain/blocks/importBlock.ts +17 -7
  188. package/src/chain/blocks/verifyBlock.ts +7 -6
  189. package/src/chain/chain.ts +97 -19
  190. package/src/chain/interface.ts +3 -0
  191. package/src/chain/opPools/aggregatedAttestationPool.ts +34 -10
  192. package/src/chain/opPools/opPool.ts +1 -31
  193. package/src/chain/opPools/proposerPreferencesPool.ts +34 -11
  194. package/src/chain/opPools/utils.ts +31 -0
  195. package/src/chain/options.ts +1 -1
  196. package/src/chain/prepareNextSlot.ts +15 -3
  197. package/src/chain/produceBlock/produceBlockBody.ts +68 -34
  198. package/src/chain/seenCache/seenGossipBlockInput.ts +23 -0
  199. package/src/chain/shufflingCache.ts +84 -22
  200. package/src/db/beacon.ts +6 -0
  201. package/src/db/buckets.ts +2 -0
  202. package/src/db/interface.ts +4 -0
  203. package/src/db/repositories/earliestAvailableSlot.ts +19 -0
  204. package/src/db/repositories/index.ts +1 -0
  205. package/src/db/repositories/proposerPreferences.ts +23 -0
  206. package/src/execution/engine/blobVersionedHashes.ts +72 -0
  207. package/src/execution/engine/http.ts +304 -271
  208. package/src/execution/engine/index.ts +41 -7
  209. package/src/execution/engine/interface.ts +1 -1
  210. package/src/execution/engine/jsonRpcTransport.ts +247 -0
  211. package/src/execution/engine/restHttpClient.ts +269 -0
  212. package/src/execution/engine/restTransport.ts +544 -0
  213. package/src/execution/engine/sszTypes.ts +354 -0
  214. package/src/execution/engine/transport.ts +61 -0
  215. package/src/execution/engine/types.ts +34 -74
  216. package/src/execution/engine/utils.ts +12 -2
  217. package/src/metrics/metrics/lodestar.ts +23 -0
  218. package/src/network/core/events.ts +16 -1
  219. package/src/network/processor/gossipHandlers.ts +91 -77
  220. package/src/node/nodejs.ts +4 -0
  221. package/src/node/options.ts +10 -1
  222. package/src/sync/range/batch.ts +14 -19
  223. package/src/sync/range/chain.ts +12 -7
  224. package/src/sync/range/range.ts +26 -12
  225. package/src/sync/unknownBlock.ts +57 -21
  226. package/src/sync/utils/downloadByRange.ts +123 -58
  227. package/src/util/asyncIterableToEvents.ts +44 -5
  228. package/src/util/metadata.ts +2 -1
@@ -128,7 +128,7 @@ import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
128
128
  import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
129
129
  import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
130
130
  import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
131
- import {ShufflingCache} from "./shufflingCache.js";
131
+ import {ShufflingCache, ShufflingPromiseCancelReason} from "./shufflingCache.js";
132
132
  import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
133
133
  import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
134
134
  import {CPStateDatastore} from "./stateCache/datastore/types.js";
@@ -244,6 +244,7 @@ export class BeaconChain implements IBeaconChain {
244
244
  protected readonly blockProcessor: BlockProcessor;
245
245
  protected readonly payloadEnvelopeProcessor: PayloadEnvelopeProcessor;
246
246
  protected readonly db: IBeaconDb;
247
+ private genesisStateRoot?: RootHex | null;
247
248
  // this is only available if nHistoricalStates is enabled
248
249
  private readonly cpStateDatastore?: CPStateDatastore;
249
250
  private abortController = new AbortController();
@@ -278,6 +279,7 @@ export class BeaconChain implements IBeaconChain {
278
279
  validatorMonitor,
279
280
  anchorState,
280
281
  isAnchorStateFinalized,
282
+ earliestAvailableSlot,
281
283
  executionEngine,
282
284
  executionBuilder,
283
285
  builderApiClientOpts,
@@ -297,6 +299,7 @@ export class BeaconChain implements IBeaconChain {
297
299
  validatorMonitor: ValidatorMonitor | null;
298
300
  anchorState: IBeaconStateView;
299
301
  isAnchorStateFinalized: boolean;
302
+ earliestAvailableSlot: Slot;
300
303
  executionEngine: IExecutionEngine;
301
304
  executionBuilder?: IExecutionBuilder;
302
305
  builderApiClientOpts?: BuilderApiClientOpts;
@@ -325,7 +328,7 @@ export class BeaconChain implements IBeaconChain {
325
328
 
326
329
  this.blacklistedBlocks = new Map((opts.blacklistedBlocks ?? []).map((hex) => [hex, null]));
327
330
  this.attestationPool = new AttestationPool(config, clock, this.opts?.preaggregateSlotDistance, metrics);
328
- this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics);
331
+ this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics, logger);
329
332
  this.syncCommitteeMessagePool = new SyncCommitteeMessagePool(config, clock, this.opts?.preaggregateSlotDistance);
330
333
  this.syncContributionAndProofPool = new SyncContributionAndProofPool(config, clock, metrics, logger);
331
334
  this.executionPayloadBidPool = new ExecutionPayloadBidPool();
@@ -362,7 +365,7 @@ export class BeaconChain implements IBeaconChain {
362
365
  logger,
363
366
  });
364
367
 
365
- this._earliestAvailableSlot = anchorState.slot;
368
+ this._earliestAvailableSlot = earliestAvailableSlot;
366
369
 
367
370
  this.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [
368
371
  {
@@ -573,7 +576,11 @@ export class BeaconChain implements IBeaconChain {
573
576
  }
574
577
 
575
578
  seenBlock(blockRoot: RootHex): boolean {
576
- return this.seenBlockInputCache.hasBlock(blockRoot) || this.forkChoice.hasBlockHexUnsafe(blockRoot);
579
+ return (
580
+ this.seenBlockInputCache.hasBlock(blockRoot) ||
581
+ this.seenBlockInputCache.isValidatingBlock(blockRoot) ||
582
+ this.forkChoice.hasBlockHexUnsafe(blockRoot)
583
+ );
577
584
  }
578
585
 
579
586
  seenPayloadEnvelope(blockRoot: RootHex): boolean {
@@ -613,12 +620,14 @@ export class BeaconChain implements IBeaconChain {
613
620
  async loadFromDisk(): Promise<void> {
614
621
  await this.regen.init();
615
622
  await this.opPool.fromPersisted(this.db, this.getHeadState(), this.bls, this.clock.currentSlot);
623
+ await this.proposerPreferencesPool.fromPersisted(this.db, this.clock.currentSlot);
616
624
  }
617
625
 
618
626
  /** Persist in-memory data to the DB. Call at least once before stopping the process */
619
627
  async persistToDisk(): Promise<void> {
620
628
  await this.archiveStore.persistToDisk();
621
629
  await this.opPool.toPersisted(this.db);
630
+ await this.proposerPreferencesPool.toPersisted(this.db);
622
631
  }
623
632
 
624
633
  getHeadState(): IBeaconStateView {
@@ -723,6 +732,17 @@ export class BeaconChain implements IBeaconChain {
723
732
  }
724
733
  }
725
734
 
735
+ if (this.genesisStateRoot === undefined) {
736
+ const genesisBlock = await this.db.blockArchive.get(GENESIS_SLOT);
737
+ this.genesisStateRoot = genesisBlock ? toRootHex(genesisBlock.message.stateRoot) : null;
738
+ }
739
+ if (this.genesisStateRoot === stateRoot) {
740
+ const state = await this.db.stateArchive.getBinary(GENESIS_SLOT);
741
+ if (state) {
742
+ return {state, executionOptimistic: false, finalized: finalizedCheckpoint.epoch !== GENESIS_EPOCH};
743
+ }
744
+ }
745
+
726
746
  if (opts?.allowRegen) {
727
747
  const state = await this.regen.getState(stateRoot, RegenCaller.restApi);
728
748
  const block = this.forkChoice.getBlockDefaultStatus(
@@ -1524,6 +1544,22 @@ export class BeaconChain implements IBeaconChain {
1524
1544
  }
1525
1545
  }
1526
1546
 
1547
+ persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void {
1548
+ const blockRoot = toRootHex(envelope.beaconBlockRoot);
1549
+ void this.persistSszObject(
1550
+ "ExecutionPayloadEnvelope",
1551
+ ssz.gloas.ExecutionPayloadEnvelope.serialize(envelope),
1552
+ blockRoot,
1553
+ "produced_execution_payload_envelope"
1554
+ ).catch((e) => {
1555
+ this.logger.error(
1556
+ "Error persisting produced execution payload envelope",
1557
+ {slot: envelope.payload.slotNumber, blockRoot},
1558
+ e as Error
1559
+ );
1560
+ });
1561
+ }
1562
+
1527
1563
  /**
1528
1564
  * Invalid state root error is critical and it causes the node to stale most of the time so we want to always
1529
1565
  * persist preState, postState and block for further investigation.
@@ -1595,21 +1631,48 @@ export class BeaconChain implements IBeaconChain {
1595
1631
  const blockEpoch = computeEpochAtSlot(attHeadBlock.slot);
1596
1632
 
1597
1633
  let state: IBeaconStateView;
1598
- if (blockEpoch < attEpoch - 1) {
1599
- // thanks to one epoch look ahead, we don't need to dial up to attEpoch
1600
- const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
1601
- this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
1602
- state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
1603
- } else if (blockEpoch > attEpoch) {
1604
- // should not happen, handled inside attestation verification code
1605
- throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
1606
- } else {
1607
- // should use either current or next shuffling of head state
1608
- // it's not likely to hit this since these shufflings are cached already
1609
- // so handle just in case
1610
- this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
1611
- state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
1634
+ try {
1635
+ if (blockEpoch < attEpoch - 1) {
1636
+ // thanks to one epoch look ahead, we don't need to dial up to attEpoch
1637
+ const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
1638
+ this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
1639
+ state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
1640
+ } else if (blockEpoch > attEpoch) {
1641
+ // should not happen, handled inside attestation verification code
1642
+ throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
1643
+ } else {
1644
+ // should use either current or next shuffling of head state
1645
+ // it's not likely to hit this since these shufflings are cached already
1646
+ // so handle just in case
1647
+ this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
1648
+ state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
1649
+ }
1650
+ } catch (e) {
1651
+ this.shufflingCache.cancelPromise(attEpoch, shufflingDependentRoot, ShufflingPromiseCancelReason.regenError);
1652
+ throw e;
1612
1653
  }
1654
+
1655
+ const stateDecisionRoot = state.getShufflingDecisionRoot(attEpoch);
1656
+ if (stateDecisionRoot !== shufflingDependentRoot) {
1657
+ // the promise would never be resolved by the state's shuffling
1658
+ this.shufflingCache.cancelPromise(
1659
+ attEpoch,
1660
+ shufflingDependentRoot,
1661
+ ShufflingPromiseCancelReason.decisionRootMismatch
1662
+ );
1663
+ this.logger.debug("Shuffling decision root mismatch on attestation regen", {
1664
+ slot: attHeadBlock.slot,
1665
+ root: attHeadBlock.blockRoot,
1666
+ epoch: attEpoch,
1667
+ shufflingDependentRoot,
1668
+ stateDecisionRoot,
1669
+ });
1670
+
1671
+ throw Error(
1672
+ `Shuffling decision root mismatch epoch=${attEpoch} slot=${attHeadBlock.slot} root=${attHeadBlock.blockRoot} expected=${shufflingDependentRoot} actual=${stateDecisionRoot}`
1673
+ );
1674
+ }
1675
+
1613
1676
  // resolve the promise to unblock other calls of the same epoch and dependent root
1614
1677
  this.shufflingCache.processState(state);
1615
1678
  return state.getShufflingAtEpoch(attEpoch);
@@ -1719,7 +1782,7 @@ export class BeaconChain implements IBeaconChain {
1719
1782
  // remove date suffixes in file name, and check duplicate to avoid redundant persistence
1720
1783
  await writeIfNotExist(filepath, bytes);
1721
1784
 
1722
- this.logger.debug("Persisted invalid ssz object", {id: logStr, filepath});
1785
+ this.logger.debug("Persisted SSZ object", {id: logStr, filepath});
1723
1786
  }
1724
1787
 
1725
1788
  private onScrapeMetrics(metrics: Metrics): void {
@@ -1873,6 +1936,21 @@ export class BeaconChain implements IBeaconChain {
1873
1936
  }
1874
1937
  }
1875
1938
 
1939
+ async updateAttachedValidators(epoch: Epoch, validatorIndices: ValidatorIndex[]): Promise<void> {
1940
+ let hasNewValidators = false;
1941
+ for (const validatorIndex of validatorIndices) {
1942
+ if (this.beaconProposerCache.track(epoch, validatorIndex)) {
1943
+ hasNewValidators = true;
1944
+ }
1945
+ }
1946
+
1947
+ // Only update validator custody if we discovered new validators
1948
+ if (hasNewValidators) {
1949
+ const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint();
1950
+ await this.updateValidatorsCustodyRequirement(finalizedCheckpoint);
1951
+ }
1952
+ }
1953
+
1876
1954
  private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise<void> {
1877
1955
  if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) {
1878
1956
  // Custody requirements can only be increased, we can disable dynamic custody updates
@@ -288,8 +288,11 @@ export interface IBeaconChain {
288
288
  waitForBlock(slot: Slot, root: RootHex): Promise<boolean>;
289
289
 
290
290
  updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void>;
291
+ /** Track validators that use this node for duties, e.g. by subscribing to attestation subnets */
292
+ updateAttachedValidators(epoch: Epoch, validatorIndices: ValidatorIndex[]): Promise<void>;
291
293
 
292
294
  persistBlock(data: BeaconBlock | BlindedBeaconBlock, suffix?: string): void;
295
+ persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void;
293
296
  persistInvalidStateRoot(
294
297
  preState: IBeaconStateView,
295
298
  postState: IBeaconStateView,
@@ -30,7 +30,7 @@ import {
30
30
  isStatePostGloas,
31
31
  } from "@lodestar/state-transition";
32
32
  import {Attestation, Epoch, RootHex, Slot, electra, isElectraAttestation, phase0, ssz} from "@lodestar/types";
33
- import {MapDef, assert, toRootHex} from "@lodestar/utils";
33
+ import {Logger, MapDef, assert, toRootHex} from "@lodestar/utils";
34
34
  import {Metrics} from "../../metrics/metrics.js";
35
35
  import {IntersectResult, intersectUint8Arrays} from "../../util/bitArray.js";
36
36
  import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
@@ -58,7 +58,7 @@ export type AttestationsConsolidation = {
58
58
 
59
59
  /**
60
60
  * This function returns not seen participation for a given epoch and slot and committee index.
61
- * Return null if all validators are seen or no info to check.
61
+ * Return an empty set if all validators are seen, or null if the committee cannot be computed (e.g. missing shuffling).
62
62
  */
63
63
  type GetNotSeenValidatorsFn = (epoch: Epoch, slot: Slot, committeeIndex: number) => Set<number> | null;
64
64
 
@@ -140,7 +140,8 @@ export class AggregatedAttestationPool {
140
140
 
141
141
  constructor(
142
142
  private readonly config: BeaconConfig,
143
- private readonly metrics: Metrics | null = null
143
+ private readonly metrics: Metrics | null = null,
144
+ private readonly logger: Logger | null = null
144
145
  ) {
145
146
  metrics?.opPool.aggregatedAttestationPool.attDataPerSlot.addCollect(() => this.onScrapeMetrics(metrics));
146
147
  }
@@ -244,6 +245,7 @@ export class AggregatedAttestationPool {
244
245
  const consolidations = new Map<AttestationsConsolidation, number>();
245
246
  let scannedSlots = 0;
246
247
  let stopReason: ScannedSlotsTerminationReason | null = null;
248
+ const missingShufflingCommitteesByEpoch = new Map<Epoch, number>();
247
249
  slot: for (const slot of slots) {
248
250
  const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.get(slot);
249
251
  // should not happen
@@ -295,7 +297,12 @@ export class AggregatedAttestationPool {
295
297
 
296
298
  for (const [committeeIndex, attestationGroup] of attestationGroupByIndex.entries()) {
297
299
  const notSeenCommitteeMembers = notSeenValidatorsFn(epoch, slot, committeeIndex);
298
- if (notSeenCommitteeMembers === null || notSeenCommitteeMembers.size === 0) {
300
+ if (notSeenCommitteeMembers === null) {
301
+ // skip this committee instead of failing the whole block production, other epochs may still have shuffling
302
+ missingShufflingCommitteesByEpoch.set(epoch, (missingShufflingCommitteesByEpoch.get(epoch) ?? 0) + 1);
303
+ continue;
304
+ }
305
+ if (notSeenCommitteeMembers.size === 0) {
299
306
  this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.seenCommittees.inc();
300
307
  continue;
301
308
  }
@@ -386,6 +393,18 @@ export class AggregatedAttestationPool {
386
393
  scannedSlots++;
387
394
  }
388
395
 
396
+ for (const [epoch, missingCommittees] of missingShufflingCommitteesByEpoch) {
397
+ this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.missingShufflingCommittees.inc(
398
+ missingCommittees
399
+ );
400
+ this.logger?.debug("Missing shuffling when packing attestations, skipped committees", {
401
+ slot: stateSlot,
402
+ epoch,
403
+ decisionRoot: state.getShufflingDecisionRoot(epoch),
404
+ missingCommittees,
405
+ });
406
+ }
407
+
389
408
  this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.totalConsolidations.set(consolidations.size);
390
409
 
391
410
  const sortedConsolidationsByScore = Array.from(consolidations.entries())
@@ -770,13 +789,18 @@ export function getNotSeenValidatorsFn(
770
789
  }
771
790
  const cacheKey = slot + "_" + committeeIndex;
772
791
  let notSeenCommitteeMembers = cachedNotSeenValidators.get(cacheKey);
773
- if (notSeenCommitteeMembers != null) {
774
- // if all validators are seen then return null, we don't need to check for any attestations of same committee again
775
- return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
792
+ if (notSeenCommitteeMembers !== undefined) {
793
+ return notSeenCommitteeMembers;
776
794
  }
777
795
 
778
796
  const decisionRoot = state.getShufflingDecisionRoot(computeEpochAtSlot(slot));
779
- const committee = shufflingCache.getBeaconCommittee(epoch, decisionRoot, slot, committeeIndex);
797
+ const committee = shufflingCache.getBeaconCommitteeOrNull(epoch, decisionRoot, slot, committeeIndex);
798
+ if (committee === null) {
799
+ // attestations in the pool were validated against a cached shuffling of the previous or current epoch,
800
+ // so a miss is a ShufflingCache bug. Return null so the caller tracks it via metrics/logs without failing
801
+ // block production. Don't fall back to the state's shuffling, that would hide the bug
802
+ return null;
803
+ }
780
804
  notSeenCommitteeMembers = new Set<number>();
781
805
  for (const [i, validatorIndex] of committee.entries()) {
782
806
  // no need to check flagIsTimelySource as if validator is not seen, it's participation status is 0
@@ -786,8 +810,8 @@ export function getNotSeenValidatorsFn(
786
810
  }
787
811
  }
788
812
  cachedNotSeenValidators.set(cacheKey, notSeenCommitteeMembers);
789
- // if all validators are seen then return null, we don't need to check for any attestations of same committee again
790
- return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
813
+ // an empty set means all validators are seen, we don't need to check for any attestations of same committee again
814
+ return notSeenCommitteeMembers;
791
815
  };
792
816
  }
793
817
 
@@ -1,5 +1,4 @@
1
1
  import {BeaconConfig} from "@lodestar/config";
2
- import {DbBatch, Id, Repository} from "@lodestar/db";
3
2
  import {
4
3
  BLS_WITHDRAWAL_PREFIX,
5
4
  ForkName,
@@ -42,6 +41,7 @@ import {
42
41
  getAttesterSlashingSignatureDomains,
43
42
  getProposerSlashingSignatureDomain,
44
43
  isValidBlsToExecutionChangeForBlockInclusion,
44
+ persistDiff,
45
45
  } from "./utils.js";
46
46
 
47
47
  type HexRoot = string;
@@ -500,33 +500,3 @@ function isVoluntaryExitSignatureIncludable(stateFork: ForkSeq, voluntaryExitFor
500
500
  function isSlashableAtEpoch(validator: phase0.Validator, epoch: Epoch): boolean {
501
501
  return !validator.slashed && validator.activationEpoch <= epoch && epoch < validator.withdrawableEpoch;
502
502
  }
503
-
504
- /**
505
- * Persist target items `items` in `dbRepo` doing minimum put and delete writes.
506
- * Reads all keys in repository to compute the diff between current persisted data and target data.
507
- */
508
- async function persistDiff<K extends Id, V>(
509
- dbRepo: Repository<K, V>,
510
- items: {key: K; value: V}[],
511
- serializeKey: (key: K) => number | string,
512
- opts: {updateExisting?: boolean} = {}
513
- ): Promise<void> {
514
- const persistedKeys = await dbRepo.keys();
515
- const batch: DbBatch<K, V> = [];
516
-
517
- const persistedKeysSerialized = new Set(persistedKeys.map(serializeKey));
518
- for (const item of items) {
519
- if (opts.updateExisting || !persistedKeysSerialized.has(serializeKey(item.key))) {
520
- batch.push({type: "put", key: item.key, value: item.value});
521
- }
522
- }
523
-
524
- const targetKeysSerialized = new Set(items.map((item) => serializeKey(item.key)));
525
- for (const persistedKey of persistedKeys) {
526
- if (!targetKeysSerialized.has(serializeKey(persistedKey))) {
527
- batch.push({type: "del", key: persistedKey});
528
- }
529
- }
530
-
531
- if (batch.length > 0) await dbRepo.batch(batch);
532
- }
@@ -1,13 +1,15 @@
1
1
  import {RootHex, Slot, ValidatorIndex, gloas} from "@lodestar/types";
2
- import {toRootHex} from "@lodestar/utils";
2
+ import {toHex, toRootHex} from "@lodestar/utils";
3
+ import {IBeaconDb} from "../../db/index.js";
4
+ import {persistDiff} from "./utils.js";
3
5
 
4
6
  /**
5
7
  * Pool of validated `SignedProposerPreferences` indexed by `(slot, dependent_root)`.
6
8
  *
7
9
  * The primary consumer is `validateExecutionPayloadBid`, which looks up the matching
8
10
  * preferences via `get(bid.slot, dependent_root)` to enforce the IGNORE-existence and
9
- * REJECT-equality rules from the gloas spec. The beacon API `/pool/proposer_preferences`
10
- * GET endpoint reads from the same pool via `getAll`.
11
+ * REJECT-equality rules from the gloas spec. The beacon API `getProposerPreferences`
12
+ * endpoint reads from the same pool via `getAll`.
11
13
  *
12
14
  * `validator_index` is intentionally not part of the key: gossip validation enforces
13
15
  * `proposers[proposalSlot % SLOTS_PER_EPOCH] === validatorIndex` against the shuffling
@@ -37,15 +39,19 @@ export class ProposerPreferencesPool {
37
39
  byRoot.set(rootHex, signed);
38
40
  }
39
41
 
40
- /** API read-out: flatten across branches, optionally filtered by slot. */
41
- getAll(slot?: Slot): gloas.SignedProposerPreferences[] {
42
- if (slot !== undefined) {
43
- const byRoot = this.bySlot.get(slot);
44
- return byRoot ? Array.from(byRoot.values()) : [];
45
- }
42
+ /** API read-out: flatten across branches, optionally filtered by slot and dependent root. */
43
+ getAll(slot?: Slot, dependentRootHex?: RootHex): gloas.SignedProposerPreferences[] {
46
44
  const out: gloas.SignedProposerPreferences[] = [];
47
- for (const byRoot of this.bySlot.values()) {
48
- for (const v of byRoot.values()) out.push(v);
45
+ for (const [s, byRoot] of this.bySlot) {
46
+ if (slot !== undefined && s !== slot) {
47
+ continue;
48
+ }
49
+ for (const [root, signed] of byRoot) {
50
+ if (dependentRootHex !== undefined && root !== dependentRootHex) {
51
+ continue;
52
+ }
53
+ out.push(signed);
54
+ }
49
55
  }
50
56
  return out;
51
57
  }
@@ -60,4 +66,21 @@ export class ProposerPreferencesPool {
60
66
  if (slot < currentSlot) this.bySlot.delete(slot);
61
67
  }
62
68
  }
69
+
70
+ /** Restore the preferences of upcoming slots, they were validated before they were persisted */
71
+ async fromPersisted(db: IBeaconDb, currentSlot: Slot): Promise<void> {
72
+ for (const signed of await db.proposerPreferences.values()) {
73
+ if (signed.message.proposalSlot >= currentSlot) {
74
+ this.add(signed);
75
+ }
76
+ }
77
+ }
78
+
79
+ async toPersisted(db: IBeaconDb): Promise<void> {
80
+ await persistDiff(
81
+ db.proposerPreferences,
82
+ this.getAll().map((signed) => ({key: db.proposerPreferences.getId(signed), value: signed})),
83
+ toHex
84
+ );
85
+ }
63
86
  }
@@ -1,5 +1,6 @@
1
1
  import {Signature} from "@chainsafe/lodestar-z/blst";
2
2
  import {BeaconConfig} from "@lodestar/config";
3
+ import {DbBatch, Id, Repository} from "@lodestar/db";
3
4
  import {BLS_WITHDRAWAL_PREFIX, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_PROPOSER} from "@lodestar/params";
4
5
  import {IBeaconStateView, computeStartSlotAtEpoch} from "@lodestar/state-transition";
5
6
  import {AttesterSlashing, Domain, Slot, capella, phase0} from "@lodestar/types";
@@ -91,3 +92,33 @@ export function isValidBlsToExecutionChangeForBlockInclusion(
91
92
  export function isElectraAggregate(aggregate: AggregateFast): aggregate is AggregateFastElectra {
92
93
  return (aggregate as AggregateFastElectra).committeeBits !== undefined;
93
94
  }
95
+
96
+ /**
97
+ * Persist target items `items` in `dbRepo` doing minimum put and delete writes.
98
+ * Reads all keys in repository to compute the diff between current persisted data and target data.
99
+ */
100
+ export async function persistDiff<K extends Id, V>(
101
+ dbRepo: Repository<K, V>,
102
+ items: {key: K; value: V}[],
103
+ serializeKey: (key: K) => number | string,
104
+ opts: {updateExisting?: boolean} = {}
105
+ ): Promise<void> {
106
+ const persistedKeys = await dbRepo.keys();
107
+ const batch: DbBatch<K, V> = [];
108
+
109
+ const persistedKeysSerialized = new Set(persistedKeys.map(serializeKey));
110
+ for (const item of items) {
111
+ if (opts.updateExisting || !persistedKeysSerialized.has(serializeKey(item.key))) {
112
+ batch.push({type: "put", key: item.key, value: item.value});
113
+ }
114
+ }
115
+
116
+ const targetKeysSerialized = new Set(items.map((item) => serializeKey(item.key)));
117
+ for (const persistedKey of persistedKeys) {
118
+ if (!targetKeysSerialized.has(serializeKey(persistedKey))) {
119
+ batch.push({type: "del", key: persistedKey});
120
+ }
121
+ }
122
+
123
+ if (batch.length > 0) await dbRepo.batch(batch);
124
+ }
@@ -27,8 +27,8 @@ export type IChainOptions = BlockProcessOpts &
27
27
  blsVerifyAllMainThread?: boolean;
28
28
  blsVerifyAllMultiThread?: boolean;
29
29
  blacklistedBlocks?: string[];
30
- // TODO GLOAS: add similar option for execution payload envelopes?
31
30
  persistProducedBlocks?: boolean;
31
+ persistProducedPayloadEnvelopes?: boolean;
32
32
  /** Archive finalized Gloas payload envelopes as headers (bodies reconstructed from the EL on read) */
33
33
  dedupePayloads?: boolean;
34
34
  persistInvalidSszObjects?: boolean;
@@ -22,7 +22,7 @@ import {
22
22
  isStatePostGloas,
23
23
  } from "@lodestar/state-transition";
24
24
  import {Bytes32, Slot, ValidatorIndex} from "@lodestar/types";
25
- import {Logger, fromHex, isErrorAborted, sleep} from "@lodestar/utils";
25
+ import {Logger, fromHex, isErrorAborted, sleep, toHex} from "@lodestar/utils";
26
26
  import {GENESIS_SLOT} from "../constants/constants.js";
27
27
  import {BuilderStatus} from "../execution/builder/http.js";
28
28
  import {PayloadAttributes} from "../execution/index.js";
@@ -31,7 +31,11 @@ import {ClockEvent} from "../util/clock.js";
31
31
  import {isQueueErrorAborted} from "../util/queue/index.js";
32
32
  import {ForkchoiceCaller} from "./forkChoice/index.js";
33
33
  import {IBeaconChain} from "./interface.js";
34
- import {getPayloadAttributesForSSE, prepareExecutionPayload} from "./produceBlock/produceBlockBody.js";
34
+ import {
35
+ getPayloadAttributesForSSE,
36
+ getPooledProposerPreferences,
37
+ prepareExecutionPayload,
38
+ } from "./produceBlock/produceBlockBody.js";
35
39
  import {RegenCaller} from "./regen/index.js";
36
40
 
37
41
  // TODO GLOAS: re-evaluate this timing
@@ -158,7 +162,15 @@ export class PrepareNextSlotScheduler {
158
162
  };
159
163
 
160
164
  const proposerIndex = await getProposerIndex();
161
- feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
165
+ if (this.chain.beaconProposerCache.has(proposerIndex)) {
166
+ const pooledPreferences = isForkPostGloas(fork)
167
+ ? getPooledProposerPreferences(this.chain, prepareSlot, headRoot)
168
+ : null;
169
+ feeRecipient =
170
+ pooledPreferences !== null
171
+ ? toHex(pooledPreferences.feeRecipient)
172
+ : this.chain.beaconProposerCache.getOrDefault(proposerIndex);
173
+ }
162
174
 
163
175
  // Predict the proposer head of the next slot when either:
164
176
  // - we are proposing the next slot (single-slot proposer-boost-reorg), or