@lodestar/beacon-node 1.42.0-dev.b10dfaca8d → 1.42.0-dev.bc0be71fb0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +24 -12
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/debug/index.js.map +1 -1
  5. package/lib/chain/ColumnReconstructionTracker.d.ts +2 -1
  6. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  7. package/lib/chain/ColumnReconstructionTracker.js +5 -5
  8. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  9. package/lib/chain/GetBlobsTracker.d.ts +2 -1
  10. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  11. package/lib/chain/GetBlobsTracker.js +14 -12
  12. package/lib/chain/GetBlobsTracker.js.map +1 -1
  13. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  14. package/lib/chain/archiveStore/archiveStore.js +1 -0
  15. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  16. package/lib/chain/archiveStore/historicalState/getHistoricalState.d.ts +3 -3
  17. package/lib/chain/archiveStore/historicalState/getHistoricalState.d.ts.map +1 -1
  18. package/lib/chain/archiveStore/historicalState/getHistoricalState.js +6 -4
  19. package/lib/chain/archiveStore/historicalState/getHistoricalState.js.map +1 -1
  20. package/lib/chain/archiveStore/historicalState/historicalStateRegen.d.ts +2 -2
  21. package/lib/chain/archiveStore/historicalState/historicalStateRegen.d.ts.map +1 -1
  22. package/lib/chain/archiveStore/historicalState/historicalStateRegen.js +1 -0
  23. package/lib/chain/archiveStore/historicalState/historicalStateRegen.js.map +1 -1
  24. package/lib/chain/archiveStore/historicalState/types.d.ts +2 -0
  25. package/lib/chain/archiveStore/historicalState/types.d.ts.map +1 -1
  26. package/lib/chain/archiveStore/historicalState/types.js.map +1 -1
  27. package/lib/chain/archiveStore/historicalState/worker.js +1 -4
  28. package/lib/chain/archiveStore/historicalState/worker.js.map +1 -1
  29. package/lib/chain/archiveStore/interface.d.ts +1 -0
  30. package/lib/chain/archiveStore/interface.d.ts.map +1 -1
  31. package/lib/chain/blocks/blockInput/blockInput.d.ts +5 -5
  32. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  33. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  34. package/lib/chain/blocks/blockInput/types.d.ts +4 -4
  35. package/lib/chain/blocks/blockInput/types.d.ts.map +1 -1
  36. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  37. package/lib/chain/blocks/importBlock.js +30 -18
  38. package/lib/chain/blocks/importBlock.js.map +1 -1
  39. package/lib/chain/blocks/importExecutionPayload.d.ts +10 -8
  40. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  41. package/lib/chain/blocks/importExecutionPayload.js +76 -48
  42. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  43. package/lib/chain/blocks/index.d.ts.map +1 -1
  44. package/lib/chain/blocks/index.js +2 -1
  45. package/lib/chain/blocks/index.js.map +1 -1
  46. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +14 -6
  47. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  48. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +33 -2
  49. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  50. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +2 -1
  51. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  52. package/lib/chain/blocks/types.d.ts +20 -14
  53. package/lib/chain/blocks/types.d.ts.map +1 -1
  54. package/lib/chain/blocks/verifyBlocksExecutionPayloads.d.ts +2 -2
  55. package/lib/chain/blocks/verifyBlocksExecutionPayloads.d.ts.map +1 -1
  56. package/lib/chain/blocks/verifyBlocksExecutionPayloads.js +1 -1
  57. package/lib/chain/blocks/verifyBlocksExecutionPayloads.js.map +1 -1
  58. package/lib/chain/chain.d.ts +3 -2
  59. package/lib/chain/chain.d.ts.map +1 -1
  60. package/lib/chain/chain.js +55 -20
  61. package/lib/chain/chain.js.map +1 -1
  62. package/lib/chain/emitter.d.ts +29 -7
  63. package/lib/chain/emitter.d.ts.map +1 -1
  64. package/lib/chain/emitter.js +12 -3
  65. package/lib/chain/emitter.js.map +1 -1
  66. package/lib/chain/errors/dataColumnSidecarError.d.ts +31 -1
  67. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  68. package/lib/chain/errors/dataColumnSidecarError.js +7 -0
  69. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  70. package/lib/chain/interface.d.ts +4 -2
  71. package/lib/chain/interface.d.ts.map +1 -1
  72. package/lib/chain/options.d.ts +1 -0
  73. package/lib/chain/options.d.ts.map +1 -1
  74. package/lib/chain/options.js +1 -0
  75. package/lib/chain/options.js.map +1 -1
  76. package/lib/chain/seenCache/seenGossipBlockInput.d.ts +1 -1
  77. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  78. package/lib/chain/seenCache/seenGossipBlockInput.js +2 -2
  79. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  80. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +1 -1
  81. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  82. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +2 -2
  83. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  84. package/lib/chain/validation/dataColumnSidecar.d.ts +11 -4
  85. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  86. package/lib/chain/validation/dataColumnSidecar.js +184 -5
  87. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  88. package/lib/db/buckets.d.ts +2 -2
  89. package/lib/db/buckets.d.ts.map +1 -1
  90. package/lib/db/buckets.js +2 -2
  91. package/lib/db/buckets.js.map +1 -1
  92. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  93. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  94. package/lib/db/repositories/dataColumnSidecar.d.ts.map +1 -1
  95. package/lib/db/repositories/dataColumnSidecar.js +4 -2
  96. package/lib/db/repositories/dataColumnSidecar.js.map +1 -1
  97. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  98. package/lib/db/repositories/dataColumnSidecarArchive.js +4 -2
  99. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  100. package/lib/metrics/metrics/lodestar.d.ts +20 -0
  101. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  102. package/lib/metrics/metrics/lodestar.js +33 -0
  103. package/lib/metrics/metrics/lodestar.js.map +1 -1
  104. package/lib/network/interface.d.ts +3 -2
  105. package/lib/network/interface.d.ts.map +1 -1
  106. package/lib/network/libp2p/index.d.ts.map +1 -1
  107. package/lib/network/libp2p/index.js +22 -11
  108. package/lib/network/libp2p/index.js.map +1 -1
  109. package/lib/network/network.d.ts +3 -2
  110. package/lib/network/network.d.ts.map +1 -1
  111. package/lib/network/network.js +3 -0
  112. package/lib/network/network.js.map +1 -1
  113. package/lib/network/options.d.ts.map +1 -1
  114. package/lib/network/options.js +7 -2
  115. package/lib/network/options.js.map +1 -1
  116. package/lib/network/processor/extractSlotRootFns.d.ts +1 -1
  117. package/lib/network/processor/extractSlotRootFns.d.ts.map +1 -1
  118. package/lib/network/processor/extractSlotRootFns.js +25 -5
  119. package/lib/network/processor/extractSlotRootFns.js.map +1 -1
  120. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  121. package/lib/network/processor/gossipHandlers.js +256 -73
  122. package/lib/network/processor/gossipHandlers.js.map +1 -1
  123. package/lib/network/processor/index.d.ts +11 -1
  124. package/lib/network/processor/index.d.ts.map +1 -1
  125. package/lib/network/processor/index.js +234 -22
  126. package/lib/network/processor/index.js.map +1 -1
  127. package/lib/network/reqresp/types.d.ts +3 -3
  128. package/lib/network/reqresp/types.d.ts.map +1 -1
  129. package/lib/network/reqresp/types.js +9 -3
  130. package/lib/network/reqresp/types.js.map +1 -1
  131. package/lib/sync/unknownBlock.js +2 -2
  132. package/lib/sync/unknownBlock.js.map +1 -1
  133. package/lib/sync/utils/downloadByRange.d.ts +3 -3
  134. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  135. package/lib/sync/utils/downloadByRange.js +4 -2
  136. package/lib/sync/utils/downloadByRange.js.map +1 -1
  137. package/lib/sync/utils/downloadByRoot.d.ts +3 -3
  138. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  139. package/lib/sync/utils/downloadByRoot.js +10 -5
  140. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  141. package/lib/util/blobs.d.ts +3 -3
  142. package/lib/util/blobs.d.ts.map +1 -1
  143. package/lib/util/blobs.js +21 -10
  144. package/lib/util/blobs.js.map +1 -1
  145. package/lib/util/dataColumns.d.ts +18 -11
  146. package/lib/util/dataColumns.d.ts.map +1 -1
  147. package/lib/util/dataColumns.js +51 -17
  148. package/lib/util/dataColumns.js.map +1 -1
  149. package/lib/util/execution.d.ts +6 -2
  150. package/lib/util/execution.d.ts.map +1 -1
  151. package/lib/util/execution.js +49 -25
  152. package/lib/util/execution.js.map +1 -1
  153. package/lib/util/sszBytes.d.ts +25 -1
  154. package/lib/util/sszBytes.d.ts.map +1 -1
  155. package/lib/util/sszBytes.js +189 -2
  156. package/lib/util/sszBytes.js.map +1 -1
  157. package/package.json +16 -16
  158. package/src/api/impl/beacon/blocks/index.ts +32 -15
  159. package/src/api/impl/debug/index.ts +2 -2
  160. package/src/chain/ColumnReconstructionTracker.ts +6 -5
  161. package/src/chain/GetBlobsTracker.ts +14 -12
  162. package/src/chain/archiveStore/archiveStore.ts +1 -0
  163. package/src/chain/archiveStore/historicalState/getHistoricalState.ts +6 -5
  164. package/src/chain/archiveStore/historicalState/historicalStateRegen.ts +2 -1
  165. package/src/chain/archiveStore/historicalState/types.ts +2 -0
  166. package/src/chain/archiveStore/historicalState/worker.ts +1 -5
  167. package/src/chain/archiveStore/interface.ts +1 -0
  168. package/src/chain/blocks/blockInput/blockInput.ts +8 -8
  169. package/src/chain/blocks/blockInput/types.ts +4 -4
  170. package/src/chain/blocks/importBlock.ts +36 -18
  171. package/src/chain/blocks/importExecutionPayload.ts +84 -53
  172. package/src/chain/blocks/index.ts +2 -1
  173. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +53 -12
  174. package/src/chain/blocks/payloadEnvelopeInput/types.ts +2 -1
  175. package/src/chain/blocks/types.ts +25 -14
  176. package/src/chain/blocks/verifyBlocksExecutionPayloads.ts +4 -4
  177. package/src/chain/chain.ts +63 -24
  178. package/src/chain/emitter.ts +25 -7
  179. package/src/chain/errors/dataColumnSidecarError.ts +32 -1
  180. package/src/chain/interface.ts +4 -2
  181. package/src/chain/options.ts +2 -0
  182. package/src/chain/seenCache/seenGossipBlockInput.ts +2 -2
  183. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +2 -2
  184. package/src/chain/validation/dataColumnSidecar.ts +230 -7
  185. package/src/db/buckets.ts +2 -2
  186. package/src/db/repositories/dataColumnSidecar.ts +4 -2
  187. package/src/db/repositories/dataColumnSidecarArchive.ts +4 -2
  188. package/src/metrics/metrics/lodestar.ts +34 -0
  189. package/src/network/interface.ts +3 -2
  190. package/src/network/libp2p/index.ts +24 -13
  191. package/src/network/network.ts +7 -4
  192. package/src/network/options.ts +7 -2
  193. package/src/network/processor/extractSlotRootFns.ts +32 -6
  194. package/src/network/processor/gossipHandlers.ts +320 -86
  195. package/src/network/processor/index.ts +304 -22
  196. package/src/network/reqresp/types.ts +13 -5
  197. package/src/sync/unknownBlock.ts +3 -3
  198. package/src/sync/utils/downloadByRange.ts +9 -7
  199. package/src/sync/utils/downloadByRoot.ts +16 -12
  200. package/src/util/blobs.ts +35 -15
  201. package/src/util/dataColumns.ts +69 -25
  202. package/src/util/execution.ts +49 -30
  203. package/src/util/sszBytes.ts +245 -3
@@ -3,6 +3,7 @@ import {routes} from "@lodestar/api";
3
3
  import {
4
4
  AncestorStatus,
5
5
  EpochDifference,
6
+ ExecutionStatus,
6
7
  ForkChoiceError,
7
8
  ForkChoiceErrorCode,
8
9
  NotReorgedReason,
@@ -84,7 +85,7 @@ export async function importBlock(
84
85
  fullyVerifiedBlock: FullyVerifiedBlock,
85
86
  opts: ImportBlockOpts
86
87
  ): Promise<void> {
87
- const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
88
+ const {blockInput, postBlockState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
88
89
  fullyVerifiedBlock;
89
90
  const block = blockInput.getBlock();
90
91
  const source = blockInput.getBlockSource();
@@ -96,7 +97,7 @@ export async function importBlock(
96
97
  const blockEpoch = computeEpochAtSlot(blockSlot);
97
98
  const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
98
99
  const blockDelaySec =
99
- fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
100
+ fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postBlockState.genesisTime);
100
101
  const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
101
102
  const fork = this.config.getForkSeq(blockSlot);
102
103
 
@@ -119,13 +120,13 @@ export async function importBlock(
119
120
  // 2. Import block to fork choice
120
121
 
121
122
  // Should compute checkpoint balances before forkchoice.onBlock
122
- this.checkpointBalancesCache.processState(blockRootHex, postState);
123
+ this.checkpointBalancesCache.processState(blockRootHex, postBlockState);
123
124
  const blockSummary = this.forkChoice.onBlock(
124
125
  block.message,
125
- postState,
126
+ postBlockState,
126
127
  blockDelaySec,
127
128
  currentSlot,
128
- executionStatus,
129
+ fork >= ForkSeq.gloas ? ExecutionStatus.PayloadSeparated : executionStatus,
129
130
  dataAvailabilityStatus
130
131
  );
131
132
 
@@ -135,13 +136,14 @@ export async function importBlock(
135
136
  // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
136
137
  const payloadPresent = !isGloasBlock(blockSummary);
137
138
  // processState manages both block state and payload state variants together for memory/disk management
138
- this.regen.processBlockState(blockRootHex, postState);
139
+ this.regen.processBlockState(blockRootHex, postBlockState);
139
140
 
140
141
  // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import
141
142
  if (fork >= ForkSeq.gloas) {
142
- this.seenPayloadEnvelopeInputCache.add({
143
+ const payloadInput = this.seenPayloadEnvelopeInputCache.add({
143
144
  blockRootHex,
144
145
  block: block as SignedBeaconBlock<ForkPostGloas>,
146
+ forkName: blockInput.forkName,
145
147
  sampledColumns: this.custodyConfig.sampledColumns,
146
148
  custodyColumns: this.custodyConfig.custodyColumns,
147
149
  timeCreatedSec: fullyVerifiedBlock.seenTimestampSec,
@@ -152,6 +154,22 @@ export async function importBlock(
152
154
  source: source.source,
153
155
  ...(opts.seenTimestampSec !== undefined ? {recvToImport: Date.now() / 1000 - opts.seenTimestampSec} : {}),
154
156
  });
157
+
158
+ // Immediately attempt fetch of data columns from execution engine as the bid contains kzg commitments
159
+ // which is all the information we need so there is no reason to delay until execution payload arrives
160
+ // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
161
+ // (for example later in the slot). Do not couple retries to incoming gossip columns.
162
+ this.getBlobsTracker.triggerGetBlobs(payloadInput, () => {
163
+ // TODO GLOAS: come up with a better mechanism to trigger processExecutionPayload after data becomes available,
164
+ // similar to how pre-gloas uses waitForBlockAndAllData with a cutoff timeout and incompleteBlockInput event
165
+ this.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
166
+ this.logger.debug(
167
+ "Error processing execution payload after getBlobs",
168
+ {slot: blockSlot, root: blockRootHex},
169
+ e as Error
170
+ );
171
+ });
172
+ });
155
173
  }
156
174
 
157
175
  this.metrics?.importBlock.bySource.inc({source: source.source});
@@ -171,7 +189,7 @@ export async function importBlock(
171
189
  (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
172
190
  ) {
173
191
  const attestations = block.message.body.attestations;
174
- const rootCache = new RootCache(postState);
192
+ const rootCache = new RootCache(postBlockState);
175
193
  const invalidAttestationErrorsByCode = new Map<string, {error: Error; count: number}>();
176
194
 
177
195
  const addAttestation = fork >= ForkSeq.electra ? addAttestationPostElectra : addAttestationPreElectra;
@@ -185,7 +203,7 @@ export async function importBlock(
185
203
  const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data));
186
204
  addAttestation.call(
187
205
  this,
188
- postState,
206
+ postBlockState,
189
207
  target,
190
208
  attDataRoot,
191
209
  attestation as Attestation<ForkPostElectra>,
@@ -300,7 +318,7 @@ export async function importBlock(
300
318
 
301
319
  if (newHead.blockRoot !== oldHead.blockRoot) {
302
320
  // Set head state as strong reference
303
- this.regen.updateHeadState(newHead, postState);
321
+ this.regen.updateHeadState(newHead, postBlockState);
304
322
 
305
323
  try {
306
324
  this.emitter.emit(routes.events.EventType.head, {
@@ -372,7 +390,7 @@ export async function importBlock(
372
390
  try {
373
391
  this.lightClientServer?.onImportBlockHead(
374
392
  block.message as BeaconBlock<ForkPostAltair>,
375
- postState,
393
+ postBlockState,
376
394
  parentBlockSlot
377
395
  );
378
396
  } catch (e) {
@@ -393,11 +411,11 @@ export async function importBlock(
393
411
  // and the block is weak and can potentially be reorged out.
394
412
  let shouldOverrideFcu = false;
395
413
 
396
- if (blockSlot >= currentSlot && postState.isExecutionStateType) {
414
+ if (blockSlot >= currentSlot && postBlockState.isExecutionStateType) {
397
415
  let notOverrideFcuReason = NotReorgedReason.Unknown;
398
416
  const proposalSlot = blockSlot + 1;
399
417
  try {
400
- const proposerIndex = postState.getBeaconProposer(proposalSlot);
418
+ const proposerIndex = postBlockState.getBeaconProposer(proposalSlot);
401
419
  const feeRecipient = this.beaconProposerCache.get(proposerIndex);
402
420
 
403
421
  if (feeRecipient) {
@@ -477,20 +495,20 @@ export async function importBlock(
477
495
  }
478
496
  }
479
497
 
480
- if (!postState.isStateValidatorsNodesPopulated()) {
481
- this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot});
498
+ if (!postBlockState.isStateValidatorsNodesPopulated()) {
499
+ this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postBlockState.slot});
482
500
  }
483
501
 
484
502
  // Cache shufflings when crossing an epoch boundary
485
503
  const parentEpoch = computeEpochAtSlot(parentBlockSlot);
486
504
  if (parentEpoch < blockEpoch) {
487
- this.shufflingCache.processState(postState);
505
+ this.shufflingCache.processState(postBlockState);
488
506
  this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
489
507
  }
490
508
 
491
509
  if (blockSlot % SLOTS_PER_EPOCH === 0) {
492
510
  // Cache state to preserve epoch transition work
493
- const checkpointState = postState;
511
+ const checkpointState = postBlockState;
494
512
  const cp = getCheckpointFromState(checkpointState);
495
513
  this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
496
514
  // consumers should not mutate state ever
@@ -584,7 +602,7 @@ export async function importBlock(
584
602
  this.validatorMonitor?.registerSyncAggregateInBlock(
585
603
  blockEpoch,
586
604
  (block as altair.SignedBeaconBlock).message.body.syncAggregate,
587
- fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
605
+ fullyVerifiedBlock.postBlockState.currentSyncCommitteeIndexed.validatorIndices
588
606
  );
589
607
  }
590
608
 
@@ -1,5 +1,6 @@
1
1
  import {routes} from "@lodestar/api";
2
- import {ForkName} from "@lodestar/params";
2
+ import {ExecutionStatus, PayloadExecutionStatus} from "@lodestar/fork-choice";
3
+ import {SLOTS_PER_EPOCH} from "@lodestar/params";
3
4
  import {getExecutionPayloadEnvelopeSignatureSet} from "@lodestar/state-transition";
4
5
  import {byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils";
5
6
  import {ExecutionPayloadStatus} from "../../execution/index.js";
@@ -51,18 +52,33 @@ export class PayloadError extends Error {
51
52
  }
52
53
  }
53
54
 
55
+ function toForkChoiceExecutionStatus(status: ExecutionPayloadStatus): PayloadExecutionStatus {
56
+ switch (status) {
57
+ case ExecutionPayloadStatus.VALID:
58
+ return ExecutionStatus.Valid;
59
+ // TODO GLOAS: Handle optimistic import for payload
60
+ case ExecutionPayloadStatus.SYNCING:
61
+ case ExecutionPayloadStatus.ACCEPTED:
62
+ return ExecutionStatus.Syncing;
63
+ default:
64
+ throw new Error(`Unexpected execution payload status for fork choice: ${status}`);
65
+ }
66
+ }
67
+
54
68
  /**
55
69
  * Import an execution payload envelope after all data is available.
56
70
  *
57
71
  * This function:
58
- * 1. Gets the ProtoBlock from fork choice
59
- * 2. Applies write-queue backpressure (waitForSpace) early, before verification
60
- * 3. Regenerates the block state
61
- * 4. Runs EL verification (notifyNewPayload) in parallel with signature verification and processExecutionPayloadEnvelope
62
- * 5. Persists verified payload envelope to hot DB
63
- * 6. Updates fork choice
64
- * 7. Caches the post-execution payload state
65
- * 8. Records metrics for column sources
72
+ * 1. Emits `execution_payload_available` if payload is for current slot
73
+ * 2. Gets the ProtoBlock from fork choice
74
+ * 3. Applies write-queue backpressure (waitForSpace) early, before verification
75
+ * 4. Regenerates the block state
76
+ * 5. Runs EL verification (notifyNewPayload) in parallel with signature verification and processExecutionPayloadEnvelope
77
+ * 6. Persists verified payload envelope to hot DB
78
+ * 7. Updates fork choice
79
+ * 8. Caches the post-execution payload state
80
+ * 9. Records metrics for column sources
81
+ * 10. Emits `execution_payload` for recent enough payloads after successful import
66
82
  *
67
83
  */
68
84
  export async function importExecutionPayload(
@@ -70,10 +86,24 @@ export async function importExecutionPayload(
70
86
  payloadInput: PayloadEnvelopeInput,
71
87
  opts: ImportPayloadOpts = {}
72
88
  ): Promise<void> {
73
- const envelope = payloadInput.getPayloadEnvelope();
89
+ const signedEnvelope = payloadInput.getPayloadEnvelope();
90
+ const envelope = signedEnvelope.message;
74
91
  const blockRootHex = payloadInput.blockRootHex;
92
+ const blockHashHex = payloadInput.getBlockHashHex();
93
+ const fork = this.config.getForkName(envelope.slot);
75
94
 
76
- // 1. Get ProtoBlock for parent root lookup
95
+ // 1. Emit `execution_payload_available` event at the start of import. At this point the payload input
96
+ // is already complete, so the payload and required data are available for payload attestation.
97
+ // This event is only about availability, not validity of the execution payload, hence we can emit
98
+ // it before getting a response from the execution client on whether the payload is valid or not.
99
+ if (this.clock.currentSlot === envelope.slot) {
100
+ this.emitter.emit(routes.events.EventType.executionPayloadAvailable, {
101
+ slot: envelope.slot,
102
+ blockRoot: blockRootHex,
103
+ });
104
+ }
105
+
106
+ // 2. Get ProtoBlock for parent root lookup
77
107
  const protoBlock = this.forkChoice.getBlockHexDefaultStatus(blockRootHex);
78
108
  if (!protoBlock) {
79
109
  throw new PayloadError({
@@ -82,11 +112,11 @@ export async function importExecutionPayload(
82
112
  });
83
113
  }
84
114
 
85
- // 2. Apply backpressure from the write queue early, before doing verification work.
115
+ // 3. Apply backpressure from the write queue early, before doing verification work.
86
116
  // The actual DB write is deferred until after verification succeeds.
87
117
  await this.unfinalizedPayloadEnvelopeWrites.waitForSpace();
88
118
 
89
- // 3. Get pre-state for processExecutionPayloadEnvelope
119
+ // 4. Get pre-state for processExecutionPayloadEnvelope
90
120
  // We need the block state (post-block, pre-payload) to process the envelope
91
121
  const blockState = await this.regen.getBlockSlotState(
92
122
  protoBlock,
@@ -95,16 +125,16 @@ export async function importExecutionPayload(
95
125
  RegenCaller.processBlock
96
126
  );
97
127
 
98
- // 4. Run verification steps in parallel
128
+ // 5. Run verification steps in parallel
99
129
  // Note: No data availability check needed here - importExecutionPayload is only
100
130
  // called when payloadInput.isComplete() is true, so all data is already available.
101
131
  const [execResult, signatureValid, postPayloadResult] = await Promise.all([
102
132
  this.executionEngine.notifyNewPayload(
103
- ForkName.gloas,
104
- envelope.message.payload,
133
+ fork,
134
+ envelope.payload,
105
135
  payloadInput.getVersionedHashes(),
106
136
  fromHex(protoBlock.parentRoot),
107
- envelope.message.executionRequests
137
+ envelope.executionRequests
108
138
  ),
109
139
 
110
140
  opts.validSignature === true
@@ -114,7 +144,7 @@ export async function importExecutionPayload(
114
144
  this.config,
115
145
  this.pubkeyCache,
116
146
  blockState,
117
- envelope,
147
+ signedEnvelope,
118
148
  payloadInput.proposerIndex
119
149
  );
120
150
  return this.bls.verifySignatureSets([signatureSet]);
@@ -125,7 +155,7 @@ export async function importExecutionPayload(
125
155
  (async () => {
126
156
  try {
127
157
  return {
128
- postPayloadState: blockState.processExecutionPayloadEnvelope(envelope, {
158
+ postPayloadState: blockState.processExecutionPayloadEnvelope(signedEnvelope, {
129
159
  verifySignature: false,
130
160
  verifyStateRoot: false,
131
161
  }),
@@ -142,12 +172,12 @@ export async function importExecutionPayload(
142
172
  })(),
143
173
  ]);
144
174
 
145
- // 4b. Check signature verification result
175
+ // 5a. Check signature verification result
146
176
  if (!signatureValid) {
147
177
  throw new PayloadError({code: PayloadErrorCode.INVALID_SIGNATURE});
148
178
  }
149
179
 
150
- // 5. Handle EL response
180
+ // 5b. Handle EL response
151
181
  switch (execResult.status) {
152
182
  case ExecutionPayloadStatus.VALID:
153
183
  break;
@@ -161,12 +191,7 @@ export async function importExecutionPayload(
161
191
 
162
192
  case ExecutionPayloadStatus.ACCEPTED:
163
193
  case ExecutionPayloadStatus.SYNCING:
164
- // TODO GLOAS: Handle optimistic import for payload - for now treat as error
165
- throw new PayloadError({
166
- code: PayloadErrorCode.EXECUTION_ENGINE_ERROR,
167
- execStatus: execResult.status,
168
- errorMessage: execResult.validationError ?? "EL syncing, payload not yet validated",
169
- });
194
+ break;
170
195
 
171
196
  case ExecutionPayloadStatus.INVALID_BLOCK_HASH:
172
197
  case ExecutionPayloadStatus.ELERROR:
@@ -178,59 +203,65 @@ export async function importExecutionPayload(
178
203
  });
179
204
  }
180
205
 
181
- // 5b. Verify envelope state root matches post-state
206
+ // 5c. Verify envelope state root matches post-state
182
207
  const postPayloadState = postPayloadResult.postPayloadState;
183
208
  const postPayloadStateRoot = postPayloadState.hashTreeRoot();
184
- if (!byteArrayEquals(envelope.message.stateRoot, postPayloadStateRoot)) {
209
+ if (!byteArrayEquals(envelope.stateRoot, postPayloadStateRoot)) {
185
210
  throw new PayloadError({
186
211
  code: PayloadErrorCode.STATE_TRANSITION_ERROR,
187
- message: `Envelope state root mismatch expected=${toRootHex(envelope.message.stateRoot)} actual=${toRootHex(postPayloadStateRoot)}`,
212
+ message: `Envelope state root mismatch expected=${toRootHex(envelope.stateRoot)} actual=${toRootHex(postPayloadStateRoot)}`,
188
213
  });
189
214
  }
190
215
 
191
- // 5c. Persist payload envelope to hot DB (performed asynchronously to avoid blocking)
216
+ // 6. Persist payload envelope to hot DB (performed asynchronously to avoid blocking)
192
217
  this.unfinalizedPayloadEnvelopeWrites.push(payloadInput).catch((e) => {
193
218
  if (!isQueueErrorAborted(e)) {
194
219
  this.logger.error(
195
220
  "Error pushing payload envelope to unfinalized write queue",
196
- {slot: payloadInput.slot, root: blockRootHex},
221
+ {slot: envelope.slot, root: blockRootHex},
197
222
  e as Error
198
223
  );
199
224
  }
200
225
  });
201
226
 
202
- // 6. Update fork choice
227
+ // 7. Update fork choice
203
228
  this.forkChoice.onExecutionPayload(
204
229
  blockRootHex,
205
- payloadInput.getBlockHashHex(),
206
- envelope.message.payload.blockNumber,
207
- toRootHex(postPayloadStateRoot)
230
+ blockHashHex,
231
+ envelope.payload.blockNumber,
232
+ toRootHex(postPayloadStateRoot),
233
+ toForkChoiceExecutionStatus(execResult.status)
208
234
  );
209
235
 
210
- // 7. Cache payload state
211
- // TODO GLOAS: Enable when PR #8868 merged (adds processPayloadState)
212
- // this.regen.processPayloadState(postPayloadState);
213
- // if epoch boundary also call
214
- // this.regen.addCheckpointState(cp, checkpointState, true);
236
+ // 8. Cache payload state
237
+ this.regen.processPayloadState(postPayloadState);
238
+ if (postPayloadState.slot % SLOTS_PER_EPOCH === 0) {
239
+ const {checkpoint} = postPayloadState.computeAnchorCheckpoint();
240
+ this.regen.addCheckpointState(checkpoint, postPayloadState, true);
241
+ }
215
242
 
216
- // 8. Record metrics for payload envelope and column sources
243
+ // 9. Record metrics for payload envelope and column sources
217
244
  this.metrics?.importPayload.bySource.inc({source: payloadInput.getPayloadEnvelopeSource().source});
218
245
  for (const {source} of payloadInput.getSampledColumnsWithSource()) {
219
246
  this.metrics?.importPayload.columnsBySource.inc({source});
220
247
  }
221
248
 
222
- this.logger.verbose("Execution payload imported", {
223
- slot: payloadInput.slot,
224
- root: blockRootHex,
225
- blockHash: payloadInput.getBlockHashHex(),
226
- });
227
-
228
- // 9. Emit event after payload is fully verified and imported to fork choice, only for recent enough payloads
229
- const currentSlot = this.clock.currentSlot;
230
- if (currentSlot - payloadInput.slot < EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS) {
231
- this.emitter.emit(routes.events.EventType.executionPayloadAvailable, {
232
- slot: payloadInput.slot,
249
+ // 10. Emit event after payload is fully verified and imported to fork choice, only for recent enough payloads
250
+ if (this.clock.currentSlot - envelope.slot < EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS) {
251
+ this.emitter.emit(routes.events.EventType.executionPayload, {
252
+ slot: envelope.slot,
253
+ builderIndex: envelope.builderIndex,
254
+ blockHash: blockHashHex,
233
255
  blockRoot: blockRootHex,
256
+ stateRoot: toRootHex(envelope.stateRoot),
257
+ // TODO GLOAS: revisit once we support optimistic import
258
+ executionOptimistic: false,
234
259
  });
235
260
  }
261
+
262
+ this.logger.verbose("Execution payload imported", {
263
+ slot: envelope.slot,
264
+ root: blockRootHex,
265
+ blockHash: blockHashHex,
266
+ });
236
267
  }
@@ -88,7 +88,8 @@ export async function processBlocks(
88
88
  const fullyVerifiedBlocks = relevantBlocks.map(
89
89
  (block, i): FullyVerifiedBlock => ({
90
90
  blockInput: block,
91
- postState: postStates[i],
91
+ postBlockState: postStates[i],
92
+ postEnvelopeState: null,
92
93
  parentBlockSlot: parentSlots[i],
93
94
  executionStatus: executionStatuses[i],
94
95
  // start supporting optimistic syncing/processing
@@ -1,8 +1,9 @@
1
- import {NUMBER_OF_COLUMNS} from "@lodestar/params";
2
- import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
1
+ import {ForkName, NUMBER_OF_COLUMNS} from "@lodestar/params";
2
+ import {ColumnIndex, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
3
3
  import {toRootHex, withTimeout} from "@lodestar/utils";
4
4
  import {VersionedHashes} from "../../../execution/index.js";
5
5
  import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
6
+ import {MissingColumnMeta} from "../blockInput/types.js";
6
7
  import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js";
7
8
 
8
9
  export type PayloadEnvelopeInputState =
@@ -59,6 +60,7 @@ function createPromise<T>(): PromiseParts<T> {
59
60
  export class PayloadEnvelopeInput {
60
61
  readonly blockRootHex: RootHex;
61
62
  readonly slot: Slot;
63
+ readonly forkName: ForkName;
62
64
  readonly proposerIndex: ValidatorIndex;
63
65
  readonly bid: gloas.ExecutionPayloadBid;
64
66
  readonly versionedHashes: VersionedHashes;
@@ -71,13 +73,14 @@ export class PayloadEnvelopeInput {
71
73
  private timeCreatedSec: number;
72
74
 
73
75
  private readonly payloadEnvelopeDataPromise: PromiseParts<gloas.SignedExecutionPayloadEnvelope>;
74
- private readonly columnsDataPromise: PromiseParts<DataColumnSidecars>;
76
+ private readonly columnsDataPromise: PromiseParts<gloas.DataColumnSidecar[]>;
75
77
 
76
78
  state: PayloadEnvelopeInputState;
77
79
 
78
80
  private constructor(props: {
79
81
  blockRootHex: RootHex;
80
82
  slot: Slot;
83
+ forkName: ForkName;
81
84
  proposerIndex: ValidatorIndex;
82
85
  bid: gloas.ExecutionPayloadBid;
83
86
  sampledColumns: ColumnIndex[];
@@ -86,6 +89,7 @@ export class PayloadEnvelopeInput {
86
89
  }) {
87
90
  this.blockRootHex = props.blockRootHex;
88
91
  this.slot = props.slot;
92
+ this.forkName = props.forkName;
89
93
  this.proposerIndex = props.proposerIndex;
90
94
  this.bid = props.bid;
91
95
  this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash);
@@ -112,6 +116,7 @@ export class PayloadEnvelopeInput {
112
116
  return new PayloadEnvelopeInput({
113
117
  blockRootHex: props.blockRootHex,
114
118
  slot: props.block.message.slot,
119
+ forkName: props.forkName,
115
120
  proposerIndex: props.block.message.proposerIndex,
116
121
  bid,
117
122
  sampledColumns: props.sampledColumns,
@@ -173,8 +178,12 @@ export class PayloadEnvelopeInput {
173
178
  }
174
179
  }
175
180
 
176
- addColumn(columnWithSource: ColumnWithSource): void {
181
+ addColumn(columnWithSource: ColumnWithSource): boolean {
177
182
  const {columnSidecar, seenTimestampSec} = columnWithSource;
183
+ if (this.columnsCache.has(columnSidecar.index)) {
184
+ return false;
185
+ }
186
+
178
187
  this.columnsCache.set(columnSidecar.index, columnWithSource);
179
188
 
180
189
  const sampledColumns = this.getSampledColumns();
@@ -191,7 +200,7 @@ export class PayloadEnvelopeInput {
191
200
  sampledColumns.length === this.sampledColumns.length;
192
201
 
193
202
  if (!hasAllData) {
194
- return;
203
+ return true;
195
204
  }
196
205
 
197
206
  if (hasComputedAllData) {
@@ -217,6 +226,20 @@ export class PayloadEnvelopeInput {
217
226
  hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
218
227
  };
219
228
  }
229
+
230
+ return true;
231
+ }
232
+
233
+ hasColumn(index: ColumnIndex): boolean {
234
+ return this.columnsCache.has(index);
235
+ }
236
+
237
+ getColumn(index: ColumnIndex): gloas.DataColumnSidecar | undefined {
238
+ return this.columnsCache.get(index)?.columnSidecar;
239
+ }
240
+
241
+ getAllColumns(): gloas.DataColumnSidecar[] {
242
+ return [...this.columnsCache.values()].map(({columnSidecar}) => columnSidecar);
220
243
  }
221
244
 
222
245
  getVersionedHashes(): VersionedHashes {
@@ -237,8 +260,8 @@ export class PayloadEnvelopeInput {
237
260
  return this.state.payloadEnvelopeSource;
238
261
  }
239
262
 
240
- getSampledColumns(): gloas.DataColumnSidecars {
241
- const columns: gloas.DataColumnSidecars = [];
263
+ getSampledColumns(): gloas.DataColumnSidecar[] {
264
+ const columns: gloas.DataColumnSidecar[] = [];
242
265
  for (const index of this.sampledColumns) {
243
266
  const column = this.columnsCache.get(index);
244
267
  if (column) {
@@ -259,8 +282,8 @@ export class PayloadEnvelopeInput {
259
282
  return columns;
260
283
  }
261
284
 
262
- getCustodyColumns(): gloas.DataColumnSidecars {
263
- const columns: gloas.DataColumnSidecars = [];
285
+ getCustodyColumns(): gloas.DataColumnSidecar[] {
286
+ const columns: gloas.DataColumnSidecar[] = [];
264
287
  for (const index of this.custodyColumns) {
265
288
  const column = this.columnsCache.get(index);
266
289
  if (column) {
@@ -270,11 +293,29 @@ export class PayloadEnvelopeInput {
270
293
  return columns;
271
294
  }
272
295
 
296
+ hasAllData(): boolean {
297
+ return this.state.hasAllData;
298
+ }
299
+
300
+ getMissingSampledColumnMeta(): MissingColumnMeta {
301
+ if (this.state.hasAllData) {
302
+ return {missing: [], versionedHashes: this.versionedHashes};
303
+ }
304
+
305
+ const missing: ColumnIndex[] = [];
306
+ for (const index of this.sampledColumns) {
307
+ if (!this.columnsCache.has(index)) {
308
+ missing.push(index);
309
+ }
310
+ }
311
+ return {missing, versionedHashes: this.versionedHashes};
312
+ }
313
+
273
314
  hasComputedAllData(): boolean {
274
315
  return this.state.hasComputedAllData;
275
316
  }
276
317
 
277
- waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecars> {
318
+ waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<gloas.DataColumnSidecar[]> {
278
319
  if (this.state.hasComputedAllData) {
279
320
  return Promise.resolve(this.getSampledColumns());
280
321
  }
@@ -319,7 +360,7 @@ export class PayloadEnvelopeInput {
319
360
  hasAllData: boolean;
320
361
  hasComputedAllData: boolean;
321
362
  isComplete: boolean;
322
- columnsCount: number;
363
+ receivedColumns: number;
323
364
  sampledColumnsCount: number;
324
365
  } {
325
366
  return {
@@ -329,7 +370,7 @@ export class PayloadEnvelopeInput {
329
370
  hasAllData: this.state.hasAllData,
330
371
  hasComputedAllData: this.state.hasComputedAllData,
331
372
  isComplete: this.isComplete(),
332
- columnsCount: this.columnsCache.size,
373
+ receivedColumns: this.columnsCache.size,
333
374
  sampledColumnsCount: this.sampledColumns.length,
334
375
  };
335
376
  }
@@ -1,4 +1,4 @@
1
- import {ForkPostGloas} from "@lodestar/params";
1
+ import {ForkName, ForkPostGloas} from "@lodestar/params";
2
2
  import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types";
3
3
 
4
4
  export enum PayloadEnvelopeInputSource {
@@ -23,6 +23,7 @@ export type ColumnWithSource = SourceMeta & {
23
23
  export type CreateFromBlockProps = {
24
24
  blockRootHex: RootHex;
25
25
  block: SignedBeaconBlock<ForkPostGloas>;
26
+ forkName: ForkName;
26
27
  sampledColumns: ColumnIndex[];
27
28
  custodyColumns: ColumnIndex[];
28
29
  timeCreatedSec: number;
@@ -1,5 +1,5 @@
1
1
  import type {ChainForkConfig} from "@lodestar/config";
2
- import {MaybeValidExecutionStatus} from "@lodestar/fork-choice";
2
+ import {BlockExecutionStatus, PayloadExecutionStatus} from "@lodestar/fork-choice";
3
3
  import {ForkSeq} from "@lodestar/params";
4
4
  import {DataAvailabilityStatus, IBeaconStateView, computeEpochAtSlot} from "@lodestar/state-transition";
5
5
  import type {IndexedAttestation, Slot, fulu} from "@lodestar/types";
@@ -88,24 +88,35 @@ export type ImportBlockOpts = {
88
88
  seenTimestampSec?: number;
89
89
  };
90
90
 
91
- /**
92
- * A wrapper around a `SignedBeaconBlock` that indicates that this block is fully verified and ready to import
93
- */
94
- export type FullyVerifiedBlock = {
91
+ type FullyVerifiedBlockBase = {
95
92
  blockInput: IBlockInput;
96
- postState: IBeaconStateView;
93
+ postBlockState: IBeaconStateView;
97
94
  parentBlockSlot: Slot;
98
95
  proposerBalanceDelta: number;
99
- /**
100
- * If the execution payload couldnt be verified because of EL syncing status,
101
- * used in optimistic sync or for merge block
102
- */
103
- executionStatus: MaybeValidExecutionStatus;
104
96
  dataAvailabilityStatus: DataAvailabilityStatus;
105
- /**
106
- * Pre-computed indexed attestations from signature verification to avoid duplicate work
107
- */
97
+ /** Pre-computed indexed attestations from signature verification to avoid duplicate work */
108
98
  indexedAttestations: IndexedAttestation[];
109
99
  /** Seen timestamp seconds */
110
100
  seenTimestampSec: number;
111
101
  };
102
+
103
+ /**
104
+ * A wrapper around a `SignedBeaconBlock` that indicates that this block is fully verified and ready to import.
105
+ *
106
+ * Discriminated union on `postEnvelopeState`:
107
+ * - `null` → block has no pre-verified envelope; `executionStatus` is any `BlockExecutionStatus`
108
+ * - non-null → envelope was pre-verified during state transition; `executionStatus` is narrowed to
109
+ * `Valid | Syncing` (matching what `forkChoice.onExecutionPayload` expects)
110
+ */
111
+ export type FullyVerifiedBlock = FullyVerifiedBlockBase &
112
+ (
113
+ | {
114
+ postEnvelopeState: null;
115
+ /** If the execution payload couldn't be verified because of EL syncing status, used in optimistic sync or for merge block */
116
+ executionStatus: BlockExecutionStatus;
117
+ }
118
+ | {
119
+ postEnvelopeState: IBeaconStateView;
120
+ executionStatus: PayloadExecutionStatus;
121
+ }
122
+ );