@lodestar/beacon-node 1.43.0-dev.5f9285892c → 1.43.0-dev.9fa9f08ef6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/lib/api/impl/validator/index.d.ts.map +1 -1
  2. package/lib/api/impl/validator/index.js +4 -3
  3. package/lib/api/impl/validator/index.js.map +1 -1
  4. package/lib/chain/GetBlobsTracker.d.ts +1 -1
  5. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  6. package/lib/chain/GetBlobsTracker.js +1 -2
  7. package/lib/chain/GetBlobsTracker.js.map +1 -1
  8. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts.map +1 -1
  9. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js +2 -4
  10. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js.map +1 -1
  11. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  12. package/lib/chain/blocks/importBlock.js +23 -31
  13. package/lib/chain/blocks/importBlock.js.map +1 -1
  14. package/lib/chain/blocks/importExecutionPayload.d.ts +1 -1
  15. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  16. package/lib/chain/blocks/importExecutionPayload.js +10 -8
  17. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  18. package/lib/chain/blocks/index.js +1 -1
  19. package/lib/chain/blocks/index.js.map +1 -1
  20. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +3 -0
  21. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  22. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +20 -0
  23. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  24. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -0
  25. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts.map +1 -1
  26. package/lib/chain/blocks/payloadEnvelopeProcessor.js +6 -4
  27. package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -1
  28. package/lib/chain/blocks/types.d.ts +1 -1
  29. package/lib/chain/blocks/types.d.ts.map +1 -1
  30. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +14 -0
  31. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -0
  32. package/lib/chain/blocks/verifyPayloadsDataAvailability.js +25 -0
  33. package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -0
  34. package/lib/chain/chain.d.ts.map +1 -1
  35. package/lib/chain/chain.js +14 -30
  36. package/lib/chain/chain.js.map +1 -1
  37. package/lib/chain/emitter.d.ts +13 -1
  38. package/lib/chain/emitter.d.ts.map +1 -1
  39. package/lib/chain/emitter.js +5 -0
  40. package/lib/chain/emitter.js.map +1 -1
  41. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  42. package/lib/chain/forkChoice/index.js +6 -2
  43. package/lib/chain/forkChoice/index.js.map +1 -1
  44. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  45. package/lib/chain/prepareNextSlot.js +22 -16
  46. package/lib/chain/prepareNextSlot.js.map +1 -1
  47. package/lib/chain/produceBlock/computeNewStateRoot.d.ts +3 -9
  48. package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
  49. package/lib/chain/produceBlock/computeNewStateRoot.js +5 -32
  50. package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
  51. package/lib/chain/produceBlock/produceBlockBody.d.ts +3 -8
  52. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  53. package/lib/chain/produceBlock/produceBlockBody.js +24 -19
  54. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  55. package/lib/chain/regen/errors.d.ts +1 -11
  56. package/lib/chain/regen/errors.d.ts.map +1 -1
  57. package/lib/chain/regen/errors.js +0 -2
  58. package/lib/chain/regen/errors.js.map +1 -1
  59. package/lib/chain/regen/interface.d.ts +6 -11
  60. package/lib/chain/regen/interface.d.ts.map +1 -1
  61. package/lib/chain/regen/queued.d.ts +6 -10
  62. package/lib/chain/regen/queued.d.ts.map +1 -1
  63. package/lib/chain/regen/queued.js +3 -10
  64. package/lib/chain/regen/queued.js.map +1 -1
  65. package/lib/chain/regen/regen.d.ts +0 -5
  66. package/lib/chain/regen/regen.d.ts.map +1 -1
  67. package/lib/chain/regen/regen.js +0 -8
  68. package/lib/chain/regen/regen.js.map +1 -1
  69. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +1 -7
  70. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  71. package/lib/chain/stateCache/persistentCheckpointsCache.js +0 -8
  72. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  73. package/lib/chain/stateCache/types.d.ts +0 -6
  74. package/lib/chain/stateCache/types.d.ts.map +1 -1
  75. package/lib/chain/stateCache/types.js.map +1 -1
  76. package/lib/chain/validation/executionPayloadEnvelope.js +1 -1
  77. package/lib/chain/validation/executionPayloadEnvelope.js.map +1 -1
  78. package/lib/network/gossip/topic.d.ts +727 -0
  79. package/lib/network/gossip/topic.d.ts.map +1 -1
  80. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  81. package/lib/network/processor/gossipHandlers.js +19 -3
  82. package/lib/network/processor/gossipHandlers.js.map +1 -1
  83. package/lib/node/nodejs.d.ts.map +1 -1
  84. package/lib/node/nodejs.js +4 -2
  85. package/lib/node/nodejs.js.map +1 -1
  86. package/package.json +15 -15
  87. package/src/api/impl/validator/index.ts +6 -5
  88. package/src/chain/GetBlobsTracker.ts +1 -2
  89. package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +2 -4
  90. package/src/chain/blocks/importBlock.ts +22 -35
  91. package/src/chain/blocks/importExecutionPayload.ts +11 -7
  92. package/src/chain/blocks/index.ts +1 -1
  93. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +27 -0
  94. package/src/chain/blocks/payloadEnvelopeProcessor.ts +6 -5
  95. package/src/chain/blocks/types.ts +1 -1
  96. package/src/chain/blocks/verifyPayloadsDataAvailability.ts +38 -0
  97. package/src/chain/chain.ts +13 -32
  98. package/src/chain/emitter.ts +12 -0
  99. package/src/chain/forkChoice/index.ts +6 -2
  100. package/src/chain/prepareNextSlot.ts +25 -16
  101. package/src/chain/produceBlock/computeNewStateRoot.ts +6 -43
  102. package/src/chain/produceBlock/produceBlockBody.ts +34 -20
  103. package/src/chain/regen/errors.ts +1 -6
  104. package/src/chain/regen/interface.ts +6 -11
  105. package/src/chain/regen/queued.ts +6 -14
  106. package/src/chain/regen/regen.ts +0 -8
  107. package/src/chain/stateCache/persistentCheckpointsCache.ts +1 -14
  108. package/src/chain/stateCache/types.ts +0 -3
  109. package/src/chain/validation/executionPayloadEnvelope.ts +1 -1
  110. package/src/network/processor/gossipHandlers.ts +23 -7
  111. package/src/node/nodejs.ts +4 -2
@@ -8,7 +8,6 @@ import {
8
8
  ForkChoiceErrorCode,
9
9
  NotReorgedReason,
10
10
  getSafeExecutionBlockHash,
11
- isGloasBlock,
12
11
  } from "@lodestar/fork-choice";
13
12
  import {
14
13
  ForkPostAltair,
@@ -87,7 +86,7 @@ export async function importBlock(
87
86
  fullyVerifiedBlock: FullyVerifiedBlock,
88
87
  opts: ImportBlockOpts
89
88
  ): Promise<void> {
90
- const {blockInput, postBlockState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
89
+ const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
91
90
  fullyVerifiedBlock;
92
91
  const block = blockInput.getBlock();
93
92
  const source = blockInput.getBlockSource();
@@ -99,7 +98,7 @@ export async function importBlock(
99
98
  const blockEpoch = computeEpochAtSlot(blockSlot);
100
99
  const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
101
100
  const blockDelaySec =
102
- fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postBlockState.genesisTime);
101
+ fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
103
102
  const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
104
103
  const fork = this.config.getForkSeq(blockSlot);
105
104
 
@@ -122,10 +121,10 @@ export async function importBlock(
122
121
  // 2. Import block to fork choice
123
122
 
124
123
  // Should compute checkpoint balances before forkchoice.onBlock
125
- this.checkpointBalancesCache.processState(blockRootHex, postBlockState);
124
+ this.checkpointBalancesCache.processState(blockRootHex, postState);
126
125
  const blockSummary = this.forkChoice.onBlock(
127
126
  block.message,
128
- postBlockState,
127
+ postState,
129
128
  blockDelaySec,
130
129
  currentSlot,
131
130
  fork >= ForkSeq.gloas ? ExecutionStatus.PayloadSeparated : executionStatus,
@@ -134,11 +133,7 @@ export async function importBlock(
134
133
 
135
134
  // This adds the state necessary to process the next block
136
135
  // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
137
- // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true
138
- // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
139
- const payloadPresent = !isGloasBlock(blockSummary);
140
- // processState manages both block state and payload state variants together for memory/disk management
141
- this.regen.processBlockState(blockRootHex, postBlockState);
136
+ this.regen.processState(blockRootHex, postState);
142
137
 
143
138
  // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import
144
139
  if (fork >= ForkSeq.gloas) {
@@ -161,17 +156,9 @@ export async function importBlock(
161
156
  // which is all the information we need so there is no reason to delay until execution payload arrives
162
157
  // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
163
158
  // (for example later in the slot). Do not couple retries to incoming gossip columns.
164
- this.getBlobsTracker.triggerGetBlobs(payloadInput, () => {
165
- // TODO GLOAS: come up with a better mechanism to trigger processExecutionPayload after data becomes available,
166
- // similar to how pre-gloas uses waitForBlockAndAllData with a cutoff timeout and incompleteBlockInput event
167
- this.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
168
- this.logger.debug(
169
- "Error processing execution payload after getBlobs",
170
- {slot: blockSlot, root: blockRootHex},
171
- e as Error
172
- );
173
- });
174
- });
159
+ // Columns fetched here feed payloadInput.addColumn, which resolves waitForAllData for any
160
+ // in-flight importExecutionPayload. No processExecutionPayload trigger needed from this path.
161
+ this.getBlobsTracker.triggerGetBlobs(payloadInput);
175
162
  }
176
163
 
177
164
  this.metrics?.importBlock.bySource.inc({source: source.source});
@@ -191,7 +178,7 @@ export async function importBlock(
191
178
  (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
192
179
  ) {
193
180
  const attestations = block.message.body.attestations;
194
- const rootCache = new RootCache(postBlockState);
181
+ const rootCache = new RootCache(postState);
195
182
  const invalidAttestationErrorsByCode = new Map<string, {error: Error; count: number}>();
196
183
 
197
184
  const addAttestation = fork >= ForkSeq.electra ? addAttestationPostElectra : addAttestationPreElectra;
@@ -205,7 +192,7 @@ export async function importBlock(
205
192
  const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data));
206
193
  addAttestation.call(
207
194
  this,
208
- postBlockState,
195
+ postState,
209
196
  target,
210
197
  attDataRoot,
211
198
  attestation as Attestation<ForkPostElectra>,
@@ -320,7 +307,7 @@ export async function importBlock(
320
307
 
321
308
  if (newHead.blockRoot !== oldHead.blockRoot) {
322
309
  // Set head state as strong reference
323
- this.regen.updateHeadState(newHead, postBlockState);
310
+ this.regen.updateHeadState(newHead, postState);
324
311
 
325
312
  try {
326
313
  this.emitter.emit(routes.events.EventType.head, {
@@ -390,10 +377,10 @@ export async function importBlock(
390
377
  // we want to import block asap so do this in the next event loop
391
378
  callInNextEventLoop(() => {
392
379
  try {
393
- if (isStatePostAltair(postBlockState)) {
380
+ if (isStatePostAltair(postState)) {
394
381
  this.lightClientServer?.onImportBlockHead(
395
382
  block.message as BeaconBlock<ForkPostAltair>,
396
- postBlockState,
383
+ postState,
397
384
  parentBlockSlot
398
385
  );
399
386
  }
@@ -415,11 +402,11 @@ export async function importBlock(
415
402
  // and the block is weak and can potentially be reorged out.
416
403
  let shouldOverrideFcu = false;
417
404
 
418
- if (blockSlot >= currentSlot && isStatePostBellatrix(postBlockState) && postBlockState.isExecutionStateType) {
405
+ if (blockSlot >= currentSlot && isStatePostBellatrix(postState) && postState.isExecutionStateType) {
419
406
  let notOverrideFcuReason = NotReorgedReason.Unknown;
420
407
  const proposalSlot = blockSlot + 1;
421
408
  try {
422
- const proposerIndex = postBlockState.getBeaconProposer(proposalSlot);
409
+ const proposerIndex = postState.getBeaconProposer(proposalSlot);
423
410
  const feeRecipient = this.beaconProposerCache.get(proposerIndex);
424
411
 
425
412
  if (feeRecipient) {
@@ -499,22 +486,22 @@ export async function importBlock(
499
486
  }
500
487
  }
501
488
 
502
- if (!postBlockState.isStateValidatorsNodesPopulated()) {
503
- this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postBlockState.slot});
489
+ if (!postState.isStateValidatorsNodesPopulated()) {
490
+ this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot});
504
491
  }
505
492
 
506
493
  // Cache shufflings when crossing an epoch boundary
507
494
  const parentEpoch = computeEpochAtSlot(parentBlockSlot);
508
495
  if (parentEpoch < blockEpoch) {
509
- this.shufflingCache.processState(postBlockState);
496
+ this.shufflingCache.processState(postState);
510
497
  this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
511
498
  }
512
499
 
513
500
  if (blockSlot % SLOTS_PER_EPOCH === 0) {
514
501
  // Cache state to preserve epoch transition work
515
- const checkpointState = postBlockState;
502
+ const checkpointState = postState;
516
503
  const cp = getCheckpointFromState(checkpointState);
517
- this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
504
+ this.regen.addCheckpointState(cp, checkpointState);
518
505
  // consumers should not mutate state ever
519
506
  this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
520
507
 
@@ -602,11 +589,11 @@ export async function importBlock(
602
589
  this.metrics?.parentBlockDistance.observe(blockSlot - parentBlockSlot);
603
590
  this.metrics?.proposerBalanceDeltaAny.observe(fullyVerifiedBlock.proposerBalanceDelta);
604
591
  this.validatorMonitor?.registerImportedBlock(block.message, fullyVerifiedBlock);
605
- if (isStatePostAltair(fullyVerifiedBlock.postBlockState)) {
592
+ if (isStatePostAltair(fullyVerifiedBlock.postState)) {
606
593
  this.validatorMonitor?.registerSyncAggregateInBlock(
607
594
  blockEpoch,
608
595
  (block as altair.SignedBeaconBlock).message.body.syncAggregate,
609
- fullyVerifiedBlock.postBlockState.currentSyncCommitteeIndexed.validatorIndices
596
+ fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
610
597
  );
611
598
  }
612
599
 
@@ -9,6 +9,7 @@ import {BeaconChain} from "../chain.js";
9
9
  import {RegenCaller} from "../regen/interface.js";
10
10
  import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
11
11
  import {ImportPayloadOpts} from "./types.js";
12
+ import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
12
13
 
13
14
  const EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS = 64;
14
15
 
@@ -84,6 +85,7 @@ function toForkChoiceExecutionStatus(status: ExecutionPayloadStatus): PayloadExe
84
85
  export async function importExecutionPayload(
85
86
  this: BeaconChain,
86
87
  payloadInput: PayloadEnvelopeInput,
88
+ signal: AbortSignal,
87
89
  opts: ImportPayloadOpts = {}
88
90
  ): Promise<void> {
89
91
  const signedEnvelope = payloadInput.getPayloadEnvelope();
@@ -112,11 +114,15 @@ export async function importExecutionPayload(
112
114
  });
113
115
  }
114
116
 
115
- // 3. Apply backpressure from the write queue early, before doing verification work.
117
+ // 3. Wait for data columns to be available before claiming a write-queue slot.
118
+ // The helper is shared with future gloas sync services; take the single-item batch form here.
119
+ await verifyPayloadsDataAvailability([payloadInput], signal);
120
+
121
+ // 4. Apply backpressure from the write queue, before doing verification work.
116
122
  // The actual DB write is deferred until after verification succeeds.
117
123
  await this.unfinalizedPayloadEnvelopeWrites.waitForSpace();
118
124
 
119
- // 4. Get pre-state for processExecutionPayloadEnvelope
125
+ // 5. Get pre-state for processExecutionPayloadEnvelope
120
126
  // We need the block state (post-block, pre-payload) to process the envelope
121
127
  const blockState = await this.regen.getBlockSlotState(
122
128
  protoBlock,
@@ -131,9 +137,7 @@ export async function importExecutionPayload(
131
137
  });
132
138
  }
133
139
 
134
- // 5. Run verification steps in parallel
135
- // Note: No data availability check needed here - importExecutionPayload is only
136
- // called when payloadInput.isComplete() is true, so all data is already available.
140
+ // 6. Run verification steps in parallel
137
141
  const [execResult, signatureValid, postPayloadResult] = await Promise.all([
138
142
  this.executionEngine.notifyNewPayload(
139
143
  fork,
@@ -240,10 +244,10 @@ export async function importExecutionPayload(
240
244
  );
241
245
 
242
246
  // 8. Cache payload state
243
- this.regen.processPayloadState(postPayloadState);
247
+ this.regen.processState(blockRootHex, postPayloadState);
244
248
  if (postPayloadState.slot % SLOTS_PER_EPOCH === 0) {
245
249
  const {checkpoint} = postPayloadState.computeAnchorCheckpoint();
246
- this.regen.addCheckpointState(checkpoint, postPayloadState, true);
250
+ this.regen.addCheckpointState(checkpoint, postPayloadState);
247
251
  }
248
252
 
249
253
  // 9. Record metrics for payload envelope and column sources
@@ -88,7 +88,7 @@ export async function processBlocks(
88
88
  const fullyVerifiedBlocks = relevantBlocks.map(
89
89
  (block, i): FullyVerifiedBlock => ({
90
90
  blockInput: block,
91
- postBlockState: postStates[i],
91
+ postState: postStates[i],
92
92
  postPayloadState: null,
93
93
  parentBlockSlot: parentSlots[i],
94
94
  executionStatus: executionStatuses[i],
@@ -73,6 +73,7 @@ export class PayloadEnvelopeInput {
73
73
  private timeCreatedSec: number;
74
74
 
75
75
  private readonly payloadEnvelopeDataPromise: PromiseParts<gloas.SignedExecutionPayloadEnvelope>;
76
+ private readonly allDataPromise: PromiseParts<gloas.DataColumnSidecar[]>;
76
77
  private readonly columnsDataPromise: PromiseParts<gloas.DataColumnSidecar[]>;
77
78
 
78
79
  state: PayloadEnvelopeInputState;
@@ -97,6 +98,7 @@ export class PayloadEnvelopeInput {
97
98
  this.custodyColumns = props.custodyColumns;
98
99
  this.timeCreatedSec = props.timeCreatedSec;
99
100
  this.payloadEnvelopeDataPromise = createPromise();
101
+ this.allDataPromise = createPromise();
100
102
  this.columnsDataPromise = createPromise();
101
103
 
102
104
  const noBlobs = props.bid.blobKzgCommitments.length === 0;
@@ -105,6 +107,7 @@ export class PayloadEnvelopeInput {
105
107
 
106
108
  if (hasAllData) {
107
109
  this.state = {hasPayload: false, hasAllData: true, hasComputedAllData: true};
110
+ this.allDataPromise.resolve(this.getSampledColumns());
108
111
  this.columnsDataPromise.resolve(this.getSampledColumns());
109
112
  } else {
110
113
  this.state = {hasPayload: false, hasAllData: false, hasComputedAllData: false};
@@ -203,6 +206,12 @@ export class PayloadEnvelopeInput {
203
206
  return true;
204
207
  }
205
208
 
209
+ // Resolve allDataPromise on the first transition to hasAllData (either sampled-complete or
210
+ // reconstruction-threshold branch). Guarded so it fires exactly once.
211
+ if (!this.state.hasAllData && hasAllData) {
212
+ this.allDataPromise.resolve(sampledColumns);
213
+ }
214
+
206
215
  if (hasComputedAllData) {
207
216
  this.columnsDataPromise.resolve(sampledColumns);
208
217
  }
@@ -315,6 +324,24 @@ export class PayloadEnvelopeInput {
315
324
  return this.state.hasComputedAllData;
316
325
  }
317
326
 
327
+ waitForAllData(timeout: number, signal?: AbortSignal): Promise<gloas.DataColumnSidecar[]> {
328
+ if (this.state.hasAllData) {
329
+ return Promise.resolve(this.getSampledColumns());
330
+ }
331
+ return withTimeout(() => this.allDataPromise.promise, timeout, signal);
332
+ }
333
+
334
+ async waitForEnvelopeAndAllData(timeout: number, signal?: AbortSignal): Promise<this> {
335
+ if (!this.state.hasPayload || !this.state.hasAllData) {
336
+ await withTimeout(
337
+ () => Promise.all([this.payloadEnvelopeDataPromise.promise, this.allDataPromise.promise]),
338
+ timeout,
339
+ signal
340
+ );
341
+ }
342
+ return this;
343
+ }
344
+
318
345
  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<gloas.DataColumnSidecar[]> {
319
346
  if (this.state.hasComputedAllData) {
320
347
  return Promise.resolve(this.getSampledColumns());
@@ -16,6 +16,11 @@ enum PayloadEnvelopeImportStatus {
16
16
 
17
17
  /**
18
18
  * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other.
19
+ *
20
+ * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
21
+ * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
22
+ * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
23
+ * waiting for columns. Duplicate triggers for the same payloadInput are deduped via `importStatus`.
19
24
  */
20
25
  export class PayloadEnvelopeProcessor {
21
26
  readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
@@ -25,7 +30,7 @@ export class PayloadEnvelopeProcessor {
25
30
  this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>(
26
31
  (payloadInput, opts) => {
27
32
  this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing);
28
- return importExecutionPayload.call(chain, payloadInput, opts);
33
+ return importExecutionPayload.call(chain, payloadInput, signal, opts);
29
34
  },
30
35
  {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal},
31
36
  metrics?.payloadEnvelopeProcessorQueue ?? undefined
@@ -33,10 +38,6 @@ export class PayloadEnvelopeProcessor {
33
38
  }
34
39
 
35
40
  async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise<void> {
36
- if (!payloadInput.isComplete()) {
37
- return;
38
- }
39
-
40
41
  if (this.importStatus.get(payloadInput) !== undefined) {
41
42
  return;
42
43
  }
@@ -90,7 +90,7 @@ export type ImportBlockOpts = {
90
90
 
91
91
  type FullyVerifiedBlockBase = {
92
92
  blockInput: IBlockInput;
93
- postBlockState: IBeaconStateView;
93
+ postState: IBeaconStateView;
94
94
  parentBlockSlot: Slot;
95
95
  proposerBalanceDelta: number;
96
96
  dataAvailabilityStatus: DataAvailabilityStatus;
@@ -0,0 +1,38 @@
1
+ import {DataAvailabilityStatus} from "@lodestar/state-transition";
2
+ import {gloas} from "@lodestar/types";
3
+ import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
4
+
5
+ // we can now wait for full 12 seconds because sync and reconstruction will try pulling
6
+ // the data columns from the network anyway while the envelope is being processed
7
+ export const PAYLOAD_DATA_AVAILABILITY_TIMEOUT = 12_000;
8
+
9
+ /**
10
+ * Verifies that all payload envelope inputs have their data columns available.
11
+ * - Waits a max of PAYLOAD_DATA_AVAILABILITY_TIMEOUT for all data to be available
12
+ * - Returns the time at which all data was available
13
+ * - Returns the data availability status for each payload input
14
+ */
15
+ export async function verifyPayloadsDataAvailability(
16
+ payloadInputs: PayloadEnvelopeInput[],
17
+ signal: AbortSignal
18
+ ): Promise<{
19
+ dataAvailabilityStatuses: DataAvailabilityStatus[];
20
+ availableTime: number;
21
+ }> {
22
+ const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
23
+ for (const payloadInput of payloadInputs) {
24
+ if (!payloadInput.hasAllData()) {
25
+ promises.push(payloadInput.waitForAllData(PAYLOAD_DATA_AVAILABILITY_TIMEOUT, signal));
26
+ }
27
+ }
28
+ await Promise.all(promises);
29
+
30
+ const availableTime = Math.max(0, Math.max(...payloadInputs.map((payloadInput) => payloadInput.getTimeComplete())));
31
+ const dataAvailabilityStatuses: DataAvailabilityStatus[] = payloadInputs.map((payloadInput) =>
32
+ payloadInput.getBlobKzgCommitments().length === 0
33
+ ? DataAvailabilityStatus.NotRequired
34
+ : DataAvailabilityStatus.Available
35
+ );
36
+
37
+ return {dataAvailabilityStatuses, availableTime};
38
+ }
@@ -5,7 +5,6 @@ import {BeaconConfig} from "@lodestar/config";
5
5
  import {CheckpointWithPayloadStatus, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
6
6
  import {LoggerNode} from "@lodestar/logger/node";
7
7
  import {
8
- BUILDER_INDEX_SELF_BUILD,
9
8
  EFFECTIVE_BALANCE_INCREMENT,
10
9
  type ForkPostFulu,
11
10
  type ForkPostGloas,
@@ -24,7 +23,6 @@ import {
24
23
  getEffectiveBalancesFromStateBytes,
25
24
  isStatePostAltair,
26
25
  isStatePostElectra,
27
- isStatePostGloas,
28
26
  } from "@lodestar/state-transition";
29
27
  import {
30
28
  BeaconBlock,
@@ -93,8 +91,8 @@ import {
93
91
  } from "./opPools/index.js";
94
92
  import {IChainOptions} from "./options.js";
95
93
  import {PrepareNextSlotScheduler} from "./prepareNextSlot.js";
96
- import {computeNewStateRoot, computePayloadEnvelopeStateRoot} from "./produceBlock/computeNewStateRoot.js";
97
- import {AssembledBlockType, BlockType, ProduceFullGloas, ProduceResult} from "./produceBlock/index.js";
94
+ import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js";
95
+ import {AssembledBlockType, BlockType, ProduceResult} from "./produceBlock/index.js";
98
96
  import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produceBlock/produceBlockBody.js";
99
97
  import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js";
100
98
  import {ReprocessController} from "./reprocess.js";
@@ -118,7 +116,7 @@ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datasto
118
116
  import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
119
117
  import {CPStateDatastore} from "./stateCache/datastore/types.js";
120
118
  import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
121
- import {PersistentCheckpointStateCache, fcCheckpointToHexPayload} from "./stateCache/persistentCheckpointsCache.js";
119
+ import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
122
120
  import {CheckpointStateCache} from "./stateCache/types.js";
123
121
  import {ValidatorMonitor} from "./validatorMonitor.js";
124
122
 
@@ -685,8 +683,8 @@ export class BeaconChain implements IBeaconChain {
685
683
  checkpoint: CheckpointWithPayloadStatus
686
684
  ): {state: IBeaconStateView; executionOptimistic: boolean; finalized: boolean} | null {
687
685
  // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array
688
- const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
689
- const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHexPayload);
686
+ const checkpointHex = {epoch: checkpoint.epoch, rootHex: checkpoint.rootHex};
687
+ const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHex);
690
688
  if (cachedStateCtx) {
691
689
  const block = this.forkChoice.getBlockDefaultStatus(
692
690
  ssz.phase0.BeaconBlockHeader.hashTreeRoot(cachedStateCtx.latestBlockHeader)
@@ -705,8 +703,8 @@ export class BeaconChain implements IBeaconChain {
705
703
  async getStateOrBytesByCheckpoint(
706
704
  checkpoint: CheckpointWithPayloadStatus
707
705
  ): Promise<{state: IBeaconStateView | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> {
708
- const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
709
- const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload);
706
+ const checkpointHex = {epoch: checkpoint.epoch, rootHex: checkpoint.rootHex};
707
+ const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHex);
710
708
  if (cachedStateCtx) {
711
709
  const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root);
712
710
  const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
@@ -1062,7 +1060,7 @@ export class BeaconChain implements IBeaconChain {
1062
1060
  body,
1063
1061
  } as AssembledBlockType<T>;
1064
1062
 
1065
- const {newStateRoot, proposerReward, postBlockState} = computeNewStateRoot(this.metrics, state, block);
1063
+ const {newStateRoot, proposerReward} = computeNewStateRoot(this.metrics, state, block);
1066
1064
  block.stateRoot = newStateRoot;
1067
1065
  const blockRoot =
1068
1066
  produceResult.type === BlockType.Full
@@ -1071,26 +1069,9 @@ export class BeaconChain implements IBeaconChain {
1071
1069
  const blockRootHex = toRootHex(blockRoot);
1072
1070
 
1073
1071
  const fork = this.config.getForkName(slot);
1074
- if (isForkPostGloas(fork)) {
1075
- // TODO GLOAS: we should retire BlockType post-gloas, may need a new enum for self vs non-self built
1076
- if (produceResult.type !== BlockType.Full) {
1077
- throw Error(`Unexpected block type=${produceResult.type} for post-gloas fork=${fork}`);
1078
- }
1079
-
1080
- const gloasResult = produceResult as ProduceFullGloas;
1081
- const envelope: gloas.ExecutionPayloadEnvelope = {
1082
- payload: gloasResult.executionPayload,
1083
- executionRequests: gloasResult.executionRequests,
1084
- builderIndex: BUILDER_INDEX_SELF_BUILD,
1085
- beaconBlockRoot: blockRoot,
1086
- slot,
1087
- stateRoot: ZERO_HASH,
1088
- };
1089
- if (!isStatePostGloas(postBlockState)) {
1090
- throw Error(`Expected gloas+ post-state for execution payload envelope, got fork=${postBlockState.forkName}`);
1091
- }
1092
- const payloadEnvelopeStateRoot = computePayloadEnvelopeStateRoot(this.metrics, postBlockState, envelope);
1093
- gloasResult.payloadEnvelopeStateRoot = payloadEnvelopeStateRoot;
1072
+ // TODO GLOAS: we should retire BlockType post-gloas, may need a new enum for self vs non-self built
1073
+ if (isForkPostGloas(fork) && produceResult.type !== BlockType.Full) {
1074
+ throw Error(`Unexpected block type=${produceResult.type} for post-gloas fork=${fork}`);
1094
1075
  }
1095
1076
 
1096
1077
  // Track the produced block for consensus broadcast validations, later validation, etc.
@@ -1338,8 +1319,8 @@ export class BeaconChain implements IBeaconChain {
1338
1319
  checkpoint: CheckpointWithPayloadStatus,
1339
1320
  blockState: IBeaconStateView
1340
1321
  ): {state: IBeaconStateView; stateId: string; shouldWarn: boolean} {
1341
- const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
1342
- const state = this.regen.getCheckpointStateSync(checkpointHexPayload);
1322
+ const checkpointHex = {epoch: checkpoint.epoch, rootHex: checkpoint.rootHex};
1323
+ const state = this.regen.getCheckpointStateSync(checkpointHex);
1343
1324
  if (state) {
1344
1325
  return {state, stateId: "checkpoint_state", shouldWarn: false};
1345
1326
  }
@@ -7,6 +7,7 @@ import {DataColumnSidecar, RootHex, deneb, phase0} from "@lodestar/types";
7
7
  import {SignedExecutionPayloadEnvelope} from "@lodestar/types/gloas";
8
8
  import {PeerIdStr} from "../util/peerId.js";
9
9
  import {BlockInputSource, IBlockInput} from "./blocks/blockInput/types.js";
10
+ import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
10
11
 
11
12
  /**
12
13
  * Important chain events that occur during normal chain operation.
@@ -76,6 +77,11 @@ export enum ChainEvent {
76
77
  * cut-off window passes for waiting on gossip
77
78
  */
78
79
  incompleteBlockInput = "incompleteBlockInput",
80
+ /**
81
+ * Post-gloas: trigger BlockInputSync for payload envelopes whose envelope and/or sampled columns are partially
82
+ * received via gossip but are not complete by time the cut-off window passes for waiting on gossip
83
+ */
84
+ incompletePayloadEnvelope = "incompletePayloadEnvelope",
79
85
  }
80
86
 
81
87
  export type HeadEventData = routes.events.EventData[routes.events.EventType.head];
@@ -93,6 +99,11 @@ export type ChainEventData = {
93
99
  };
94
100
  [ChainEvent.unknownBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
95
101
  [ChainEvent.incompleteBlockInput]: {blockInput: IBlockInput; peer: PeerIdStr; source: BlockInputSource};
102
+ [ChainEvent.incompletePayloadEnvelope]: {
103
+ payloadInput: PayloadEnvelopeInput;
104
+ peer: PeerIdStr;
105
+ source: BlockInputSource;
106
+ };
96
107
  [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
97
108
  };
98
109
 
@@ -116,6 +127,7 @@ export type IChainEvents = ApiEvents & {
116
127
  [ChainEvent.envelopeUnknownBlock]: (data: ChainEventData[ChainEvent.envelopeUnknownBlock]) => void;
117
128
  [ChainEvent.unknownBlockRoot]: (data: ChainEventData[ChainEvent.unknownBlockRoot]) => void;
118
129
  [ChainEvent.incompleteBlockInput]: (data: ChainEventData[ChainEvent.incompleteBlockInput]) => void;
130
+ [ChainEvent.incompletePayloadEnvelope]: (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]) => void;
119
131
  [ChainEvent.unknownEnvelopeBlockRoot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]) => void;
120
132
  };
121
133
 
@@ -150,7 +150,9 @@ export function initializeForkChoiceFromFinalizedState(
150
150
 
151
151
  ...(isStatePostBellatrix(state) && state.isExecutionStateType && state.isMergeTransitionComplete
152
152
  ? {
153
- executionPayloadBlockHash: toRootHex(state.latestBlockHash),
153
+ executionPayloadBlockHash: isStatePostGloas(state)
154
+ ? toRootHex(state.latestBlockHash)
155
+ : toRootHex(state.latestExecutionPayloadHeader.blockHash),
154
156
  // TODO GLOAS: executionPayloadNumber is not tracked in BeaconState post-gloas (EIP-7732 removed
155
157
  // latestExecutionPayloadHeader). Using 0 as unavailable fallback until a solution is found.
156
158
  executionPayloadNumber: isStatePostGloas(state) ? 0 : state.payloadBlockNumber,
@@ -247,7 +249,9 @@ export function initializeForkChoiceFromUnfinalizedState(
247
249
  unfinalizedState.isExecutionStateType &&
248
250
  unfinalizedState.isMergeTransitionComplete
249
251
  ? {
250
- executionPayloadBlockHash: toRootHex(unfinalizedState.latestBlockHash),
252
+ executionPayloadBlockHash: isStatePostGloas(unfinalizedState)
253
+ ? toRootHex(unfinalizedState.latestBlockHash)
254
+ : toRootHex(unfinalizedState.latestExecutionPayloadHeader.blockHash),
251
255
  // TODO GLOAS: executionPayloadNumber is not tracked in BeaconState post-gloas (EIP-7732 removed
252
256
  // latestExecutionPayloadHeader). Using 0 as unavailable fallback until a solution is found.
253
257
  executionPayloadNumber: isStatePostGloas(unfinalizedState) ? 0 : unfinalizedState.payloadBlockNumber,
@@ -1,6 +1,6 @@
1
1
  import {routes} from "@lodestar/api";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
- import {PayloadStatus, getSafeExecutionBlockHash} from "@lodestar/fork-choice";
3
+ import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
4
4
  import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params";
5
5
  import {
6
6
  IBeaconStateView,
@@ -8,8 +8,9 @@ import {
8
8
  computeEpochAtSlot,
9
9
  computeTimeAtSlot,
10
10
  isStatePostBellatrix,
11
+ isStatePostGloas,
11
12
  } from "@lodestar/state-transition";
12
- import {Slot} from "@lodestar/types";
13
+ import {Bytes32, Slot} from "@lodestar/types";
13
14
  import {Logger, fromHex, isErrorAborted, sleep} from "@lodestar/utils";
14
15
  import {GENESIS_SLOT, ZERO_HASH_HEX} from "../constants/constants.js";
15
16
  import {BuilderStatus} from "../execution/builder/http.js";
@@ -81,6 +82,8 @@ export class PrepareNextSlotScheduler {
81
82
  // calling updateHead() here before we produce a block to reduce reorg possibility
82
83
  const headBlock = this.chain.recomputeForkChoiceHead(ForkchoiceCaller.prepareNextSlot);
83
84
  const {slot: headSlot, blockRoot: headRoot} = headBlock;
85
+ // may be updated below if we predict a proposer-boost-reorg
86
+ let updatedHeadRoot = headRoot;
84
87
 
85
88
  // PS: previously this was comparing slots, but that gave no leway on the skipped
86
89
  // slots on epoch bounday. Making it more fluid.
@@ -123,7 +126,6 @@ export class PrepareNextSlotScheduler {
123
126
  const proposerIndex = prepareState.getBeaconProposer(prepareSlot);
124
127
  const feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
125
128
  let updatedPrepareState = prepareState;
126
- let updatedHeadRoot = headRoot;
127
129
 
128
130
  if (feeRecipient) {
129
131
  // If we are proposing next slot, we need to predict if we can proposer-boost-reorg or not
@@ -156,25 +158,39 @@ export class PrepareNextSlotScheduler {
156
158
  this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
157
159
  });
158
160
  }
161
+ }
162
+
163
+ if (!isStatePostBellatrix(updatedPrepareState)) {
164
+ throw new Error("Expected Bellatrix state for payload attributes");
165
+ }
166
+
167
+ let parentBlockHash: Bytes32;
168
+ if (isStatePostGloas(updatedPrepareState)) {
169
+ parentBlockHash = this.chain.forkChoice.shouldExtendPayload(updatedHeadRoot)
170
+ ? updatedPrepareState.latestExecutionPayloadBid.blockHash
171
+ : updatedPrepareState.latestExecutionPayloadBid.parentBlockHash;
172
+ } else {
173
+ parentBlockHash = updatedPrepareState.latestExecutionPayloadHeader.blockHash;
174
+ }
159
175
 
176
+ if (feeRecipient) {
160
177
  const preparationTime =
161
178
  computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
162
179
  this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
163
- if (!isStatePostBellatrix(updatedPrepareState)) {
164
- throw new Error("Expected Bellatrix state for payload preparation");
165
- }
166
180
 
167
181
  const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice);
168
182
  const finalizedBlockHash =
169
183
  this.chain.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
184
+
170
185
  // awaiting here instead of throwing an async call because there is no other task
171
- // left for scheduler and this gives nice sematics to catch and log errors in the
186
+ // left for scheduler and this gives nice semantics to catch and log errors in the
172
187
  // try/catch wrapper here.
173
188
  await prepareExecutionPayload(
174
189
  this.chain,
175
190
  this.logger,
176
191
  fork as ForkPostBellatrix, // State is of execution type
177
192
  fromHex(updatedHeadRoot),
193
+ parentBlockHash,
178
194
  safeBlockHash,
179
195
  finalizedBlockHash,
180
196
  updatedPrepareState,
@@ -187,10 +203,6 @@ export class PrepareNextSlotScheduler {
187
203
  });
188
204
  }
189
205
 
190
- if (!isStatePostBellatrix(updatedPrepareState)) {
191
- throw new Error("Expected Bellatrix state for payload attributes");
192
- }
193
-
194
206
  this.computeStateHashTreeRoot(updatedPrepareState, isEpochTransition);
195
207
 
196
208
  // If emitPayloadAttributes is true emit a SSE payloadAttributes event
@@ -202,6 +214,7 @@ export class PrepareNextSlotScheduler {
202
214
  prepareState: updatedPrepareState,
203
215
  prepareSlot,
204
216
  parentBlockRoot: fromHex(headRoot),
217
+ parentBlockHash,
205
218
  // The likely consumers of this API are builders and will anyway ignore the
206
219
  // feeRecipient, so just pass zero hash for now till a real use case arises
207
220
  feeRecipient: "0x0000000000000000000000000000000000000000000000000000000000000000",
@@ -217,11 +230,7 @@ export class PrepareNextSlotScheduler {
217
230
  // + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations
218
231
  if (isEpochTransition) {
219
232
  this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1);
220
- // Determine payloadPresent from head block's payload status
221
- // Pre-Gloas: payloadStatus is always FULL → payloadPresent = true
222
- // Post-Gloas: FULL → true, EMPTY → false, PENDING → false (conservative, treat as block state)
223
- const payloadPresent = headBlock.payloadStatus === PayloadStatus.FULL;
224
- const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch, payloadPresent);
233
+ const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch);
225
234
  if (previousHits === 0) {
226
235
  this.metrics?.precomputeNextEpochTransition.waste.inc();
227
236
  }