@lodestar/beacon-node 1.51.0-dev.e22f67e7bc → 1.51.0-dev.e8f7876d67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/lib/api/impl/api.js +1 -1
  2. package/lib/api/impl/api.js.map +1 -1
  3. package/lib/api/impl/beacon/blocks/index.d.ts +2 -1
  4. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/blocks/index.js +12 -7
  6. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  7. package/lib/api/impl/beacon/index.d.ts +2 -1
  8. package/lib/api/impl/beacon/index.d.ts.map +1 -1
  9. package/lib/api/impl/beacon/index.js +2 -2
  10. package/lib/api/impl/beacon/index.js.map +1 -1
  11. package/lib/api/impl/validator/index.d.ts.map +1 -1
  12. package/lib/api/impl/validator/index.js +4 -1
  13. package/lib/api/impl/validator/index.js.map +1 -1
  14. package/lib/api/options.d.ts +2 -0
  15. package/lib/api/options.d.ts.map +1 -1
  16. package/lib/api/options.js +1 -0
  17. package/lib/api/options.js.map +1 -1
  18. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +4 -0
  19. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  20. package/lib/chain/archiveStore/utils/archiveBlocks.js +61 -97
  21. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  22. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  23. package/lib/chain/blocks/importBlock.js +6 -5
  24. package/lib/chain/blocks/importBlock.js.map +1 -1
  25. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  26. package/lib/chain/blocks/importExecutionPayload.js +3 -2
  27. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  28. package/lib/chain/blocks/index.js +2 -2
  29. package/lib/chain/blocks/index.js.map +1 -1
  30. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -5
  31. package/lib/chain/blocks/payloadEnvelopeProcessor.js +5 -5
  32. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  33. package/lib/chain/blocks/verifyBlock.js +1 -1
  34. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  35. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +3 -3
  36. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -1
  37. package/lib/chain/blocks/verifyPayloadsDataAvailability.js +6 -6
  38. package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -1
  39. package/lib/chain/chain.js +4 -4
  40. package/lib/chain/chain.js.map +1 -1
  41. package/lib/chain/opPools/inclusionListStore.d.ts.map +1 -1
  42. package/lib/chain/opPools/inclusionListStore.js +2 -2
  43. package/lib/chain/opPools/inclusionListStore.js.map +1 -1
  44. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  45. package/lib/chain/produceBlock/produceBlockBody.js +0 -1
  46. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  47. package/lib/chain/regen/interface.d.ts +0 -1
  48. package/lib/chain/regen/interface.d.ts.map +1 -1
  49. package/lib/chain/regen/interface.js +0 -1
  50. package/lib/chain/regen/interface.js.map +1 -1
  51. package/lib/chain/validation/executionPayloadBid.d.ts +0 -15
  52. package/lib/chain/validation/executionPayloadBid.d.ts.map +1 -1
  53. package/lib/chain/validation/executionPayloadBid.js +0 -153
  54. package/lib/chain/validation/executionPayloadBid.js.map +1 -1
  55. package/lib/chain/validation/inclusionList.d.ts.map +1 -1
  56. package/lib/chain/validation/inclusionList.js +11 -2
  57. package/lib/chain/validation/inclusionList.js.map +1 -1
  58. package/lib/db/beacon.d.ts +7 -0
  59. package/lib/db/beacon.d.ts.map +1 -1
  60. package/lib/db/beacon.js +34 -0
  61. package/lib/db/beacon.js.map +1 -1
  62. package/lib/db/dataColumnStore.d.ts +1 -1
  63. package/lib/db/dataColumnStore.d.ts.map +1 -1
  64. package/lib/db/dataColumnStore.js +1 -4
  65. package/lib/db/dataColumnStore.js.map +1 -1
  66. package/lib/db/interface.d.ts +1 -0
  67. package/lib/db/interface.d.ts.map +1 -1
  68. package/lib/db/repositories/dataColumnSidecarArchive.d.ts +6 -0
  69. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  70. package/lib/db/repositories/dataColumnSidecarArchive.js +19 -0
  71. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  72. package/lib/execution/engine/blobVersionedHashes.d.ts +1 -1
  73. package/lib/execution/engine/blobVersionedHashes.d.ts.map +1 -1
  74. package/lib/execution/engine/blobVersionedHashes.js.map +1 -1
  75. package/lib/execution/engine/http.d.ts.map +1 -1
  76. package/lib/execution/engine/http.js +0 -1
  77. package/lib/execution/engine/http.js.map +1 -1
  78. package/lib/execution/engine/mock.d.ts.map +1 -1
  79. package/lib/execution/engine/mock.js +6 -4
  80. package/lib/execution/engine/mock.js.map +1 -1
  81. package/lib/execution/engine/payloadIdCache.d.ts +1 -2
  82. package/lib/execution/engine/payloadIdCache.d.ts.map +1 -1
  83. package/lib/execution/engine/payloadIdCache.js +3 -3
  84. package/lib/execution/engine/payloadIdCache.js.map +1 -1
  85. package/lib/execution/engine/sszTypes.d.ts +38 -38
  86. package/lib/execution/engine/types.d.ts +1 -1
  87. package/lib/execution/engine/types.d.ts.map +1 -1
  88. package/lib/execution/engine/types.js +7 -6
  89. package/lib/execution/engine/types.js.map +1 -1
  90. package/lib/index.d.ts +1 -0
  91. package/lib/index.d.ts.map +1 -1
  92. package/lib/index.js +2 -0
  93. package/lib/index.js.map +1 -1
  94. package/lib/metrics/metrics/lodestar.d.ts +3 -0
  95. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  96. package/lib/metrics/metrics/lodestar.js +5 -0
  97. package/lib/metrics/metrics/lodestar.js.map +1 -1
  98. package/lib/network/gossip/topic.d.ts +6 -6
  99. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  100. package/lib/network/processor/gossipHandlers.js +27 -18
  101. package/lib/network/processor/gossipHandlers.js.map +1 -1
  102. package/lib/node/nodejs.d.ts.map +1 -1
  103. package/lib/node/nodejs.js +9 -1
  104. package/lib/node/nodejs.js.map +1 -1
  105. package/lib/sync/unknownBlock.d.ts.map +1 -1
  106. package/lib/sync/unknownBlock.js +42 -18
  107. package/lib/sync/unknownBlock.js.map +1 -1
  108. package/package.json +30 -30
  109. package/src/api/impl/api.ts +1 -1
  110. package/src/api/impl/beacon/blocks/index.ts +22 -17
  111. package/src/api/impl/beacon/index.ts +3 -1
  112. package/src/api/impl/validator/index.ts +4 -1
  113. package/src/api/options.ts +3 -0
  114. package/src/chain/archiveStore/utils/archiveBlocks.ts +76 -130
  115. package/src/chain/blocks/importBlock.ts +8 -7
  116. package/src/chain/blocks/importExecutionPayload.ts +3 -2
  117. package/src/chain/blocks/index.ts +6 -2
  118. package/src/chain/blocks/payloadEnvelopeProcessor.ts +5 -5
  119. package/src/chain/blocks/verifyBlock.ts +1 -0
  120. package/src/chain/blocks/verifyPayloadsDataAvailability.ts +7 -6
  121. package/src/chain/chain.ts +4 -4
  122. package/src/chain/opPools/inclusionListStore.ts +3 -3
  123. package/src/chain/produceBlock/produceBlockBody.ts +0 -1
  124. package/src/chain/regen/interface.ts +0 -1
  125. package/src/chain/validation/executionPayloadBid.ts +0 -180
  126. package/src/chain/validation/inclusionList.ts +11 -2
  127. package/src/db/beacon.ts +44 -0
  128. package/src/db/dataColumnStore.ts +2 -4
  129. package/src/db/interface.ts +2 -0
  130. package/src/db/repositories/dataColumnSidecarArchive.ts +20 -0
  131. package/src/execution/engine/blobVersionedHashes.ts +1 -1
  132. package/src/execution/engine/http.ts +0 -1
  133. package/src/execution/engine/mock.ts +6 -4
  134. package/src/execution/engine/payloadIdCache.ts +6 -7
  135. package/src/execution/engine/types.ts +8 -8
  136. package/src/index.ts +2 -0
  137. package/src/metrics/metrics/lodestar.ts +5 -0
  138. package/src/network/processor/gossipHandlers.ts +35 -20
  139. package/src/node/nodejs.ts +18 -1
  140. package/src/sync/unknownBlock.ts +47 -28
@@ -1,11 +1,18 @@
1
1
  import path from "node:path";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
3
  import {KeyValue} from "@lodestar/db";
4
- import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
4
+ import {
5
+ CheckpointWithHex,
6
+ ExecutionStatus,
7
+ IForkChoice,
8
+ PayloadStatus,
9
+ ProtoBlock,
10
+ isGloasBlock,
11
+ } from "@lodestar/fork-choice";
5
12
  import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
6
13
  import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
7
14
  import {Epoch, Slot} from "@lodestar/types";
8
- import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
15
+ import {Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
9
16
  import {IBeaconDb} from "../../../db/index.js";
10
17
  import {BlockArchiveBatchPutBinaryItem, encodeArchivedHeaderEnvelope} from "../../../db/repositories/index.js";
11
18
  import {Metrics} from "../../../metrics/metrics.js";
@@ -49,6 +56,17 @@ async function persistOrphanedBlock(
49
56
  await writeIfNotExist(filepath, bytes);
50
57
  }
51
58
 
59
+ /**
60
+ * Oldest epoch for which data column sidecars are kept
61
+ */
62
+ export function getDataColumnSidecarsMinEpoch(
63
+ config: ChainForkConfig,
64
+ currentEpoch: Epoch,
65
+ archiveDataEpochs?: number
66
+ ): Epoch {
67
+ return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
68
+ }
69
+
52
70
  /**
53
71
  * Archives finalized blocks from active bucket to archive bucket.
54
72
  *
@@ -136,23 +154,6 @@ export async function archiveBlocks(
136
154
  logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
137
155
  }
138
156
 
139
- // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
140
- // directly to flat files, so this only moves fallback data left by older versions.
141
- if (finalizedPostFulu) {
142
- const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
143
- config,
144
- db,
145
- logger,
146
- finalizedCanonicalBlocks,
147
- currentEpoch
148
- );
149
- logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
150
- ...logCtx,
151
- migratedEntries: migratedSlots.length,
152
- slotRange: prettyPrintIndices(migratedSlots),
153
- });
154
- }
155
-
156
157
  if (finalizedPostGloas) {
157
158
  const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
158
159
  config,
@@ -169,44 +170,34 @@ export async function archiveBlocks(
169
170
  }
170
171
  }
171
172
 
172
- // deleteNonCanonicalBlocks
173
- // loop through forkchoice single time
173
+ const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
174
+ const orphanedBlocks: ProtoBlock[] = [];
175
+ const orphanedPayloads: ProtoBlock[] = [];
174
176
 
175
- const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
176
- if (nonCanonicalBlockRoots.length > 0) {
177
- if (persistOrphanedBlocks) {
178
- // Persist orphaned blocks to disk before deleting them from hot db
179
- await Promise.all(
180
- nonCanonicalBlockRoots.map(async (root, index) => {
181
- const block = finalizedNonCanonicalBlocks[index];
182
- const blockBytes = await db.block.getBinary(root);
183
- const blockLogCtx = {slot: block.slot, root: block.blockRoot};
184
- if (blockBytes) {
185
- await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
186
- persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
187
- });
188
- logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
189
- } else {
190
- logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
191
- }
192
- })
193
- );
177
+ for (const block of finalizedNonCanonicalBlocks) {
178
+ if (block.payloadStatus === PayloadStatus.FULL) {
179
+ orphanedPayloads.push(block);
194
180
  }
181
+ // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
182
+ // already handled the root's block-level data.
183
+ const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
184
+ if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
185
+ orphanedBlocks.push(block);
186
+ }
187
+ }
188
+ const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
195
189
 
196
- const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
197
- const nonCanonicalLogCtx = {
190
+ if (orphanedPayloads.length > 0) {
191
+ const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
192
+ const orphanedPayloadLogCtx = {
198
193
  ...logCtx,
199
- count: nonCanonicalBlockRoots.length,
200
- slotRange: prettyPrintIndices(nonCanonicalSlots),
194
+ count: orphanedPayloads.length,
195
+ slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
201
196
  };
202
197
 
203
- const columnItems = finalizedNonCanonicalBlocks
204
- // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
205
- // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
206
- .filter(
207
- (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
208
- )
209
- .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
198
+ const columnItems = orphanedPayloads
199
+ .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
200
+ .map((block) => ({slot: block.slot, blockRoot: block.blockRoot}));
210
201
  if (columnItems.length > 0) {
211
202
  // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
212
203
  await db.dataColumns.deleteMany(columnItems);
@@ -217,20 +208,44 @@ export async function archiveBlocks(
217
208
  });
218
209
  }
219
210
 
220
- await db.block.batchDelete(nonCanonicalBlockRoots);
221
- logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
222
-
223
211
  if (finalizedPostDeneb) {
224
- await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
225
- logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
212
+ await db.blobSidecars.batchDelete(orphanedPayloadRoots);
213
+ logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
226
214
  }
227
215
 
228
216
  if (finalizedPostGloas) {
229
- await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
230
- logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
217
+ await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
218
+ logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
231
219
  }
232
220
  }
233
221
 
222
+ if (orphanedBlocks.length > 0) {
223
+ if (persistOrphanedBlocks) {
224
+ // Persist orphaned blocks to disk before deleting them from hot db
225
+ await Promise.all(
226
+ orphanedBlocks.map(async (block) => {
227
+ const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
228
+ const blockLogCtx = {slot: block.slot, root: block.blockRoot};
229
+ if (blockBytes) {
230
+ await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
231
+ persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
232
+ });
233
+ logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
234
+ } else {
235
+ logger.debug("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
236
+ }
237
+ })
238
+ );
239
+ }
240
+
241
+ await db.block.batchDelete(orphanedBlockRoots);
242
+ logger.verbose("Deleted non canonical blocks from hot DB", {
243
+ ...logCtx,
244
+ count: orphanedBlocks.length,
245
+ slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
246
+ });
247
+ }
248
+
234
249
  // Delete expired blobs
235
250
  // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
236
251
  // if archiveDataEpochs set to Infinity do not prune`
@@ -256,10 +271,7 @@ export async function archiveBlocks(
256
271
  // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
257
272
  if (finalizedPostFulu) {
258
273
  if (archiveDataEpochs !== Infinity) {
259
- const dataColumnSidecarsMinEpoch = Math.min(
260
- getBlobDataRetentionStartEpoch(config, currentEpoch),
261
- currentEpoch - (archiveDataEpochs ?? 0)
262
- );
274
+ const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
263
275
  if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
264
276
  const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
265
277
  const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -283,9 +295,9 @@ export async function archiveBlocks(
283
295
  }
284
296
  }
285
297
 
286
- // Prunning potential checkpoint data
298
+ // Pruning potential checkpoint data
287
299
  const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
288
- const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots];
300
+ const nonCheckpointBlockRoots: Uint8Array[] = [...orphanedBlockRoots];
289
301
  for (const block of finalizedCanonicalNonCheckpointBlocks) {
290
302
  nonCheckpointBlockRoots.push(block.root);
291
303
  }
@@ -408,72 +420,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
408
420
  return migratedWrappedBlobSidecars;
409
421
  }
410
422
 
411
- // TODO: This function can be simplified further by reducing layers of promises in a loop
412
- /**
413
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
414
- * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
415
- * blocks only have a FULL variant, so the payload-status filter passes them all.
416
- */
417
- async function migrateDataColumnSidecarsFromHotToColdDb(
418
- config: ChainForkConfig,
419
- db: IBeaconDb,
420
- logger: Logger,
421
- canonicalBlocks: ProtoBlock[],
422
- currentEpoch: Epoch
423
- ): Promise<Slot[]> {
424
- const columnBlocks = canonicalBlocks.filter(
425
- (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
426
- );
427
- if (columnBlocks.length === 0) return [];
428
- const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
429
-
430
- const migratedSlots: Slot[] = [];
431
- for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
432
- const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
433
- const batch = blocks.slice(i, toIdx);
434
- if (batch.length === 0) break;
435
-
436
- const promises: Promise<void>[] = [];
437
- for (const block of batch) {
438
- const blockEpoch = computeEpochAtSlot(block.slot);
439
- if (
440
- config.getForkSeq(block.slot) < ForkSeq.fulu ||
441
- blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)
442
- ) {
443
- continue;
444
- }
445
-
446
- const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
447
- if (dataColumnSidecarBytes.length === 0) {
448
- logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
449
- slot: block.slot,
450
- root: toRootHex(block.root),
451
- });
452
- continue;
453
- }
454
- logger.verbose("Migrated legacy dataColumnSidecars for block", {
455
- currentEpoch,
456
- slot: block.slot,
457
- root: toRootHex(block.root),
458
- numSidecars: dataColumnSidecarBytes.length,
459
- });
460
- promises.push(
461
- db.dataColumnSidecarArchive.putManyBinary(
462
- block.slot,
463
- dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
464
- )
465
- );
466
- migratedSlots.push(block.slot);
467
- }
468
-
469
- await Promise.all(promises);
470
- await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
471
- }
472
-
473
- // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
474
- return migratedSlots.sort((a, b) => a - b);
475
- }
476
-
477
423
  /**
478
424
  * Post-gloas given a finalized checkpoint at a block root, payload of that block root
479
425
  * is not considered finalized, hence they are archived in the next run.
@@ -21,7 +21,7 @@ import {
21
21
  IBeaconStateView,
22
22
  RootCache,
23
23
  computeEpochAtSlot,
24
- computeTimeAtSlot,
24
+ computeTimeAtSlotMs,
25
25
  getBeaconCommittees,
26
26
  isStatePostAltair,
27
27
  isStatePostBellatrix,
@@ -105,8 +105,9 @@ export async function importBlock(
105
105
  const currentEpoch = computeEpochAtSlot(currentSlot);
106
106
  const blockEpoch = computeEpochAtSlot(blockSlot);
107
107
  const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
108
- const receiveDelaySec =
109
- fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
108
+ const receiveDelayMs =
109
+ Math.round(fullyVerifiedBlock.seenTimestampSec * 1000) -
110
+ computeTimeAtSlotMs(this.config, blockSlot, postState.genesisTime * 1000);
110
111
  const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
111
112
  const fork = this.config.getForkSeq(blockSlot);
112
113
 
@@ -148,12 +149,12 @@ export async function importBlock(
148
149
  executionStatus = parentBlock.executionStatus;
149
150
  }
150
151
 
151
- const importDelaySec = this.clock.secFromSlot(blockSlot);
152
+ const importDelayMs = this.clock.msFromSlot(blockSlot);
152
153
  const blockSummary = this.forkChoice.onBlock(
153
154
  block.message,
154
155
  postState,
155
- receiveDelaySec,
156
- importDelaySec,
156
+ receiveDelayMs,
157
+ importDelayMs,
157
158
  currentSlot,
158
159
  executionStatus,
159
160
  dataAvailabilityStatus
@@ -417,7 +418,7 @@ export async function importBlock(
417
418
  // 4) Block meets the criteria of being re-orged out (this is also checked inside the function)
418
419
  const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
419
420
  blockSummary,
420
- this.clock.secFromSlot(currentSlot),
421
+ this.clock.msFromSlot(currentSlot),
421
422
  currentSlot
422
423
  );
423
424
  shouldOverrideFcu = result.shouldOverrideFcu;
@@ -1,3 +1,4 @@
1
+ import {SerializedList} from "@chainsafe/ssz";
1
2
  import {routes} from "@lodestar/api";
2
3
  import {
3
4
  ExecutionStatus,
@@ -206,7 +207,7 @@ export async function importExecutionPayload(
206
207
  }
207
208
  inclusionListTransactions =
208
209
  dependentRoot === null
209
- ? []
210
+ ? SerializedList.empty()
210
211
  : this.inclusionListStore.getInclusionListTransactions(inclusionListSlot, dependentRoot, true);
211
212
  }
212
213
 
@@ -382,6 +383,6 @@ export async function processExecutionPayload(
382
383
  signal: AbortSignal,
383
384
  opts: ImportPayloadOpts = {}
384
385
  ): Promise<void> {
385
- const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability([payloadInput], signal);
386
+ const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(this.config, [payloadInput], signal);
386
387
  await importExecutionPayload.call(this, payloadInput, dataAvailabilityStatuses[0], opts);
387
388
  }
@@ -147,7 +147,7 @@ export async function processBlocks(
147
147
  proposerBalanceDelta: proposerBalanceDeltas[i],
148
148
  indexedAttestations: indexedAttestationsByBlock[i],
149
149
  // TODO: Make this param mandatory and capture in gossip
150
- seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
150
+ seenTimestampSec: opts.seenTimestampSec ?? Date.now() / 1000,
151
151
  });
152
152
  }
153
153
 
@@ -292,7 +292,11 @@ async function importPayloadEnvelopesOfKnownBlocks(
292
292
  return result;
293
293
  }
294
294
 
295
- const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
295
+ const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
296
+ this.config,
297
+ payloadInputs,
298
+ new AbortController().signal
299
+ );
296
300
  for (let i = 0; i < payloadInputs.length; i++) {
297
301
  this.logger.debug("Importing payload envelope of known block", {
298
302
  slot: payloadInputs[i].slot,
@@ -13,11 +13,11 @@ const QUEUE_MAX_LENGTH = 16;
13
13
  *
14
14
  * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
15
15
  * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
16
- * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
17
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
18
- * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
19
- * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
20
- * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
16
+ * internally, so a queued job can pend for up to one slot while waiting for columns. Duplicate
17
+ * triggers for the same payloadInput are deduped by sharing the in-flight import promise: every
18
+ * caller observes the real import outcome. Resolving duplicates early instead would let
19
+ * BlockInputSync mark a still-pending payload as processed, re-queue it, and spin re-processing
20
+ * the same root until OOM while the import is blocked (e.g. EL syncing).
21
21
  */
22
22
  export class PayloadEnvelopeProcessor {
23
23
  readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
@@ -143,6 +143,7 @@ export async function verifyBlocksInEpoch(
143
143
  ? Array.from(payloadEnvelopes.values()).filter((payloadInput) => payloadInput.hasPayloadEnvelope())
144
144
  : [];
145
145
  const {dataAvailabilityStatuses, availableTime} = await verifyPayloadsDataAvailability(
146
+ this.config,
146
147
  payloadInputsForDa,
147
148
  abortController.signal
148
149
  );
@@ -1,18 +1,16 @@
1
+ import {ChainForkConfig} from "@lodestar/config";
1
2
  import {DataAvailabilityStatus} from "@lodestar/state-transition";
2
3
  import {gloas} from "@lodestar/types";
3
4
  import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
4
5
 
5
- // we can now wait for full 12 seconds because sync and reconstruction will try pulling
6
- // the data columns from the network anyway while the envelope is being processed
7
- export const PAYLOAD_DATA_AVAILABILITY_TIMEOUT = 12_000;
8
-
9
6
  /**
10
7
  * Verifies that all payload envelope inputs have their data columns available.
11
- * - Waits a max of PAYLOAD_DATA_AVAILABILITY_TIMEOUT for all data to be available
8
+ * - Waits a max of one slot duration of the payload's fork for all data to be available
12
9
  * - Returns the time at which all data was available
13
10
  * - Returns the data availability status for each payload input
14
11
  */
15
12
  export async function verifyPayloadsDataAvailability(
13
+ config: ChainForkConfig,
16
14
  payloadInputs: PayloadEnvelopeInput[],
17
15
  signal: AbortSignal
18
16
  ): Promise<{
@@ -22,7 +20,10 @@ export async function verifyPayloadsDataAvailability(
22
20
  const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
23
21
  for (const payloadInput of payloadInputs) {
24
22
  if (!payloadInput.hasAllData()) {
25
- promises.push(payloadInput.waitForAllData(PAYLOAD_DATA_AVAILABILITY_TIMEOUT, signal));
23
+ // Sync and reconstruction keep pulling the data columns from the network while we wait. The wait is
24
+ // capped at one slot because the payload queue is serial: a longer wait lets consecutive unavailable
25
+ // payloads push every later payload import further behind with each slot.
26
+ promises.push(payloadInput.waitForAllData(config.getSlotDurationMs(payloadInput.forkName), signal));
26
27
  }
27
28
  }
28
29
  await Promise.all(promises);
@@ -1486,10 +1486,10 @@ export class BeaconChain implements IBeaconChain {
1486
1486
  this.metrics?.forkChoice.requests.inc();
1487
1487
  const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
1488
1488
  const slot = this.clock.currentSlot;
1489
- const secFromSlot = this.clock.secFromSlot(slot);
1489
+ const msFromSlot = this.clock.msFromSlot(slot);
1490
1490
 
1491
1491
  try {
1492
- return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, secFromSlot, slot}).head;
1492
+ return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, msFromSlot, slot}).head;
1493
1493
  } catch (e) {
1494
1494
  this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetPredictedProposerHead});
1495
1495
  throw e;
@@ -1501,13 +1501,13 @@ export class BeaconChain implements IBeaconChain {
1501
1501
  getProposerHead(slot: Slot): ProtoBlock {
1502
1502
  this.metrics?.forkChoice.requests.inc();
1503
1503
  const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.getProposerHead});
1504
- const secFromSlot = this.clock.secFromSlot(slot);
1504
+ const msFromSlot = this.clock.msFromSlot(slot);
1505
1505
 
1506
1506
  try {
1507
1507
  // Do not emit head event here, when proposing we rely on the one emitted when importing our own block
1508
1508
  const {head, isHeadTimely, notReorgedReason} = this.forkChoice.updateAndGetHead({
1509
1509
  mode: UpdateHeadOpt.GetProposerHead,
1510
- secFromSlot,
1510
+ msFromSlot,
1511
1511
  slot,
1512
1512
  });
1513
1513
 
@@ -1,4 +1,4 @@
1
- import {BitArray} from "@chainsafe/ssz";
1
+ import {BitArray, SerializedList} from "@chainsafe/ssz";
2
2
  import {BeaconConfig} from "@lodestar/config";
3
3
  import {INCLUSION_LIST_COMMITTEE_SIZE} from "@lodestar/params";
4
4
  import {RootHex, Slot, ValidatorIndex, bellatrix, heze, ssz} from "@lodestar/types";
@@ -108,7 +108,7 @@ export class InclusionListStore {
108
108
 
109
109
  /** Deduplicated transactions from valid, non-equivocating inclusion lists at `(slot, dependentRoot)`. */
110
110
  getInclusionListTransactions(slot: Slot, dependentRoot: RootHex, onlyTimely = true): bellatrix.Transactions {
111
- const transactions: bellatrix.Transactions = [];
111
+ const transactions: Uint8Array[] = [];
112
112
  const seen = new Set<string>();
113
113
 
114
114
  for (const {signedInclusionList} of this.getEligible(slot, dependentRoot, onlyTimely)) {
@@ -121,7 +121,7 @@ export class InclusionListStore {
121
121
  }
122
122
  }
123
123
 
124
- return transactions;
124
+ return SerializedList.fromArray(transactions);
125
125
  }
126
126
 
127
127
  /** Bit `i` is set when committee member `i` submitted a valid, non-equivocating inclusion list */
@@ -781,7 +781,6 @@ export async function prepareExecutionPayload(
781
781
 
782
782
  const payloadIdCached = chain.executionEngine.payloadIdCache.get({
783
783
  headBlockHash: toRootHex(parentBlockHash),
784
- finalizedBlockHash,
785
784
  timestamp: numToQuantity(timestamp),
786
785
  prevRandao: toHex(prevRandao),
787
786
  suggestedFeeRecipient,
@@ -25,7 +25,6 @@ export enum RegenCaller {
25
25
  publishDeferredVoluntaryExits = "publishDeferredVoluntaryExits",
26
26
  validateGossipExecutionPayloadBid = "validateGossipExecutionPayloadBid",
27
27
  validateGossipInclusionList = "validateGossipInclusionList",
28
- validateApiExecutionPayloadBid = "validateApiExecutionPayloadBid",
29
28
  validateGossipPayloadAttestationMessage = "validateGossipPayloadAttestationMessage",
30
29
  validateGossipProposerPreferences = "validateGossipProposerPreferences",
31
30
  onForkChoiceFinalized = "onForkChoiceFinalized",