@lodestar/beacon-node 1.43.0-dev.ade910fc78 → 1.43.0-dev.ca1fc40294

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +1 -4
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/beacon/state/utils.d.ts +2 -2
  5. package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
  6. package/lib/api/impl/beacon/state/utils.js.map +1 -1
  7. package/lib/api/impl/validator/index.d.ts.map +1 -1
  8. package/lib/api/impl/validator/index.js +1 -4
  9. package/lib/api/impl/validator/index.js.map +1 -1
  10. package/lib/chain/GetBlobsTracker.d.ts +1 -1
  11. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  12. package/lib/chain/GetBlobsTracker.js +1 -2
  13. package/lib/chain/GetBlobsTracker.js.map +1 -1
  14. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  15. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  16. package/lib/chain/archiveStore/interface.d.ts +4 -4
  17. package/lib/chain/archiveStore/interface.d.ts.map +1 -1
  18. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts +4 -4
  19. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts.map +1 -1
  20. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js +2 -4
  21. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js.map +1 -1
  22. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +2 -2
  23. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  24. package/lib/chain/archiveStore/utils/archiveBlocks.js +110 -58
  25. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  26. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  27. package/lib/chain/blocks/importBlock.js +23 -31
  28. package/lib/chain/blocks/importBlock.js.map +1 -1
  29. package/lib/chain/blocks/importExecutionPayload.d.ts +1 -1
  30. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  31. package/lib/chain/blocks/importExecutionPayload.js +19 -26
  32. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  33. package/lib/chain/blocks/index.js +1 -1
  34. package/lib/chain/blocks/index.js.map +1 -1
  35. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +3 -0
  36. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  37. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +20 -0
  38. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  39. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -0
  40. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts.map +1 -1
  41. package/lib/chain/blocks/payloadEnvelopeProcessor.js +6 -4
  42. package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -1
  43. package/lib/chain/blocks/types.d.ts +1 -1
  44. package/lib/chain/blocks/types.d.ts.map +1 -1
  45. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +14 -0
  46. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -0
  47. package/lib/chain/blocks/verifyPayloadsDataAvailability.js +25 -0
  48. package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -0
  49. package/lib/chain/chain.d.ts +3 -3
  50. package/lib/chain/chain.d.ts.map +1 -1
  51. package/lib/chain/chain.js +16 -32
  52. package/lib/chain/chain.js.map +1 -1
  53. package/lib/chain/emitter.d.ts +16 -4
  54. package/lib/chain/emitter.d.ts.map +1 -1
  55. package/lib/chain/emitter.js +5 -0
  56. package/lib/chain/emitter.js.map +1 -1
  57. package/lib/chain/errors/attestationError.d.ts +8 -1
  58. package/lib/chain/errors/attestationError.d.ts.map +1 -1
  59. package/lib/chain/errors/attestationError.js +4 -0
  60. package/lib/chain/errors/attestationError.js.map +1 -1
  61. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  62. package/lib/chain/forkChoice/index.js +9 -13
  63. package/lib/chain/forkChoice/index.js.map +1 -1
  64. package/lib/chain/interface.d.ts +2 -2
  65. package/lib/chain/interface.d.ts.map +1 -1
  66. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  67. package/lib/chain/prepareNextSlot.js +22 -16
  68. package/lib/chain/prepareNextSlot.js.map +1 -1
  69. package/lib/chain/produceBlock/computeNewStateRoot.d.ts +3 -9
  70. package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
  71. package/lib/chain/produceBlock/computeNewStateRoot.js +5 -32
  72. package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
  73. package/lib/chain/produceBlock/produceBlockBody.d.ts +3 -8
  74. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  75. package/lib/chain/produceBlock/produceBlockBody.js +29 -19
  76. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  77. package/lib/chain/regen/errors.d.ts +1 -11
  78. package/lib/chain/regen/errors.d.ts.map +1 -1
  79. package/lib/chain/regen/errors.js +0 -2
  80. package/lib/chain/regen/errors.js.map +1 -1
  81. package/lib/chain/regen/interface.d.ts +6 -11
  82. package/lib/chain/regen/interface.d.ts.map +1 -1
  83. package/lib/chain/regen/queued.d.ts +6 -10
  84. package/lib/chain/regen/queued.d.ts.map +1 -1
  85. package/lib/chain/regen/queued.js +3 -10
  86. package/lib/chain/regen/queued.js.map +1 -1
  87. package/lib/chain/regen/regen.d.ts +0 -5
  88. package/lib/chain/regen/regen.d.ts.map +1 -1
  89. package/lib/chain/regen/regen.js +0 -8
  90. package/lib/chain/regen/regen.js.map +1 -1
  91. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +1 -7
  92. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  93. package/lib/chain/stateCache/persistentCheckpointsCache.js +4 -9
  94. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  95. package/lib/chain/stateCache/types.d.ts +0 -6
  96. package/lib/chain/stateCache/types.d.ts.map +1 -1
  97. package/lib/chain/stateCache/types.js.map +1 -1
  98. package/lib/chain/validation/aggregateAndProof.js +12 -0
  99. package/lib/chain/validation/aggregateAndProof.js.map +1 -1
  100. package/lib/chain/validation/attestation.d.ts.map +1 -1
  101. package/lib/chain/validation/attestation.js +12 -0
  102. package/lib/chain/validation/attestation.js.map +1 -1
  103. package/lib/chain/validation/executionPayloadEnvelope.js +10 -10
  104. package/lib/chain/validation/executionPayloadEnvelope.js.map +1 -1
  105. package/lib/chain/validation/payloadAttestationMessage.d.ts.map +1 -1
  106. package/lib/chain/validation/payloadAttestationMessage.js +4 -3
  107. package/lib/chain/validation/payloadAttestationMessage.js.map +1 -1
  108. package/lib/db/repositories/executionPayloadEnvelopeArchive.js +1 -1
  109. package/lib/db/repositories/executionPayloadEnvelopeArchive.js.map +1 -1
  110. package/lib/execution/engine/http.d.ts.map +1 -1
  111. package/lib/execution/engine/http.js +21 -14
  112. package/lib/execution/engine/http.js.map +1 -1
  113. package/lib/execution/engine/interface.d.ts +1 -0
  114. package/lib/execution/engine/interface.d.ts.map +1 -1
  115. package/lib/execution/engine/mock.d.ts.map +1 -1
  116. package/lib/execution/engine/mock.js +6 -0
  117. package/lib/execution/engine/mock.js.map +1 -1
  118. package/lib/execution/engine/types.d.ts +20 -0
  119. package/lib/execution/engine/types.d.ts.map +1 -1
  120. package/lib/execution/engine/types.js +18 -0
  121. package/lib/execution/engine/types.js.map +1 -1
  122. package/lib/network/gossip/topic.d.ts +3 -2
  123. package/lib/network/gossip/topic.d.ts.map +1 -1
  124. package/lib/network/network.js +1 -1
  125. package/lib/network/network.js.map +1 -1
  126. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  127. package/lib/network/processor/gossipHandlers.js +22 -6
  128. package/lib/network/processor/gossipHandlers.js.map +1 -1
  129. package/lib/node/nodejs.d.ts.map +1 -1
  130. package/lib/node/nodejs.js +4 -2
  131. package/lib/node/nodejs.js.map +1 -1
  132. package/lib/util/sszBytes.d.ts.map +1 -1
  133. package/lib/util/sszBytes.js +16 -3
  134. package/lib/util/sszBytes.js.map +1 -1
  135. package/package.json +16 -16
  136. package/src/api/impl/beacon/blocks/index.ts +1 -4
  137. package/src/api/impl/beacon/state/utils.ts +2 -2
  138. package/src/api/impl/validator/index.ts +3 -6
  139. package/src/chain/GetBlobsTracker.ts +1 -2
  140. package/src/chain/archiveStore/archiveStore.ts +5 -5
  141. package/src/chain/archiveStore/interface.ts +4 -4
  142. package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +6 -8
  143. package/src/chain/archiveStore/utils/archiveBlocks.ts +153 -94
  144. package/src/chain/blocks/importBlock.ts +22 -35
  145. package/src/chain/blocks/importExecutionPayload.ts +20 -26
  146. package/src/chain/blocks/index.ts +1 -1
  147. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +27 -0
  148. package/src/chain/blocks/payloadEnvelopeProcessor.ts +6 -5
  149. package/src/chain/blocks/types.ts +1 -1
  150. package/src/chain/blocks/verifyPayloadsDataAvailability.ts +38 -0
  151. package/src/chain/chain.ts +23 -48
  152. package/src/chain/emitter.ts +15 -3
  153. package/src/chain/errors/attestationError.ts +6 -1
  154. package/src/chain/forkChoice/index.ts +6 -18
  155. package/src/chain/interface.ts +2 -2
  156. package/src/chain/prepareNextSlot.ts +25 -16
  157. package/src/chain/produceBlock/computeNewStateRoot.ts +6 -43
  158. package/src/chain/produceBlock/produceBlockBody.ts +40 -20
  159. package/src/chain/regen/errors.ts +1 -6
  160. package/src/chain/regen/interface.ts +6 -11
  161. package/src/chain/regen/queued.ts +6 -14
  162. package/src/chain/regen/regen.ts +0 -8
  163. package/src/chain/stateCache/persistentCheckpointsCache.ts +5 -15
  164. package/src/chain/stateCache/types.ts +0 -3
  165. package/src/chain/validation/aggregateAndProof.ts +13 -0
  166. package/src/chain/validation/attestation.ts +13 -0
  167. package/src/chain/validation/executionPayloadEnvelope.ts +10 -10
  168. package/src/chain/validation/payloadAttestationMessage.ts +5 -3
  169. package/src/db/repositories/executionPayloadEnvelopeArchive.ts +1 -1
  170. package/src/execution/engine/http.ts +21 -14
  171. package/src/execution/engine/interface.ts +1 -0
  172. package/src/execution/engine/mock.ts +8 -1
  173. package/src/execution/engine/types.ts +41 -0
  174. package/src/network/network.ts +1 -1
  175. package/src/network/processor/gossipHandlers.ts +26 -10
  176. package/src/node/nodejs.ts +4 -2
  177. package/src/util/sszBytes.ts +21 -3
@@ -1,7 +1,7 @@
1
1
  import path from "node:path";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
3
  import {KeyValue} from "@lodestar/db";
4
- import {CheckpointWithPayloadStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
4
+ import {CheckpointWithHex, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
5
5
  import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
6
6
  import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
7
7
  import {Epoch, Slot} from "@lodestar/types";
@@ -52,16 +52,17 @@ export async function archiveBlocks(
52
52
  forkChoice: IForkChoice,
53
53
  lightclientServer: LightClientServer | undefined,
54
54
  logger: Logger,
55
- finalizedCheckpoint: CheckpointWithPayloadStatus,
55
+ finalizedCheckpoint: CheckpointWithHex,
56
56
  currentEpoch: Epoch,
57
57
  archiveDataEpochs?: number,
58
58
  persistOrphanedBlocks?: boolean,
59
59
  persistOrphanedBlocksDir?: string
60
60
  ): Promise<void> {
61
- // Use fork choice to determine the blocks to archive and delete
62
- // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db
61
+ // Use fork choice to determine the blocks to archive and delete.
62
+ // `ancestors` is the canonical walk back from the finalized root, including the previous finalized
63
+ // block as its last element.
63
64
  const {ancestors: finalizedCanonicalBlocks, nonAncestors: finalizedNonCanonicalBlocks} =
64
- forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex, finalizedCheckpoint.payloadStatus);
65
+ forkChoice.getAllAncestorAndNonAncestorBlocksDefaultStatus(finalizedCheckpoint.rootHex);
65
66
 
66
67
  // NOTE: The finalized block will be exactly the first block of `epoch` or previous
67
68
  const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH;
@@ -76,18 +77,21 @@ export async function archiveBlocks(
76
77
  const logCtx = {currentEpoch, finalizedEpoch: finalizedCheckpoint.epoch, finalizedRoot: finalizedCheckpoint.rootHex};
77
78
 
78
79
  if (finalizedCanonicalBlockRoots.length > 0) {
79
- await migrateBlocksFromHotToColdDb(db, finalizedCanonicalBlockRoots);
80
+ const migratedSlots = await migrateBlocksFromHotToColdDb(db, logger, finalizedCanonicalBlockRoots);
80
81
  logger.verbose("Migrated blocks from hot DB to cold DB", {
81
82
  ...logCtx,
82
83
  fromSlot: finalizedCanonicalBlockRoots[0].slot,
83
84
  toSlot: finalizedCanonicalBlockRoots.at(-1)?.slot,
84
85
  size: finalizedCanonicalBlockRoots.length,
86
+ migratedEntries: migratedSlots.length,
87
+ slotRange: prettyPrintIndices(migratedSlots),
85
88
  });
86
89
 
87
90
  if (finalizedPostDeneb) {
88
91
  const migratedEntries = await migrateBlobSidecarsFromHotToColdDb(
89
92
  config,
90
93
  db,
94
+ logger,
91
95
  finalizedCanonicalBlockRoots,
92
96
  currentEpoch
93
97
  );
@@ -95,24 +99,32 @@ export async function archiveBlocks(
95
99
  }
96
100
 
97
101
  if (finalizedPostFulu) {
98
- const migratedEntries = await migrateDataColumnSidecarsFromHotToColdDb(
102
+ const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
99
103
  config,
100
104
  db,
101
105
  logger,
102
- finalizedCanonicalBlockRoots,
106
+ finalizedCanonicalBlocks,
103
107
  currentEpoch
104
108
  );
105
- logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
109
+ logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {
110
+ ...logCtx,
111
+ migratedEntries: migratedSlots.length,
112
+ slotRange: prettyPrintIndices(migratedSlots),
113
+ });
106
114
  }
107
115
 
108
116
  if (finalizedPostGloas) {
109
- const migratedEntries = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
117
+ const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
110
118
  config,
111
119
  db,
112
120
  logger,
113
121
  finalizedCanonicalBlocks
114
122
  );
115
- logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {...logCtx, migratedEntries});
123
+ logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {
124
+ ...logCtx,
125
+ migratedEntries: migratedSlots.length,
126
+ slotRange: prettyPrintIndices(migratedSlots),
127
+ });
116
128
  }
117
129
  }
118
130
 
@@ -140,25 +152,29 @@ export async function archiveBlocks(
140
152
  );
141
153
  }
142
154
 
143
- await db.block.batchDelete(nonCanonicalBlockRoots);
144
- logger.verbose("Deleted non canonical blocks from hot DB", {
155
+ const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
156
+ const nonCanonicalLogCtx = {
145
157
  ...logCtx,
146
- slots: finalizedNonCanonicalBlocks.map((summary) => summary.slot).join(","),
147
- });
158
+ count: nonCanonicalBlockRoots.length,
159
+ slotRange: prettyPrintIndices(nonCanonicalSlots),
160
+ };
161
+
162
+ await db.block.batchDelete(nonCanonicalBlockRoots);
163
+ logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
148
164
 
149
165
  if (finalizedPostDeneb) {
150
166
  await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
151
- logger.verbose("Deleted non canonical blobSidecars from hot DB", logCtx);
167
+ logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
152
168
  }
153
169
 
154
170
  if (finalizedPostFulu) {
155
171
  await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
156
- logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", logCtx);
172
+ logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", nonCanonicalLogCtx);
157
173
  }
158
174
 
159
175
  if (finalizedPostGloas) {
160
176
  await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
161
- logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", logCtx);
177
+ logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
162
178
  }
163
179
  }
164
180
 
@@ -239,42 +255,51 @@ export async function archiveBlocks(
239
255
  });
240
256
  }
241
257
 
242
- async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot[]): Promise<void> {
243
- // Start from `i=0`: 1st block in iterateAncestorBlocks() is the finalized block itself
244
- // we move it to blockArchive but forkchoice still have it to check next onBlock calls
245
- // the next iterateAncestorBlocks call does not return this block
258
+ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, logger: Logger, blocks: BlockRootSlot[]): Promise<Slot[]> {
259
+ // The input includes the previous finalized block as the last ancestor; its SignedBeaconBlock
260
+ // was archived on a previous run and is no longer in hot db. `getBinary` returning null for any
261
+ // block in the batch is therefore treated as "already migrated, skip" rather than an error.
262
+ const migratedSlots: Slot[] = [];
246
263
  for (let i = 0; i < blocks.length; i += BLOCK_BATCH_SIZE) {
247
264
  const toIdx = Math.min(i + BLOCK_BATCH_SIZE, blocks.length);
248
265
  const canonicalBlocks = blocks.slice(i, toIdx);
249
266
 
250
- // processCanonicalBlocks
251
- if (canonicalBlocks.length === 0) return;
267
+ if (canonicalBlocks.length === 0) break;
252
268
 
253
269
  // load Buffer instead of SignedBeaconBlock to improve performance
254
- const canonicalBlockEntries: BlockArchiveBatchPutBinaryItem[] = await Promise.all(
255
- canonicalBlocks.map(async (block) => {
256
- // Here we assume the blocks are already in the hot db
257
- const blockBuffer = await db.block.getBinary(block.root);
258
- if (!blockBuffer) {
259
- throw Error(`Block not found for slot ${block.slot} root ${toRootHex(block.root)}`);
260
- }
261
- return {
262
- key: block.slot,
263
- value: blockBuffer,
264
- slot: block.slot,
265
- blockRoot: block.root,
266
- // TODO: Benchmark if faster to slice Buffer or fromHex()
267
- parentRoot: getParentRootFromSignedBlock(blockBuffer),
268
- };
269
- })
270
- );
270
+ const canonicalBlockEntries = (
271
+ await Promise.all(
272
+ canonicalBlocks.map(async (block): Promise<BlockArchiveBatchPutBinaryItem | null> => {
273
+ const blockBuffer = await db.block.getBinary(block.root);
274
+ if (!blockBuffer) {
275
+ logger.debug("Block in forkchoice but missing in hot db, could be already archived", {
276
+ slot: block.slot,
277
+ root: toRootHex(block.root),
278
+ });
279
+ return null;
280
+ }
281
+ return {
282
+ key: block.slot,
283
+ value: blockBuffer,
284
+ slot: block.slot,
285
+ blockRoot: block.root,
286
+ // TODO: Benchmark if faster to slice Buffer or fromHex()
287
+ parentRoot: getParentRootFromSignedBlock(blockBuffer),
288
+ };
289
+ })
290
+ )
291
+ ).filter((entry): entry is BlockArchiveBatchPutBinaryItem => entry !== null);
292
+
293
+ if (canonicalBlockEntries.length === 0) continue;
271
294
 
272
- // put to blockArchive db and delete block db
273
295
  await Promise.all([
274
296
  db.blockArchive.batchPutBinary(canonicalBlockEntries),
275
- db.block.batchDelete(canonicalBlocks.map((block) => block.root)),
297
+ db.block.batchDelete(canonicalBlockEntries.map((entry) => entry.blockRoot)),
276
298
  ]);
299
+ for (const entry of canonicalBlockEntries) migratedSlots.push(entry.slot);
277
300
  }
301
+ // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
302
+ return migratedSlots.sort((a, b) => a - b);
278
303
  }
279
304
 
280
305
  /**
@@ -284,6 +309,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot
284
309
  async function migrateBlobSidecarsFromHotToColdDb(
285
310
  config: ChainForkConfig,
286
311
  db: IBeaconDb,
312
+ logger: Logger,
287
313
  blocks: BlockRootSlot[],
288
314
  currentEpoch: Epoch
289
315
  ): Promise<number> {
@@ -296,29 +322,36 @@ async function migrateBlobSidecarsFromHotToColdDb(
296
322
  if (canonicalBlocks.length === 0) break;
297
323
 
298
324
  // load Buffer instead of ssz deserialized to improve performance
299
- const canonicalBlobSidecarsEntries: KeyValue<Slot, Uint8Array>[] = await Promise.all(
300
- canonicalBlocks
301
- .filter((block) => {
302
- const blockSlot = block.slot;
303
- const blockEpoch = computeEpochAtSlot(blockSlot);
304
- const forkSeq = config.getForkSeq(blockSlot);
305
- return (
306
- forkSeq >= ForkSeq.deneb &&
307
- forkSeq < ForkSeq.fulu &&
308
- // if block is out of ${config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS}, skip this step
309
- blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
310
- );
311
- })
312
- .map(async (block) => {
313
- // Here we assume the blob sidecars are already in the hot db
314
- // instead of checking first the block input cache
315
- const bytes = await db.blobSidecars.getBinary(block.root);
316
- if (!bytes) {
317
- throw Error(`No blobSidecars found for slot ${block.slot} root ${toRootHex(block.root)}`);
318
- }
319
- return {key: block.slot, value: bytes};
320
- })
321
- );
325
+ const canonicalBlobSidecarsEntries: KeyValue<Slot, Uint8Array>[] = (
326
+ await Promise.all(
327
+ canonicalBlocks
328
+ .filter((block) => {
329
+ const blockSlot = block.slot;
330
+ const blockEpoch = computeEpochAtSlot(blockSlot);
331
+ const forkSeq = config.getForkSeq(blockSlot);
332
+ return (
333
+ forkSeq >= ForkSeq.deneb &&
334
+ forkSeq < ForkSeq.fulu &&
335
+ // if block is out of ${config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS}, skip this step
336
+ blockEpoch >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
337
+ );
338
+ })
339
+ .map(async (block): Promise<KeyValue<Slot, Uint8Array> | null> => {
340
+ // The ancestor walk includes the boundary (previous finalized) block; on first
341
+ // finalization that boundary is the anchor which has no blob sidecars in hot db.
342
+ // Treat a null hot-db entry as "nothing to migrate" rather than an error.
343
+ const bytes = await db.blobSidecars.getBinary(block.root);
344
+ if (!bytes) {
345
+ logger.debug("BlobSidecars in forkchoice but missing in hot db, could be already archived", {
346
+ slot: block.slot,
347
+ root: toRootHex(block.root),
348
+ });
349
+ return null;
350
+ }
351
+ return {key: block.slot, value: bytes};
352
+ })
353
+ )
354
+ ).filter((e): e is KeyValue<Slot, Uint8Array> => e !== null);
322
355
 
323
356
  // put to blockArchive db and delete block db
324
357
  await Promise.all([
@@ -332,24 +365,36 @@ async function migrateBlobSidecarsFromHotToColdDb(
332
365
  }
333
366
 
334
367
  // TODO: This function can be simplified further by reducing layers of promises in a loop
368
+ /**
369
+ * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope —
370
+ * columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas (Fulu)
371
+ * blocks only have a FULL variant, so the `payloadStatus === FULL` filter passes them all.
372
+ * Blocks whose canonical variant is PENDING/EMPTY are skipped here — their columns will be picked
373
+ * up on a later run once the FULL variant appears in the ancestor walk.
374
+ */
335
375
  async function migrateDataColumnSidecarsFromHotToColdDb(
336
376
  config: ChainForkConfig,
337
377
  db: IBeaconDb,
338
378
  logger: Logger,
339
- blocks: BlockRootSlot[],
379
+ canonicalBlocks: ProtoBlock[],
340
380
  currentEpoch: Epoch
341
- ): Promise<number> {
342
- let migratedWrappedDataColumns = 0;
381
+ ): Promise<Slot[]> {
382
+ const columnBlocks = canonicalBlocks.filter(
383
+ (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
384
+ );
385
+ if (columnBlocks.length === 0) return [];
386
+ const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
387
+
388
+ const migratedSlots: Slot[] = [];
343
389
  for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
344
390
  const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
345
- const canonicalBlocks = blocks.slice(i, toIdx);
391
+ const batch = blocks.slice(i, toIdx);
392
+ if (batch.length === 0) break;
346
393
 
347
- // processCanonicalBlocks
348
- if (canonicalBlocks.length === 0) break;
349
- const promises = [];
394
+ const promises: Promise<void>[] = [];
350
395
 
351
396
  // load Buffer instead of ssz deserialized to improve performance
352
- for (const block of canonicalBlocks) {
397
+ for (const block of batch) {
353
398
  const blockSlot = block.slot;
354
399
  const blockEpoch = computeEpochAtSlot(blockSlot);
355
400
 
@@ -363,8 +408,16 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
363
408
 
364
409
  // Here we assume the data column sidecars are already in the hot db
365
410
  const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
366
- // there could be 0 dataColumnSidecarBytes if block has no blob
367
- logger.verbose("migrateDataColumnSidecarsFromHotToColdDb", {
411
+ if (dataColumnSidecarBytes.length === 0) {
412
+ // Empty stream: either the block has no blobs, or columns were already archived on a
413
+ // previous run (boundary block). Nothing to migrate.
414
+ logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
415
+ slot: block.slot,
416
+ root: toRootHex(block.root),
417
+ });
418
+ continue;
419
+ }
420
+ logger.verbose("Migrated dataColumnSidecars for block", {
368
421
  currentEpoch,
369
422
  slot: block.slot,
370
423
  root: toRootHex(block.root),
@@ -376,30 +429,32 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
376
429
  dataColumnSidecarBytes.map((p) => ({key: p.id, value: p.value}))
377
430
  )
378
431
  );
379
- migratedWrappedDataColumns += dataColumnSidecarBytes.length;
432
+ migratedSlots.push(block.slot);
380
433
  }
381
434
 
382
- promises.push(db.dataColumnSidecar.deleteMany(canonicalBlocks.map((block) => block.root)));
435
+ promises.push(db.dataColumnSidecar.deleteMany(batch.map((block) => block.root)));
383
436
 
384
- // put to blockArchive db and delete block db
385
437
  await Promise.all(promises);
386
438
  }
387
439
 
388
- return migratedWrappedDataColumns;
440
+ // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
441
+ return migratedSlots.sort((a, b) => a - b);
389
442
  }
390
443
 
444
+ /**
445
+ * Post-gloas given a finalized checkpoint at a block root, payload of that block root
446
+ * is not considered finalized, hence they are archived in the next run.
447
+ */
391
448
  async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
392
449
  config: ChainForkConfig,
393
450
  db: IBeaconDb,
394
451
  logger: Logger,
395
452
  canonicalBlocks: ProtoBlock[]
396
- ): Promise<number> {
397
- let migratedEnvelopes = 0;
398
-
453
+ ): Promise<Slot[]> {
399
454
  const payloadBlocks = canonicalBlocks.filter(
400
- (block) => config.getForkSeq(block.slot) >= ForkSeq.gloas && block.payloadStatus === PayloadStatus.FULL
455
+ (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
401
456
  );
402
- if (payloadBlocks.length === 0) return 0;
457
+ if (payloadBlocks.length === 0) return [];
403
458
  const blocks = payloadBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
404
459
 
405
460
  const envelopeEntries: KeyValue<Slot, Uint8Array>[] = [];
@@ -415,19 +470,23 @@ async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
415
470
  envelopeEntries.push({key: blocks[i].slot, value: bytes});
416
471
  migratedRoots.push(blocks[i].root);
417
472
  } else {
418
- logger.debug("Payload in forkchoice but missing in db", {slot: blocks[i].slot, root: toRootHex(blocks[i].root)});
473
+ logger.debug("ExecutionPayloadEnvelope in forkchoice but missing in hot db, could be already archived", {
474
+ slot: blocks[i].slot,
475
+ root: toRootHex(blocks[i].root),
476
+ });
419
477
  }
420
478
  }
421
479
 
422
- if (envelopeEntries.length > 0) {
423
- await Promise.all([
424
- db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries),
425
- db.executionPayloadEnvelope.batchDelete(migratedRoots),
426
- ]);
427
- migratedEnvelopes = envelopeEntries.length;
428
- }
480
+ if (envelopeEntries.length === 0) return [];
481
+
482
+ await Promise.all([
483
+ db.executionPayloadEnvelopeArchive.batchPutBinary(envelopeEntries),
484
+ db.executionPayloadEnvelope.batchDelete(migratedRoots),
485
+ ]);
429
486
 
430
- return migratedEnvelopes;
487
+ // Slots are ascending in hot-db key order — sort to guarantee `prettyPrintIndices` output is clean
488
+ // regardless of ancestor-walk order (newest → oldest).
489
+ return envelopeEntries.map((entry) => entry.key).sort((a, b) => a - b);
431
490
  }
432
491
 
433
492
  /**
@@ -8,7 +8,6 @@ import {
8
8
  ForkChoiceErrorCode,
9
9
  NotReorgedReason,
10
10
  getSafeExecutionBlockHash,
11
- isGloasBlock,
12
11
  } from "@lodestar/fork-choice";
13
12
  import {
14
13
  ForkPostAltair,
@@ -87,7 +86,7 @@ export async function importBlock(
87
86
  fullyVerifiedBlock: FullyVerifiedBlock,
88
87
  opts: ImportBlockOpts
89
88
  ): Promise<void> {
90
- const {blockInput, postBlockState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
89
+ const {blockInput, postState, parentBlockSlot, executionStatus, dataAvailabilityStatus, indexedAttestations} =
91
90
  fullyVerifiedBlock;
92
91
  const block = blockInput.getBlock();
93
92
  const source = blockInput.getBlockSource();
@@ -99,7 +98,7 @@ export async function importBlock(
99
98
  const blockEpoch = computeEpochAtSlot(blockSlot);
100
99
  const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
101
100
  const blockDelaySec =
102
- fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postBlockState.genesisTime);
101
+ fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
103
102
  const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
104
103
  const fork = this.config.getForkSeq(blockSlot);
105
104
 
@@ -122,10 +121,10 @@ export async function importBlock(
122
121
  // 2. Import block to fork choice
123
122
 
124
123
  // Should compute checkpoint balances before forkchoice.onBlock
125
- this.checkpointBalancesCache.processState(blockRootHex, postBlockState);
124
+ this.checkpointBalancesCache.processState(blockRootHex, postState);
126
125
  const blockSummary = this.forkChoice.onBlock(
127
126
  block.message,
128
- postBlockState,
127
+ postState,
129
128
  blockDelaySec,
130
129
  currentSlot,
131
130
  fork >= ForkSeq.gloas ? ExecutionStatus.PayloadSeparated : executionStatus,
@@ -134,11 +133,7 @@ export async function importBlock(
134
133
 
135
134
  // This adds the state necessary to process the next block
136
135
  // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
137
- // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true
138
- // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
139
- const payloadPresent = !isGloasBlock(blockSummary);
140
- // processState manages both block state and payload state variants together for memory/disk management
141
- this.regen.processBlockState(blockRootHex, postBlockState);
136
+ this.regen.processState(blockRootHex, postState);
142
137
 
143
138
  // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import
144
139
  if (fork >= ForkSeq.gloas) {
@@ -161,17 +156,9 @@ export async function importBlock(
161
156
  // which is all the information we need so there is no reason to delay until execution payload arrives
162
157
  // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
163
158
  // (for example later in the slot). Do not couple retries to incoming gossip columns.
164
- this.getBlobsTracker.triggerGetBlobs(payloadInput, () => {
165
- // TODO GLOAS: come up with a better mechanism to trigger processExecutionPayload after data becomes available,
166
- // similar to how pre-gloas uses waitForBlockAndAllData with a cutoff timeout and incompleteBlockInput event
167
- this.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
168
- this.logger.debug(
169
- "Error processing execution payload after getBlobs",
170
- {slot: blockSlot, root: blockRootHex},
171
- e as Error
172
- );
173
- });
174
- });
159
+ // Columns fetched here feed payloadInput.addColumn, which resolves waitForAllData for any
160
+ // in-flight importExecutionPayload. No processExecutionPayload trigger needed from this path.
161
+ this.getBlobsTracker.triggerGetBlobs(payloadInput);
175
162
  }
176
163
 
177
164
  this.metrics?.importBlock.bySource.inc({source: source.source});
@@ -191,7 +178,7 @@ export async function importBlock(
191
178
  (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
192
179
  ) {
193
180
  const attestations = block.message.body.attestations;
194
- const rootCache = new RootCache(postBlockState);
181
+ const rootCache = new RootCache(postState);
195
182
  const invalidAttestationErrorsByCode = new Map<string, {error: Error; count: number}>();
196
183
 
197
184
  const addAttestation = fork >= ForkSeq.electra ? addAttestationPostElectra : addAttestationPreElectra;
@@ -205,7 +192,7 @@ export async function importBlock(
205
192
  const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data));
206
193
  addAttestation.call(
207
194
  this,
208
- postBlockState,
195
+ postState,
209
196
  target,
210
197
  attDataRoot,
211
198
  attestation as Attestation<ForkPostElectra>,
@@ -320,7 +307,7 @@ export async function importBlock(
320
307
 
321
308
  if (newHead.blockRoot !== oldHead.blockRoot) {
322
309
  // Set head state as strong reference
323
- this.regen.updateHeadState(newHead, postBlockState);
310
+ this.regen.updateHeadState(newHead, postState);
324
311
 
325
312
  try {
326
313
  this.emitter.emit(routes.events.EventType.head, {
@@ -390,10 +377,10 @@ export async function importBlock(
390
377
  // we want to import block asap so do this in the next event loop
391
378
  callInNextEventLoop(() => {
392
379
  try {
393
- if (isStatePostAltair(postBlockState)) {
380
+ if (isStatePostAltair(postState)) {
394
381
  this.lightClientServer?.onImportBlockHead(
395
382
  block.message as BeaconBlock<ForkPostAltair>,
396
- postBlockState,
383
+ postState,
397
384
  parentBlockSlot
398
385
  );
399
386
  }
@@ -415,11 +402,11 @@ export async function importBlock(
415
402
  // and the block is weak and can potentially be reorged out.
416
403
  let shouldOverrideFcu = false;
417
404
 
418
- if (blockSlot >= currentSlot && isStatePostBellatrix(postBlockState) && postBlockState.isExecutionStateType) {
405
+ if (blockSlot >= currentSlot && isStatePostBellatrix(postState) && postState.isExecutionStateType) {
419
406
  let notOverrideFcuReason = NotReorgedReason.Unknown;
420
407
  const proposalSlot = blockSlot + 1;
421
408
  try {
422
- const proposerIndex = postBlockState.getBeaconProposer(proposalSlot);
409
+ const proposerIndex = postState.getBeaconProposer(proposalSlot);
423
410
  const feeRecipient = this.beaconProposerCache.get(proposerIndex);
424
411
 
425
412
  if (feeRecipient) {
@@ -499,22 +486,22 @@ export async function importBlock(
499
486
  }
500
487
  }
501
488
 
502
- if (!postBlockState.isStateValidatorsNodesPopulated()) {
503
- this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postBlockState.slot});
489
+ if (!postState.isStateValidatorsNodesPopulated()) {
490
+ this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot});
504
491
  }
505
492
 
506
493
  // Cache shufflings when crossing an epoch boundary
507
494
  const parentEpoch = computeEpochAtSlot(parentBlockSlot);
508
495
  if (parentEpoch < blockEpoch) {
509
- this.shufflingCache.processState(postBlockState);
496
+ this.shufflingCache.processState(postState);
510
497
  this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
511
498
  }
512
499
 
513
500
  if (blockSlot % SLOTS_PER_EPOCH === 0) {
514
501
  // Cache state to preserve epoch transition work
515
- const checkpointState = postBlockState;
502
+ const checkpointState = postState;
516
503
  const cp = getCheckpointFromState(checkpointState);
517
- this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
504
+ this.regen.addCheckpointState(cp, checkpointState);
518
505
  // consumers should not mutate state ever
519
506
  this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
520
507
 
@@ -602,11 +589,11 @@ export async function importBlock(
602
589
  this.metrics?.parentBlockDistance.observe(blockSlot - parentBlockSlot);
603
590
  this.metrics?.proposerBalanceDeltaAny.observe(fullyVerifiedBlock.proposerBalanceDelta);
604
591
  this.validatorMonitor?.registerImportedBlock(block.message, fullyVerifiedBlock);
605
- if (isStatePostAltair(fullyVerifiedBlock.postBlockState)) {
592
+ if (isStatePostAltair(fullyVerifiedBlock.postState)) {
606
593
  this.validatorMonitor?.registerSyncAggregateInBlock(
607
594
  blockEpoch,
608
595
  (block as altair.SignedBeaconBlock).message.body.syncAggregate,
609
- fullyVerifiedBlock.postBlockState.currentSyncCommitteeIndexed.validatorIndices
596
+ fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
610
597
  );
611
598
  }
612
599