@lodestar/beacon-node 1.51.0-dev.83da04fef3 → 1.51.0-dev.99bbd7b976

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/lib/api/impl/api.js +1 -1
  2. package/lib/api/impl/api.js.map +1 -1
  3. package/lib/api/impl/beacon/blocks/index.d.ts +2 -1
  4. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/blocks/index.js +12 -7
  6. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  7. package/lib/api/impl/beacon/index.d.ts +2 -1
  8. package/lib/api/impl/beacon/index.d.ts.map +1 -1
  9. package/lib/api/impl/beacon/index.js +2 -2
  10. package/lib/api/impl/beacon/index.js.map +1 -1
  11. package/lib/api/impl/beacon/state/index.d.ts.map +1 -1
  12. package/lib/api/impl/beacon/state/index.js +8 -2
  13. package/lib/api/impl/beacon/state/index.js.map +1 -1
  14. package/lib/api/impl/beacon/state/utils.d.ts +2 -1
  15. package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
  16. package/lib/api/impl/beacon/state/utils.js +17 -9
  17. package/lib/api/impl/beacon/state/utils.js.map +1 -1
  18. package/lib/api/impl/debug/index.d.ts.map +1 -1
  19. package/lib/api/impl/debug/index.js +3 -0
  20. package/lib/api/impl/debug/index.js.map +1 -1
  21. package/lib/api/impl/validator/index.d.ts.map +1 -1
  22. package/lib/api/impl/validator/index.js +102 -10
  23. package/lib/api/impl/validator/index.js.map +1 -1
  24. package/lib/api/options.d.ts +2 -0
  25. package/lib/api/options.d.ts.map +1 -1
  26. package/lib/api/options.js +1 -0
  27. package/lib/api/options.js.map +1 -1
  28. package/lib/chain/ColumnReconstructionTracker.d.ts +11 -11
  29. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  30. package/lib/chain/ColumnReconstructionTracker.js +46 -40
  31. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  32. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  33. package/lib/chain/archiveStore/utils/archiveBlocks.js +54 -96
  34. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  35. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  36. package/lib/chain/blocks/importBlock.js +3 -2
  37. package/lib/chain/blocks/importBlock.js.map +1 -1
  38. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  39. package/lib/chain/blocks/importExecutionPayload.js +50 -2
  40. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  41. package/lib/chain/blocks/types.d.ts +4 -0
  42. package/lib/chain/blocks/types.d.ts.map +1 -1
  43. package/lib/chain/blocks/types.js +5 -0
  44. package/lib/chain/blocks/types.js.map +1 -1
  45. package/lib/chain/chain.d.ts +2 -1
  46. package/lib/chain/chain.d.ts.map +1 -1
  47. package/lib/chain/chain.js +5 -1
  48. package/lib/chain/chain.js.map +1 -1
  49. package/lib/chain/errors/executionPayloadBid.d.ts +11 -1
  50. package/lib/chain/errors/executionPayloadBid.d.ts.map +1 -1
  51. package/lib/chain/errors/executionPayloadBid.js +2 -0
  52. package/lib/chain/errors/executionPayloadBid.js.map +1 -1
  53. package/lib/chain/errors/inclusionList.d.ts +60 -0
  54. package/lib/chain/errors/inclusionList.d.ts.map +1 -0
  55. package/lib/chain/errors/inclusionList.js +19 -0
  56. package/lib/chain/errors/inclusionList.js.map +1 -0
  57. package/lib/chain/errors/index.d.ts +1 -0
  58. package/lib/chain/errors/index.d.ts.map +1 -1
  59. package/lib/chain/errors/index.js +1 -0
  60. package/lib/chain/errors/index.js.map +1 -1
  61. package/lib/chain/interface.d.ts +2 -1
  62. package/lib/chain/interface.d.ts.map +1 -1
  63. package/lib/chain/interface.js.map +1 -1
  64. package/lib/chain/opPools/inclusionListStore.d.ts +63 -0
  65. package/lib/chain/opPools/inclusionListStore.d.ts.map +1 -0
  66. package/lib/chain/opPools/inclusionListStore.js +171 -0
  67. package/lib/chain/opPools/inclusionListStore.js.map +1 -0
  68. package/lib/chain/opPools/index.d.ts +1 -0
  69. package/lib/chain/opPools/index.d.ts.map +1 -1
  70. package/lib/chain/opPools/index.js +1 -0
  71. package/lib/chain/opPools/index.js.map +1 -1
  72. package/lib/chain/produceBlock/produceBlockBody.d.ts +3 -1
  73. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  74. package/lib/chain/produceBlock/produceBlockBody.js +30 -10
  75. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  76. package/lib/chain/regen/interface.d.ts +1 -1
  77. package/lib/chain/regen/interface.d.ts.map +1 -1
  78. package/lib/chain/regen/interface.js +1 -1
  79. package/lib/chain/regen/interface.js.map +1 -1
  80. package/lib/chain/validation/executionPayloadBid.d.ts +0 -15
  81. package/lib/chain/validation/executionPayloadBid.d.ts.map +1 -1
  82. package/lib/chain/validation/executionPayloadBid.js +24 -145
  83. package/lib/chain/validation/executionPayloadBid.js.map +1 -1
  84. package/lib/chain/validation/inclusionList.d.ts +14 -0
  85. package/lib/chain/validation/inclusionList.d.ts.map +1 -0
  86. package/lib/chain/validation/inclusionList.js +150 -0
  87. package/lib/chain/validation/inclusionList.js.map +1 -0
  88. package/lib/chain/validation/index.d.ts +1 -0
  89. package/lib/chain/validation/index.d.ts.map +1 -1
  90. package/lib/chain/validation/index.js +1 -0
  91. package/lib/chain/validation/index.js.map +1 -1
  92. package/lib/db/beacon.d.ts.map +1 -1
  93. package/lib/db/beacon.js +2 -0
  94. package/lib/db/beacon.js.map +1 -1
  95. package/lib/execution/builder/validateBid.d.ts.map +1 -1
  96. package/lib/execution/builder/validateBid.js +14 -1
  97. package/lib/execution/builder/validateBid.js.map +1 -1
  98. package/lib/execution/engine/blobVersionedHashes.d.ts +1 -1
  99. package/lib/execution/engine/blobVersionedHashes.d.ts.map +1 -1
  100. package/lib/execution/engine/blobVersionedHashes.js.map +1 -1
  101. package/lib/execution/engine/disabled.d.ts +1 -0
  102. package/lib/execution/engine/disabled.d.ts.map +1 -1
  103. package/lib/execution/engine/disabled.js +3 -0
  104. package/lib/execution/engine/disabled.js.map +1 -1
  105. package/lib/execution/engine/http.d.ts +8 -2
  106. package/lib/execution/engine/http.d.ts.map +1 -1
  107. package/lib/execution/engine/http.js +27 -4
  108. package/lib/execution/engine/http.js.map +1 -1
  109. package/lib/execution/engine/interface.d.ts +11 -2
  110. package/lib/execution/engine/interface.d.ts.map +1 -1
  111. package/lib/execution/engine/interface.js.map +1 -1
  112. package/lib/execution/engine/jsonRpcTransport.d.ts +3 -2
  113. package/lib/execution/engine/jsonRpcTransport.d.ts.map +1 -1
  114. package/lib/execution/engine/jsonRpcTransport.js +33 -9
  115. package/lib/execution/engine/jsonRpcTransport.js.map +1 -1
  116. package/lib/execution/engine/mock.d.ts +1 -0
  117. package/lib/execution/engine/mock.d.ts.map +1 -1
  118. package/lib/execution/engine/mock.js +31 -7
  119. package/lib/execution/engine/mock.js.map +1 -1
  120. package/lib/execution/engine/payloadIdCache.d.ts +5 -3
  121. package/lib/execution/engine/payloadIdCache.d.ts.map +1 -1
  122. package/lib/execution/engine/payloadIdCache.js +15 -4
  123. package/lib/execution/engine/payloadIdCache.js.map +1 -1
  124. package/lib/execution/engine/restTransport.d.ts +2 -1
  125. package/lib/execution/engine/restTransport.d.ts.map +1 -1
  126. package/lib/execution/engine/restTransport.js +3 -0
  127. package/lib/execution/engine/restTransport.js.map +1 -1
  128. package/lib/execution/engine/sszTypes.d.ts +38 -38
  129. package/lib/execution/engine/transport.d.ts +5 -2
  130. package/lib/execution/engine/transport.d.ts.map +1 -1
  131. package/lib/execution/engine/types.d.ts +25 -1
  132. package/lib/execution/engine/types.d.ts.map +1 -1
  133. package/lib/execution/engine/types.js +17 -4
  134. package/lib/execution/engine/types.js.map +1 -1
  135. package/lib/index.d.ts +1 -0
  136. package/lib/index.d.ts.map +1 -1
  137. package/lib/index.js +2 -0
  138. package/lib/index.js.map +1 -1
  139. package/lib/metrics/metrics/beacon.d.ts +16 -0
  140. package/lib/metrics/metrics/beacon.d.ts.map +1 -1
  141. package/lib/metrics/metrics/beacon.js +35 -0
  142. package/lib/metrics/metrics/beacon.js.map +1 -1
  143. package/lib/metrics/metrics/lodestar.d.ts +19 -0
  144. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  145. package/lib/metrics/metrics/lodestar.js +49 -0
  146. package/lib/metrics/metrics/lodestar.js.map +1 -1
  147. package/lib/network/gossip/interface.d.ts +8 -2
  148. package/lib/network/gossip/interface.d.ts.map +1 -1
  149. package/lib/network/gossip/interface.js +1 -0
  150. package/lib/network/gossip/interface.js.map +1 -1
  151. package/lib/network/gossip/topic.d.ts +13 -5
  152. package/lib/network/gossip/topic.d.ts.map +1 -1
  153. package/lib/network/gossip/topic.js +13 -1
  154. package/lib/network/gossip/topic.js.map +1 -1
  155. package/lib/network/interface.d.ts +2 -1
  156. package/lib/network/interface.d.ts.map +1 -1
  157. package/lib/network/network.d.ts +2 -1
  158. package/lib/network/network.d.ts.map +1 -1
  159. package/lib/network/network.js +5 -0
  160. package/lib/network/network.js.map +1 -1
  161. package/lib/network/peers/utils/enrSubnetsDeserialize.d.ts.map +1 -1
  162. package/lib/network/peers/utils/enrSubnetsDeserialize.js +5 -2
  163. package/lib/network/peers/utils/enrSubnetsDeserialize.js.map +1 -1
  164. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  165. package/lib/network/processor/gossipHandlers.js +67 -19
  166. package/lib/network/processor/gossipHandlers.js.map +1 -1
  167. package/lib/network/processor/gossipQueues/index.d.ts.map +1 -1
  168. package/lib/network/processor/gossipQueues/index.js +6 -0
  169. package/lib/network/processor/gossipQueues/index.js.map +1 -1
  170. package/lib/network/processor/gossipValidatorFn.d.ts.map +1 -1
  171. package/lib/network/processor/gossipValidatorFn.js +5 -1
  172. package/lib/network/processor/gossipValidatorFn.js.map +1 -1
  173. package/lib/network/processor/index.d.ts +5 -0
  174. package/lib/network/processor/index.d.ts.map +1 -1
  175. package/lib/network/processor/index.js +37 -26
  176. package/lib/network/processor/index.js.map +1 -1
  177. package/lib/network/reqresp/ReqRespBeaconNode.d.ts.map +1 -1
  178. package/lib/network/reqresp/ReqRespBeaconNode.js +6 -0
  179. package/lib/network/reqresp/ReqRespBeaconNode.js.map +1 -1
  180. package/lib/network/reqresp/handlers/inclusionListsByIndices.d.ts +10 -0
  181. package/lib/network/reqresp/handlers/inclusionListsByIndices.d.ts.map +1 -0
  182. package/lib/network/reqresp/handlers/inclusionListsByIndices.js +30 -0
  183. package/lib/network/reqresp/handlers/inclusionListsByIndices.js.map +1 -0
  184. package/lib/network/reqresp/handlers/index.d.ts.map +1 -1
  185. package/lib/network/reqresp/handlers/index.js +5 -0
  186. package/lib/network/reqresp/handlers/index.js.map +1 -1
  187. package/lib/network/reqresp/protocols.d.ts +1 -0
  188. package/lib/network/reqresp/protocols.d.ts.map +1 -1
  189. package/lib/network/reqresp/protocols.js +5 -0
  190. package/lib/network/reqresp/protocols.js.map +1 -1
  191. package/lib/network/reqresp/rateLimit.d.ts.map +1 -1
  192. package/lib/network/reqresp/rateLimit.js +4 -0
  193. package/lib/network/reqresp/rateLimit.js.map +1 -1
  194. package/lib/network/reqresp/types.d.ts +5 -2
  195. package/lib/network/reqresp/types.d.ts.map +1 -1
  196. package/lib/network/reqresp/types.js +3 -0
  197. package/lib/network/reqresp/types.js.map +1 -1
  198. package/lib/sync/unknownBlock.d.ts.map +1 -1
  199. package/lib/sync/unknownBlock.js +42 -18
  200. package/lib/sync/unknownBlock.js.map +1 -1
  201. package/package.json +30 -30
  202. package/src/api/impl/api.ts +1 -1
  203. package/src/api/impl/beacon/blocks/index.ts +22 -17
  204. package/src/api/impl/beacon/index.ts +3 -1
  205. package/src/api/impl/beacon/state/index.ts +10 -1
  206. package/src/api/impl/beacon/state/utils.ts +19 -9
  207. package/src/api/impl/debug/index.ts +4 -0
  208. package/src/api/impl/validator/index.ts +132 -10
  209. package/src/api/options.ts +3 -0
  210. package/src/chain/ColumnReconstructionTracker.ts +60 -41
  211. package/src/chain/archiveStore/utils/archiveBlocks.ts +64 -126
  212. package/src/chain/blocks/importBlock.ts +3 -7
  213. package/src/chain/blocks/importExecutionPayload.ts +68 -2
  214. package/src/chain/blocks/types.ts +5 -0
  215. package/src/chain/chain.ts +5 -0
  216. package/src/chain/errors/executionPayloadBid.ts +12 -0
  217. package/src/chain/errors/inclusionList.ts +38 -0
  218. package/src/chain/errors/index.ts +1 -0
  219. package/src/chain/interface.ts +2 -0
  220. package/src/chain/opPools/inclusionListStore.ts +208 -0
  221. package/src/chain/opPools/index.ts +1 -0
  222. package/src/chain/produceBlock/produceBlockBody.ts +60 -19
  223. package/src/chain/regen/interface.ts +1 -1
  224. package/src/chain/validation/executionPayloadBid.ts +40 -170
  225. package/src/chain/validation/inclusionList.ts +192 -0
  226. package/src/chain/validation/index.ts +1 -0
  227. package/src/db/beacon.ts +5 -0
  228. package/src/execution/builder/validateBid.ts +23 -2
  229. package/src/execution/engine/blobVersionedHashes.ts +1 -1
  230. package/src/execution/engine/disabled.ts +4 -0
  231. package/src/execution/engine/http.ts +40 -5
  232. package/src/execution/engine/interface.ts +27 -3
  233. package/src/execution/engine/jsonRpcTransport.ts +46 -9
  234. package/src/execution/engine/mock.ts +32 -7
  235. package/src/execution/engine/payloadIdCache.ts +24 -6
  236. package/src/execution/engine/restTransport.ts +15 -1
  237. package/src/execution/engine/transport.ts +7 -2
  238. package/src/execution/engine/types.ts +41 -6
  239. package/src/index.ts +2 -0
  240. package/src/metrics/metrics/beacon.ts +37 -0
  241. package/src/metrics/metrics/lodestar.ts +51 -0
  242. package/src/network/gossip/interface.ts +5 -0
  243. package/src/network/gossip/topic.ts +14 -0
  244. package/src/network/interface.ts +2 -0
  245. package/src/network/network.ts +11 -0
  246. package/src/network/peers/utils/enrSubnetsDeserialize.ts +5 -2
  247. package/src/network/processor/gossipHandlers.ts +87 -21
  248. package/src/network/processor/gossipQueues/index.ts +6 -0
  249. package/src/network/processor/gossipValidatorFn.ts +5 -0
  250. package/src/network/processor/index.ts +33 -24
  251. package/src/network/processor/types.ts +1 -1
  252. package/src/network/reqresp/ReqRespBeaconNode.ts +7 -0
  253. package/src/network/reqresp/handlers/inclusionListsByIndices.ts +44 -0
  254. package/src/network/reqresp/handlers/index.ts +5 -0
  255. package/src/network/reqresp/protocols.ts +6 -0
  256. package/src/network/reqresp/rateLimit.ts +4 -0
  257. package/src/network/reqresp/types.ts +6 -0
  258. package/src/sync/unknownBlock.ts +47 -28
@@ -1,11 +1,18 @@
1
1
  import path from "node:path";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
3
  import {KeyValue} from "@lodestar/db";
4
- import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
4
+ import {
5
+ CheckpointWithHex,
6
+ ExecutionStatus,
7
+ IForkChoice,
8
+ PayloadStatus,
9
+ ProtoBlock,
10
+ isGloasBlock,
11
+ } from "@lodestar/fork-choice";
5
12
  import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
6
13
  import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
7
14
  import {Epoch, Slot} from "@lodestar/types";
8
- import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
15
+ import {Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
9
16
  import {IBeaconDb} from "../../../db/index.js";
10
17
  import {BlockArchiveBatchPutBinaryItem, encodeArchivedHeaderEnvelope} from "../../../db/repositories/index.js";
11
18
  import {Metrics} from "../../../metrics/metrics.js";
@@ -136,23 +143,6 @@ export async function archiveBlocks(
136
143
  logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
137
144
  }
138
145
 
139
- // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
140
- // directly to flat files, so this only moves fallback data left by older versions.
141
- if (finalizedPostFulu) {
142
- const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
143
- config,
144
- db,
145
- logger,
146
- finalizedCanonicalBlocks,
147
- currentEpoch
148
- );
149
- logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
150
- ...logCtx,
151
- migratedEntries: migratedSlots.length,
152
- slotRange: prettyPrintIndices(migratedSlots),
153
- });
154
- }
155
-
156
146
  if (finalizedPostGloas) {
157
147
  const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(
158
148
  config,
@@ -169,44 +159,34 @@ export async function archiveBlocks(
169
159
  }
170
160
  }
171
161
 
172
- // deleteNonCanonicalBlocks
173
- // loop through forkchoice single time
162
+ const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
163
+ const orphanedBlocks: ProtoBlock[] = [];
164
+ const orphanedPayloads: ProtoBlock[] = [];
174
165
 
175
- const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
176
- if (nonCanonicalBlockRoots.length > 0) {
177
- if (persistOrphanedBlocks) {
178
- // Persist orphaned blocks to disk before deleting them from hot db
179
- await Promise.all(
180
- nonCanonicalBlockRoots.map(async (root, index) => {
181
- const block = finalizedNonCanonicalBlocks[index];
182
- const blockBytes = await db.block.getBinary(root);
183
- const blockLogCtx = {slot: block.slot, root: block.blockRoot};
184
- if (blockBytes) {
185
- await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
186
- persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
187
- });
188
- logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
189
- } else {
190
- logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
191
- }
192
- })
193
- );
166
+ for (const block of finalizedNonCanonicalBlocks) {
167
+ if (block.payloadStatus === PayloadStatus.FULL) {
168
+ orphanedPayloads.push(block);
194
169
  }
170
+ // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
171
+ // already handled the root's block-level data.
172
+ const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
173
+ if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
174
+ orphanedBlocks.push(block);
175
+ }
176
+ }
177
+ const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
195
178
 
196
- const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
197
- const nonCanonicalLogCtx = {
179
+ if (orphanedPayloads.length > 0) {
180
+ const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
181
+ const orphanedPayloadLogCtx = {
198
182
  ...logCtx,
199
- count: nonCanonicalBlockRoots.length,
200
- slotRange: prettyPrintIndices(nonCanonicalSlots),
183
+ count: orphanedPayloads.length,
184
+ slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
201
185
  };
202
186
 
203
- const columnItems = finalizedNonCanonicalBlocks
204
- // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
205
- // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
206
- .filter(
207
- (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
208
- )
209
- .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
187
+ const columnItems = orphanedPayloads
188
+ .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
189
+ .map((block) => ({slot: block.slot, blockRoot: block.blockRoot}));
210
190
  if (columnItems.length > 0) {
211
191
  // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
212
192
  await db.dataColumns.deleteMany(columnItems);
@@ -217,18 +197,42 @@ export async function archiveBlocks(
217
197
  });
218
198
  }
219
199
 
220
- await db.block.batchDelete(nonCanonicalBlockRoots);
221
- logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
222
-
223
200
  if (finalizedPostDeneb) {
224
- await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
225
- logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
201
+ await db.blobSidecars.batchDelete(orphanedPayloadRoots);
202
+ logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
226
203
  }
227
204
 
228
205
  if (finalizedPostGloas) {
229
- await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
230
- logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
206
+ await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
207
+ logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
208
+ }
209
+ }
210
+
211
+ if (orphanedBlocks.length > 0) {
212
+ if (persistOrphanedBlocks) {
213
+ // Persist orphaned blocks to disk before deleting them from hot db
214
+ await Promise.all(
215
+ orphanedBlocks.map(async (block) => {
216
+ const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
217
+ const blockLogCtx = {slot: block.slot, root: block.blockRoot};
218
+ if (blockBytes) {
219
+ await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
220
+ persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
221
+ });
222
+ logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
223
+ } else {
224
+ logger.debug("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
225
+ }
226
+ })
227
+ );
231
228
  }
229
+
230
+ await db.block.batchDelete(orphanedBlockRoots);
231
+ logger.verbose("Deleted non canonical blocks from hot DB", {
232
+ ...logCtx,
233
+ count: orphanedBlocks.length,
234
+ slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
235
+ });
232
236
  }
233
237
 
234
238
  // Delete expired blobs
@@ -283,9 +287,9 @@ export async function archiveBlocks(
283
287
  }
284
288
  }
285
289
 
286
- // Prunning potential checkpoint data
290
+ // Pruning potential checkpoint data
287
291
  const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
288
- const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots];
292
+ const nonCheckpointBlockRoots: Uint8Array[] = [...orphanedBlockRoots];
289
293
  for (const block of finalizedCanonicalNonCheckpointBlocks) {
290
294
  nonCheckpointBlockRoots.push(block.root);
291
295
  }
@@ -408,72 +412,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
408
412
  return migratedWrappedBlobSidecars;
409
413
  }
410
414
 
411
- // TODO: This function can be simplified further by reducing layers of promises in a loop
412
- /**
413
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
414
- * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
415
- * blocks only have a FULL variant, so the payload-status filter passes them all.
416
- */
417
- async function migrateDataColumnSidecarsFromHotToColdDb(
418
- config: ChainForkConfig,
419
- db: IBeaconDb,
420
- logger: Logger,
421
- canonicalBlocks: ProtoBlock[],
422
- currentEpoch: Epoch
423
- ): Promise<Slot[]> {
424
- const columnBlocks = canonicalBlocks.filter(
425
- (block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL
426
- );
427
- if (columnBlocks.length === 0) return [];
428
- const blocks: BlockRootSlot[] = columnBlocks.map((block) => ({slot: block.slot, root: fromHex(block.blockRoot)}));
429
-
430
- const migratedSlots: Slot[] = [];
431
- for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
432
- const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
433
- const batch = blocks.slice(i, toIdx);
434
- if (batch.length === 0) break;
435
-
436
- const promises: Promise<void>[] = [];
437
- for (const block of batch) {
438
- const blockEpoch = computeEpochAtSlot(block.slot);
439
- if (
440
- config.getForkSeq(block.slot) < ForkSeq.fulu ||
441
- blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)
442
- ) {
443
- continue;
444
- }
445
-
446
- const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
447
- if (dataColumnSidecarBytes.length === 0) {
448
- logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
449
- slot: block.slot,
450
- root: toRootHex(block.root),
451
- });
452
- continue;
453
- }
454
- logger.verbose("Migrated legacy dataColumnSidecars for block", {
455
- currentEpoch,
456
- slot: block.slot,
457
- root: toRootHex(block.root),
458
- numSidecars: dataColumnSidecarBytes.length,
459
- });
460
- promises.push(
461
- db.dataColumnSidecarArchive.putManyBinary(
462
- block.slot,
463
- dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
464
- )
465
- );
466
- migratedSlots.push(block.slot);
467
- }
468
-
469
- await Promise.all(promises);
470
- await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
471
- }
472
-
473
- // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
474
- return migratedSlots.sort((a, b) => a - b);
475
- }
476
-
477
415
  /**
478
416
  * Post-gloas given a finalized checkpoint at a block root, payload of that block root
479
417
  * is not considered finalized, hence they are archived in the next run.
@@ -458,7 +458,8 @@ export async function importBlock(
458
458
  * - `headBlockHash !== null` -> Pre BELLATRIX_EPOCH
459
459
  * - `headBlockHash !== ZERO_HASH` -> Pre TTD
460
460
  */
461
- const headBlockHash = this.forkChoice.getHead().executionPayloadBlockHash ?? ZERO_HASH_HEX;
461
+ const fcuHead = this.forkChoice.getHead();
462
+ const headBlockHash = fcuHead.executionPayloadBlockHash ?? ZERO_HASH_HEX;
462
463
  /**
463
464
  * After BELLATRIX_EPOCH and TTD it's okay to send a zero hash block hash for the finalized block. This will happen if
464
465
  * the current finalized block does not contain any execution payload at all (pre MERGE_EPOCH) or if it contains a
@@ -468,12 +469,7 @@ export async function importBlock(
468
469
  const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
469
470
  if (headBlockHash !== ZERO_HASH_HEX) {
470
471
  this.executionEngine
471
- .notifyForkchoiceUpdate(
472
- this.config.getForkName(this.forkChoice.getHead().slot),
473
- headBlockHash,
474
- safeBlockHash,
475
- finalizedBlockHash
476
- )
472
+ .notifyForkchoiceUpdate(this.config.getForkName(fcuHead.slot), headBlockHash, safeBlockHash, finalizedBlockHash)
477
473
  .catch((e) => {
478
474
  if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
479
475
  this.logger.error("Error pushing notifyForkchoiceUpdate()", {headBlockHash, finalizedBlockHash}, e);
@@ -1,3 +1,4 @@
1
+ import {SerializedList} from "@chainsafe/ssz";
1
2
  import {routes} from "@lodestar/api";
2
3
  import {
3
4
  ExecutionStatus,
@@ -5,9 +6,16 @@ import {
5
6
  getFinalizedExecutionBlockHash,
6
7
  getSafeExecutionBlockHash,
7
8
  } from "@lodestar/fork-choice";
8
- import {DataAvailabilityStatus, isStatePostGloas} from "@lodestar/state-transition";
9
+ import {
10
+ DataAvailabilityStatus,
11
+ computeEpochAtSlot,
12
+ isStatePostGloas,
13
+ isStatePostHeze,
14
+ } from "@lodestar/state-transition";
15
+ import {RootHex, bellatrix} from "@lodestar/types";
9
16
  import {isErrorAborted} from "@lodestar/utils";
10
17
  import {ExecutionPayloadStatus} from "../../execution/index.js";
18
+ import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
11
19
  import {isQueueErrorAborted} from "../../util/queue/index.js";
12
20
  import {BeaconChain} from "../chain.js";
13
21
  import {RegenCaller} from "../regen/interface.js";
@@ -172,6 +180,37 @@ export async function importExecutionPayload(
172
180
  );
173
181
  }
174
182
 
183
+ // Timely inclusion lists of the preceding slot, keyed by that slot's dependent root on the block's branch and
184
+ // resolved from the parent, which is at or before that slot. Gossip validation only accepts lists whose dependent
185
+ // block is in fork choice, so a dependent root fork choice cannot resolve (below the anchor after checkpoint sync)
186
+ // keys no stored list
187
+ let inclusionListTransactions: bellatrix.Transactions | undefined;
188
+ if (isStatePostHeze(blockState)) {
189
+ const inclusionListSlot = protoBlock.slot - 1;
190
+ let dependentRoot: RootHex | null = null;
191
+ const parentBlock = this.forkChoice.getBlockHexDefaultStatus(protoBlock.parentRoot);
192
+ if (parentBlock !== null) {
193
+ try {
194
+ dependentRoot = getShufflingDependentRoot(
195
+ this.forkChoice,
196
+ computeEpochAtSlot(inclusionListSlot),
197
+ computeEpochAtSlot(parentBlock.slot),
198
+ parentBlock
199
+ );
200
+ } catch (e) {
201
+ this.logger.debug(
202
+ "Inclusion list dependent root not in fork choice, importing payload without lists",
203
+ {slot, root: blockRootHex},
204
+ e as Error
205
+ );
206
+ }
207
+ }
208
+ inclusionListTransactions =
209
+ dependentRoot === null
210
+ ? SerializedList.empty()
211
+ : this.inclusionListStore.getInclusionListTransactions(inclusionListSlot, dependentRoot, true);
212
+ }
213
+
175
214
  // 4a. Run EL and signature verification in parallel
176
215
  const logCtx = {slot, root: blockRootHex, executionBlock: envelope.payload.blockNumber};
177
216
  this.logger.debug("Call engine api newPayload", logCtx);
@@ -182,7 +221,8 @@ export async function importExecutionPayload(
182
221
  envelope.payload,
183
222
  payloadInput.getVersionedHashes(),
184
223
  envelope.parentBeaconBlockRoot,
185
- envelope.executionRequests
224
+ envelope.executionRequests,
225
+ inclusionListTransactions
186
226
  ),
187
227
 
188
228
  opts.validSignature === true
@@ -250,6 +290,32 @@ export async function importExecutionPayload(
250
290
  }
251
291
  });
252
292
 
293
+ // A VALID payload that does not satisfy the inclusion list constraints is still imported, fork choice
294
+ // only records the verdict so should_extend_payload moves the head to the EMPTY variant, which the next
295
+ // proposer builds on.
296
+ // Recorded before the payload enters fork choice. A VALID response must carry a verdict, an
297
+ // optimistically imported payload counts as satisfied until the engine validates it, anything else as
298
+ // unsatisfied. A VALID response without a verdict violates the engine API, it is logged and treated as
299
+ // satisfied so a non-compliant engine does not orphan every payload
300
+ if (inclusionListTransactions !== undefined) {
301
+ if (execResult.status === ExecutionPayloadStatus.VALID && execResult.inclusionListSatisfied == null) {
302
+ this.logger.warn("Execution engine reported no inclusion list verdict, not enforcing inclusion lists", logCtx);
303
+ }
304
+ const satisfied =
305
+ execResult.status === ExecutionPayloadStatus.VALID
306
+ ? (execResult.inclusionListSatisfied ?? true)
307
+ : execResult.status === ExecutionPayloadStatus.SYNCING || execResult.status === ExecutionPayloadStatus.ACCEPTED;
308
+ this.forkChoice.recordPayloadInclusionListSatisfaction(blockRootHex, satisfied);
309
+ if (!satisfied) {
310
+ this.logger.verbose("Payload did not satisfy inclusion list constraints", {
311
+ ...logCtx,
312
+ blockHash: blockHashHex,
313
+ status: execResult.status,
314
+ inclusionListTransactions: inclusionListTransactions.length,
315
+ });
316
+ }
317
+ }
318
+
253
319
  // 6. Update fork choice, transitions the block's PENDING variant to FULL
254
320
  const execStatus = toForkChoiceExecutionStatus(execResult.status);
255
321
  this.forkChoice.onExecutionPayload(
@@ -12,6 +12,11 @@ export enum GossipedInputType {
12
12
  dataColumn = "data_column",
13
13
  }
14
14
 
15
+ export enum InclusionListSource {
16
+ gossip = "gossip",
17
+ api = "api",
18
+ }
19
+
15
20
  type DataColumnData = {
16
21
  dataColumn: fulu.DataColumnSidecar;
17
22
  dataColumnBytes: Uint8Array | null;
@@ -100,6 +100,7 @@ import {
100
100
  AttestationPool,
101
101
  DeferredVoluntaryExitPool,
102
102
  ExecutionPayloadBidPool,
103
+ InclusionListStore,
103
104
  OpPool,
104
105
  PayloadAttestationPool,
105
106
  ProposerPreferencesPool,
@@ -198,6 +199,7 @@ export class BeaconChain implements IBeaconChain {
198
199
  readonly executionPayloadBidPool: ExecutionPayloadBidPool;
199
200
  readonly payloadAttestationPool: PayloadAttestationPool;
200
201
  readonly proposerPreferencesPool = new ProposerPreferencesPool();
202
+ readonly inclusionListStore: InclusionListStore;
201
203
  readonly opPool: OpPool;
202
204
  readonly deferredVoluntaryExitPool: DeferredVoluntaryExitPool;
203
205
 
@@ -335,6 +337,7 @@ export class BeaconChain implements IBeaconChain {
335
337
  this.payloadAttestationPool = new PayloadAttestationPool(config, clock, metrics);
336
338
  this.opPool = new OpPool(config);
337
339
  this.deferredVoluntaryExitPool = new DeferredVoluntaryExitPool(logger);
340
+ this.inclusionListStore = new InclusionListStore(config);
338
341
 
339
342
  this.seenAggregatedAttestations = new SeenAggregatedAttestations(metrics);
340
343
  this.seenContributionAndProof = new SeenContributionAndProof(metrics);
@@ -509,6 +512,7 @@ export class BeaconChain implements IBeaconChain {
509
512
  emitter,
510
513
  metrics,
511
514
  config,
515
+ signal,
512
516
  });
513
517
 
514
518
  this.archiveStore = new ArchiveStore(
@@ -1831,6 +1835,7 @@ export class BeaconChain implements IBeaconChain {
1831
1835
  this.executionPayloadBidPool.prune(slot);
1832
1836
  this.seenExecutionPayloadBids.prune(slot);
1833
1837
  this.proposerPreferencesPool.prune(slot);
1838
+ this.inclusionListStore.prune(slot);
1834
1839
  this.seenAttestationDatas.onSlot(slot);
1835
1840
  this.reprocessController.onSlot(slot);
1836
1841
 
@@ -21,6 +21,8 @@ export enum ExecutionPayloadBidErrorCode {
21
21
  NO_MATCHING_PROPOSER_PREFERENCES = "EXECUTION_PAYLOAD_BID_ERROR_NO_MATCHING_PROPOSER_PREFERENCES",
22
22
  PROPOSER_PREFERENCES_FEE_RECIPIENT_MISMATCH = "EXECUTION_PAYLOAD_BID_ERROR_PROPOSER_PREFERENCES_FEE_RECIPIENT_MISMATCH",
23
23
  PROPOSER_PREFERENCES_GAS_LIMIT_MISMATCH = "EXECUTION_PAYLOAD_BID_ERROR_PROPOSER_PREFERENCES_GAS_LIMIT_MISMATCH",
24
+ INCLUSION_LIST_BITS_NOT_INCLUSIVE = "EXECUTION_PAYLOAD_BID_ERROR_INCLUSION_LIST_BITS_NOT_INCLUSIVE",
25
+ BEFORE_INCLUSION_LIST_DEADLINE = "EXECUTION_PAYLOAD_BID_ERROR_BEFORE_INCLUSION_LIST_DEADLINE",
24
26
  }
25
27
 
26
28
  export type ExecutionPayloadBidErrorType =
@@ -86,6 +88,16 @@ export type ExecutionPayloadBidErrorType =
86
88
  bidFeeRecipient: string;
87
89
  expectedFeeRecipient: string;
88
90
  }
91
+ | {
92
+ code: ExecutionPayloadBidErrorCode.INCLUSION_LIST_BITS_NOT_INCLUSIVE;
93
+ builderIndex: BuilderIndex;
94
+ slot: Slot;
95
+ }
96
+ | {
97
+ code: ExecutionPayloadBidErrorCode.BEFORE_INCLUSION_LIST_DEADLINE;
98
+ builderIndex: BuilderIndex;
99
+ slot: Slot;
100
+ }
89
101
  | {
90
102
  code: ExecutionPayloadBidErrorCode.PROPOSER_PREFERENCES_GAS_LIMIT_MISMATCH;
91
103
  builderIndex: BuilderIndex;
@@ -0,0 +1,38 @@
1
+ import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
2
+ import {GossipActionError} from "./gossipValidation.js";
3
+
4
+ export enum InclusionListErrorCode {
5
+ EMPTY_TRANSACTIONS = "INCLUSION_LIST_ERROR_EMPTY_TRANSACTIONS",
6
+ MAXIMUM_SIZE_EXCEEDED = "INCLUSION_LIST_ERROR_MAXIMUM_SIZE_EXCEEDED",
7
+ EMPTY_TRANSACTION = "INCLUSION_LIST_ERROR_EMPTY_TRANSACTION",
8
+ INVALID_SLOT = "INCLUSION_LIST_ERROR_INVALID_SLOT",
9
+ MORE_THAN_TWO = "INCLUSION_LIST_ERROR_MORE_THAN_TWO",
10
+ VALIDATOR_NOT_IN_COMMITTEE = "INCLUSION_LIST_ERROR_VALIDATOR_NOT_IN_COMMITTEE",
11
+ UNKNOWN_DEPENDENT_ROOT = "INCLUSION_LIST_ERROR_UNKNOWN_DEPENDENT_ROOT",
12
+ INVALID_DEPENDENT_ROOT_SLOT = "INCLUSION_LIST_ERROR_INVALID_DEPENDENT_ROOT_SLOT",
13
+ INVALID_DEPENDENT_ROOT = "INCLUSION_LIST_ERROR_INVALID_DEPENDENT_ROOT",
14
+ MISSING_SHUFFLING = "INCLUSION_LIST_ERROR_MISSING_SHUFFLING",
15
+ INVALID_SIGNATURE = "INCLUSION_LIST_ERROR_INVALID_SIGNATURE",
16
+ PRE_HEZE_SLOT = "INCLUSION_LIST_ERROR_PRE_HEZE_SLOT",
17
+ }
18
+
19
+ export type InclusionListErrorType =
20
+ | {code: InclusionListErrorCode.EMPTY_TRANSACTIONS}
21
+ | {code: InclusionListErrorCode.MAXIMUM_SIZE_EXCEEDED; totalBytes: number; maxBytes: number}
22
+ | {code: InclusionListErrorCode.EMPTY_TRANSACTION; transactionIndex: number}
23
+ | {code: InclusionListErrorCode.INVALID_SLOT; inclusionListSlot: Slot; currentSlot: Slot}
24
+ | {code: InclusionListErrorCode.MORE_THAN_TWO; validatorIndex: ValidatorIndex}
25
+ | {code: InclusionListErrorCode.VALIDATOR_NOT_IN_COMMITTEE; validatorIndex: ValidatorIndex}
26
+ | {code: InclusionListErrorCode.UNKNOWN_DEPENDENT_ROOT; dependentRoot: RootHex}
27
+ | {
28
+ code: InclusionListErrorCode.INVALID_DEPENDENT_ROOT_SLOT;
29
+ dependentRoot: RootHex;
30
+ dependentSlot: Slot;
31
+ maxSlot: Slot;
32
+ }
33
+ | {code: InclusionListErrorCode.INVALID_DEPENDENT_ROOT; dependentRoot: RootHex; epoch: Epoch}
34
+ | {code: InclusionListErrorCode.MISSING_SHUFFLING; dependentRoot: RootHex; epoch: Epoch}
35
+ | {code: InclusionListErrorCode.INVALID_SIGNATURE}
36
+ | {code: InclusionListErrorCode.PRE_HEZE_SLOT; inclusionListSlot: Slot};
37
+
38
+ export class InclusionListError extends GossipActionError<InclusionListErrorType> {}
@@ -8,6 +8,7 @@ export * from "./envelopeReconstructionError.js";
8
8
  export * from "./executionPayloadBid.js";
9
9
  export * from "./executionPayloadEnvelope.js";
10
10
  export * from "./gossipValidation.js";
11
+ export * from "./inclusionList.js";
11
12
  export * from "./payloadAttestation.js";
12
13
  export * from "./proposerPreferences.js";
13
14
  export * from "./proposerSlashingError.js";
@@ -49,6 +49,7 @@ import {
49
49
  AttestationPool,
50
50
  DeferredVoluntaryExitPool,
51
51
  ExecutionPayloadBidPool,
52
+ InclusionListStore,
52
53
  OpPool,
53
54
  PayloadAttestationPool,
54
55
  ProposerPreferencesPool,
@@ -131,6 +132,7 @@ export interface IBeaconChain {
131
132
  readonly executionPayloadBidPool: ExecutionPayloadBidPool;
132
133
  readonly payloadAttestationPool: PayloadAttestationPool;
133
134
  readonly proposerPreferencesPool: ProposerPreferencesPool;
135
+ readonly inclusionListStore: InclusionListStore;
134
136
  readonly opPool: OpPool;
135
137
  readonly deferredVoluntaryExitPool: DeferredVoluntaryExitPool;
136
138