@lodestar/beacon-node 1.51.0-dev.e22f67e7bc → 1.51.0-dev.e2c534f5b1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/lib/api/impl/api.js +1 -1
  2. package/lib/api/impl/api.js.map +1 -1
  3. package/lib/api/impl/beacon/blocks/index.d.ts +2 -1
  4. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/blocks/index.js +29 -34
  6. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  7. package/lib/api/impl/beacon/index.d.ts +2 -1
  8. package/lib/api/impl/beacon/index.d.ts.map +1 -1
  9. package/lib/api/impl/beacon/index.js +2 -2
  10. package/lib/api/impl/beacon/index.js.map +1 -1
  11. package/lib/api/impl/beacon/pool/index.d.ts.map +1 -1
  12. package/lib/api/impl/beacon/pool/index.js +5 -2
  13. package/lib/api/impl/beacon/pool/index.js.map +1 -1
  14. package/lib/api/impl/validator/index.d.ts.map +1 -1
  15. package/lib/api/impl/validator/index.js +4 -1
  16. package/lib/api/impl/validator/index.js.map +1 -1
  17. package/lib/api/options.d.ts +2 -0
  18. package/lib/api/options.d.ts.map +1 -1
  19. package/lib/api/options.js +1 -0
  20. package/lib/api/options.js.map +1 -1
  21. package/lib/chain/GetBlobsTracker.d.ts +11 -0
  22. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  23. package/lib/chain/GetBlobsTracker.js +85 -27
  24. package/lib/chain/GetBlobsTracker.js.map +1 -1
  25. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  26. package/lib/chain/archiveStore/archiveStore.js +1 -0
  27. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  28. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +4 -0
  29. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  30. package/lib/chain/archiveStore/utils/archiveBlocks.js +61 -97
  31. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  32. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  33. package/lib/chain/blocks/importBlock.js +6 -5
  34. package/lib/chain/blocks/importBlock.js.map +1 -1
  35. package/lib/chain/blocks/importExecutionPayload.d.ts +4 -1
  36. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  37. package/lib/chain/blocks/importExecutionPayload.js +9 -3
  38. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  39. package/lib/chain/blocks/index.js +2 -2
  40. package/lib/chain/blocks/index.js.map +1 -1
  41. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -5
  42. package/lib/chain/blocks/payloadEnvelopeProcessor.js +7 -7
  43. package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -1
  44. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  45. package/lib/chain/blocks/verifyBlock.js +1 -1
  46. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  47. package/lib/chain/blocks/verifyBlocksDataAvailability.d.ts.map +1 -1
  48. package/lib/chain/blocks/verifyBlocksDataAvailability.js +8 -1
  49. package/lib/chain/blocks/verifyBlocksDataAvailability.js.map +1 -1
  50. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +3 -3
  51. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -1
  52. package/lib/chain/blocks/verifyPayloadsDataAvailability.js +13 -6
  53. package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -1
  54. package/lib/chain/bls/multithread/index.js +5 -5
  55. package/lib/chain/bls/multithread/index.js.map +1 -1
  56. package/lib/chain/chain.js +4 -4
  57. package/lib/chain/chain.js.map +1 -1
  58. package/lib/chain/errors/blockError.d.ts +1 -1
  59. package/lib/chain/errors/blockError.d.ts.map +1 -1
  60. package/lib/chain/errors/blockError.js +1 -1
  61. package/lib/chain/errors/blockError.js.map +1 -1
  62. package/lib/chain/interface.d.ts +1 -2
  63. package/lib/chain/interface.d.ts.map +1 -1
  64. package/lib/chain/interface.js.map +1 -1
  65. package/lib/chain/opPools/inclusionListStore.d.ts.map +1 -1
  66. package/lib/chain/opPools/inclusionListStore.js +2 -2
  67. package/lib/chain/opPools/inclusionListStore.js.map +1 -1
  68. package/lib/chain/options.d.ts +0 -1
  69. package/lib/chain/options.d.ts.map +1 -1
  70. package/lib/chain/options.js +0 -1
  71. package/lib/chain/options.js.map +1 -1
  72. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  73. package/lib/chain/produceBlock/produceBlockBody.js +0 -1
  74. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  75. package/lib/chain/regen/interface.d.ts +0 -1
  76. package/lib/chain/regen/interface.d.ts.map +1 -1
  77. package/lib/chain/regen/interface.js +0 -1
  78. package/lib/chain/regen/interface.js.map +1 -1
  79. package/lib/chain/validation/executionPayloadBid.d.ts +0 -15
  80. package/lib/chain/validation/executionPayloadBid.d.ts.map +1 -1
  81. package/lib/chain/validation/executionPayloadBid.js +0 -153
  82. package/lib/chain/validation/executionPayloadBid.js.map +1 -1
  83. package/lib/chain/validation/inclusionList.d.ts.map +1 -1
  84. package/lib/chain/validation/inclusionList.js +11 -2
  85. package/lib/chain/validation/inclusionList.js.map +1 -1
  86. package/lib/db/beacon.d.ts +7 -0
  87. package/lib/db/beacon.d.ts.map +1 -1
  88. package/lib/db/beacon.js +34 -0
  89. package/lib/db/beacon.js.map +1 -1
  90. package/lib/db/dataColumnStore.d.ts +1 -1
  91. package/lib/db/dataColumnStore.d.ts.map +1 -1
  92. package/lib/db/dataColumnStore.js +1 -4
  93. package/lib/db/dataColumnStore.js.map +1 -1
  94. package/lib/db/interface.d.ts +1 -0
  95. package/lib/db/interface.d.ts.map +1 -1
  96. package/lib/db/repositories/dataColumnSidecarArchive.d.ts +6 -0
  97. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  98. package/lib/db/repositories/dataColumnSidecarArchive.js +19 -0
  99. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  100. package/lib/execution/engine/blobVersionedHashes.d.ts +1 -1
  101. package/lib/execution/engine/blobVersionedHashes.d.ts.map +1 -1
  102. package/lib/execution/engine/blobVersionedHashes.js.map +1 -1
  103. package/lib/execution/engine/http.d.ts.map +1 -1
  104. package/lib/execution/engine/http.js +0 -1
  105. package/lib/execution/engine/http.js.map +1 -1
  106. package/lib/execution/engine/index.d.ts +1 -1
  107. package/lib/execution/engine/index.d.ts.map +1 -1
  108. package/lib/execution/engine/index.js +1 -1
  109. package/lib/execution/engine/index.js.map +1 -1
  110. package/lib/execution/engine/interface.d.ts +1 -1
  111. package/lib/execution/engine/interface.d.ts.map +1 -1
  112. package/lib/execution/engine/interface.js.map +1 -1
  113. package/lib/execution/engine/mock.d.ts.map +1 -1
  114. package/lib/execution/engine/mock.js +6 -4
  115. package/lib/execution/engine/mock.js.map +1 -1
  116. package/lib/execution/engine/payloadIdCache.d.ts +1 -2
  117. package/lib/execution/engine/payloadIdCache.d.ts.map +1 -1
  118. package/lib/execution/engine/payloadIdCache.js +3 -3
  119. package/lib/execution/engine/payloadIdCache.js.map +1 -1
  120. package/lib/execution/engine/sszTypes.d.ts +38 -38
  121. package/lib/execution/engine/types.d.ts +1 -1
  122. package/lib/execution/engine/types.d.ts.map +1 -1
  123. package/lib/execution/engine/types.js +9 -8
  124. package/lib/execution/engine/types.js.map +1 -1
  125. package/lib/index.d.ts +1 -0
  126. package/lib/index.d.ts.map +1 -1
  127. package/lib/index.js +2 -0
  128. package/lib/index.js.map +1 -1
  129. package/lib/metrics/metrics/lodestar.d.ts +3 -0
  130. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  131. package/lib/metrics/metrics/lodestar.js +5 -0
  132. package/lib/metrics/metrics/lodestar.js.map +1 -1
  133. package/lib/monitoring/service.js +1 -1
  134. package/lib/monitoring/service.js.map +1 -1
  135. package/lib/network/gossip/topic.d.ts +6 -6
  136. package/lib/network/network.d.ts.map +1 -1
  137. package/lib/network/network.js +3 -1
  138. package/lib/network/network.js.map +1 -1
  139. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  140. package/lib/network/processor/gossipHandlers.js +33 -23
  141. package/lib/network/processor/gossipHandlers.js.map +1 -1
  142. package/lib/node/nodejs.d.ts.map +1 -1
  143. package/lib/node/nodejs.js +9 -1
  144. package/lib/node/nodejs.js.map +1 -1
  145. package/lib/node/options.d.ts +1 -1
  146. package/lib/node/options.d.ts.map +1 -1
  147. package/lib/node/options.js +1 -1
  148. package/lib/node/options.js.map +1 -1
  149. package/lib/sync/interface.d.ts.map +1 -1
  150. package/lib/sync/interface.js.map +1 -1
  151. package/lib/sync/unknownBlock.d.ts.map +1 -1
  152. package/lib/sync/unknownBlock.js +42 -18
  153. package/lib/sync/unknownBlock.js.map +1 -1
  154. package/lib/util/queue/itemQueue.js +1 -1
  155. package/lib/util/queue/itemQueue.js.map +1 -1
  156. package/lib/util/sszBytes.js +2 -2
  157. package/lib/util/sszBytes.js.map +1 -1
  158. package/package.json +30 -30
  159. package/src/api/impl/api.ts +1 -1
  160. package/src/api/impl/beacon/blocks/index.ts +57 -62
  161. package/src/api/impl/beacon/index.ts +3 -1
  162. package/src/api/impl/beacon/pool/index.ts +5 -1
  163. package/src/api/impl/validator/index.ts +4 -1
  164. package/src/api/options.ts +3 -0
  165. package/src/chain/GetBlobsTracker.ts +97 -38
  166. package/src/chain/archiveStore/archiveStore.ts +1 -0
  167. package/src/chain/archiveStore/utils/archiveBlocks.ts +76 -130
  168. package/src/chain/blocks/importBlock.ts +8 -7
  169. package/src/chain/blocks/importExecutionPayload.ts +14 -3
  170. package/src/chain/blocks/index.ts +6 -2
  171. package/src/chain/blocks/payloadEnvelopeProcessor.ts +7 -7
  172. package/src/chain/blocks/verifyBlock.ts +1 -0
  173. package/src/chain/blocks/verifyBlocksDataAvailability.ts +10 -1
  174. package/src/chain/blocks/verifyPayloadsDataAvailability.ts +16 -6
  175. package/src/chain/bls/multithread/index.ts +5 -5
  176. package/src/chain/chain.ts +4 -4
  177. package/src/chain/errors/blockError.ts +1 -1
  178. package/src/chain/interface.ts +1 -2
  179. package/src/chain/opPools/inclusionListStore.ts +3 -3
  180. package/src/chain/options.ts +0 -2
  181. package/src/chain/produceBlock/produceBlockBody.ts +0 -1
  182. package/src/chain/regen/interface.ts +0 -1
  183. package/src/chain/validation/executionPayloadBid.ts +0 -180
  184. package/src/chain/validation/inclusionList.ts +11 -2
  185. package/src/db/beacon.ts +44 -0
  186. package/src/db/dataColumnStore.ts +2 -4
  187. package/src/db/interface.ts +2 -0
  188. package/src/db/repositories/dataColumnSidecarArchive.ts +20 -0
  189. package/src/execution/engine/blobVersionedHashes.ts +1 -1
  190. package/src/execution/engine/http.ts +0 -1
  191. package/src/execution/engine/index.ts +1 -1
  192. package/src/execution/engine/interface.ts +1 -1
  193. package/src/execution/engine/mock.ts +6 -4
  194. package/src/execution/engine/payloadIdCache.ts +6 -7
  195. package/src/execution/engine/types.ts +11 -9
  196. package/src/index.ts +2 -0
  197. package/src/metrics/metrics/lodestar.ts +5 -0
  198. package/src/monitoring/service.ts +1 -1
  199. package/src/network/network.ts +3 -1
  200. package/src/network/processor/gossipHandlers.ts +41 -25
  201. package/src/node/nodejs.ts +18 -1
  202. package/src/node/options.ts +4 -5
  203. package/src/sync/interface.ts +1 -0
  204. package/src/sync/unknownBlock.ts +47 -28
  205. package/src/util/queue/itemQueue.ts +1 -1
  206. package/src/util/sszBytes.ts +2 -2
@@ -83,186 +83,6 @@ function isBidCompatibleWithHead(
83
83
  return buildsOnParentPayload;
84
84
  }
85
85
 
86
- /**
87
- * Validation for bids submitted via the API by the operator's own builder.
88
- *
89
- * Only non-transient checks are applied: slot later than parent, zero execution payment, blob
90
- * commitment count, builder eligibility, version and balance coverage, prev_randao, and signature.
91
- * Transient gossip rules (head compatibility, first bid per tuple, value increment, proposer
92
- * preferences) are not applied, since those only limit forwarding of peers' messages and the
93
- * builder may legitimately bid on a branch this node does not consider head.
94
- * The parent-payload builder-exit rule is omitted because this endpoint only accepts bids from
95
- * the operator's own builder.
96
- *
97
- * Throws on any failed check. Also throws if the bid's parent block is unknown or its state is
98
- * unavailable, since the checks cannot be evaluated against the parent branch.
99
- */
100
- export async function validateApiExecutionPayloadBid(
101
- chain: IBeaconChain,
102
- signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid
103
- ): Promise<void> {
104
- const bid = signedExecutionPayloadBid.message;
105
- const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
106
-
107
- // [REJECT] `bid.execution_payment` is zero.
108
- if (bid.executionPayment !== 0n) {
109
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
110
- code: ExecutionPayloadBidErrorCode.NON_ZERO_EXECUTION_PAYMENT,
111
- builderIndex: bid.builderIndex,
112
- executionPayment: bid.executionPayment,
113
- });
114
- }
115
-
116
- // [REJECT] The bid's block hash is not equal to its parent block hash -- i.e.
117
- // `bid.block_hash != bid.parent_block_hash`.
118
- if (byteArrayEquals(bid.blockHash, bid.parentBlockHash)) {
119
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
120
- code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH,
121
- blockHash: toRootHex(bid.blockHash),
122
- });
123
- }
124
-
125
- // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in the
126
- // consensus layer -- i.e. validate that
127
- // `len(bid.blob_kzg_commitments) <= get_blob_parameters(compute_epoch_at_slot(bid.slot)).max_blobs_per_block`.
128
- const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
129
- const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
130
- if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
131
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
132
- code: ExecutionPayloadBidErrorCode.TOO_MANY_KZG_COMMITMENTS,
133
- blobKzgCommitmentsLen,
134
- commitmentLimit: maxBlobsPerBlock,
135
- });
136
- }
137
-
138
- // [IGNORE] `bid.slot` is the current slot, or the next slot (`bid.slot - 1` is current), allowing for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
139
- if (
140
- !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot) &&
141
- !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot - 1)
142
- ) {
143
- throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
144
- code: ExecutionPayloadBidErrorCode.INVALID_SLOT,
145
- builderIndex: bid.builderIndex,
146
- slot: bid.slot,
147
- });
148
- }
149
-
150
- // [IGNORE] The bid is received after the inclusion list deadline of `bid.slot - 1`
151
- if (
152
- isForkPostHeze(chain.config.getForkName(bid.slot)) &&
153
- chain.clock.msFromSlot(bid.slot - 1) < chain.config.getInclusionListDueMs(chain.config.getForkName(bid.slot - 1))
154
- ) {
155
- throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
156
- code: ExecutionPayloadBidErrorCode.BEFORE_INCLUSION_LIST_DEADLINE,
157
- builderIndex: bid.builderIndex,
158
- slot: bid.slot,
159
- });
160
- }
161
-
162
- // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon block in fork choice.
163
- // Moved earlier than the spec ordering so we can derive the proposer dependent root for the
164
- // proposer-preferences lookup below from a known fork-choice block.
165
- const parentBlock = chain.forkChoice.getBlockHexDefaultStatus(bidParentBlockRoot);
166
- if (parentBlock === null) {
167
- throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
168
- code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT,
169
- parentBlockRoot: bidParentBlockRoot,
170
- });
171
- }
172
-
173
- // [REJECT] The bid is for a higher slot than its parent block -- i.e.
174
- // validate that `bid.slot` is greater than the slot of the block with root
175
- // `bid.parent_block_root`.
176
- if (bid.slot <= parentBlock.slot) {
177
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
178
- code: ExecutionPayloadBidErrorCode.NOT_LATER_THAN_PARENT,
179
- parentSlot: parentBlock.slot,
180
- slot: bid.slot,
181
- });
182
- }
183
-
184
- const state = await chain.regen
185
- .getBlockSlotState(parentBlock, bid.slot, {dontTransferCache: true}, RegenCaller.validateApiExecutionPayloadBid)
186
- .catch(() => {
187
- throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
188
- code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT,
189
- parentBlockRoot: bidParentBlockRoot,
190
- });
191
- });
192
-
193
- if (!isStatePostGloas(state)) {
194
- throw new Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
195
- }
196
-
197
- // [REJECT] `bid.builder_index` is within bounds -- i.e. `bid.builder_index < len(state.builders)`.
198
- // `state.getBuilder` returns a lazy SSZ `getReadonly` view that is not bounds-checked eagerly; an
199
- // out-of-range index only throws (`LeafNode has no right node`) on deferred field access (e.g. inside
200
- // `isActiveBuilder`), escaping a try/catch around `getBuilder`. Check the length explicitly instead.
201
- if (bid.builderIndex >= state.getBuildersLength()) {
202
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
203
- code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,
204
- builderIndex: bid.builderIndex,
205
- });
206
- }
207
-
208
- // [REJECT] `bid.builder_index` is a valid/active builder index -- i.e.
209
- // `is_active_builder(state, bid.builder_index)` returns `True`.
210
- const builder = state.getBuilder(bid.builderIndex);
211
- if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
212
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
213
- code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,
214
- builderIndex: bid.builderIndex,
215
- });
216
- }
217
-
218
- // [REJECT] The builder version is `PAYLOAD_BUILDER_VERSION` -- i.e.
219
- // `state.builders[bid.builder_index].version == PAYLOAD_BUILDER_VERSION`.
220
- if (builder.version !== PAYLOAD_BUILDER_VERSION) {
221
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
222
- code: ExecutionPayloadBidErrorCode.INVALID_BUILDER_VERSION,
223
- builderIndex: bid.builderIndex,
224
- version: builder.version,
225
- expectedVersion: PAYLOAD_BUILDER_VERSION,
226
- });
227
- }
228
-
229
- // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
230
- // i.e. `can_builder_cover_bid(state, builder_index, amount)` returns `True`.
231
- if (!state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
232
- throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
233
- code: ExecutionPayloadBidErrorCode.BID_TOO_HIGH,
234
- bidValue: bid.value,
235
- builderBalance: builder.balance,
236
- });
237
- }
238
-
239
- // [REJECT] `bid.prev_randao` is the correct RANDAO mix -- i.e. validate that
240
- // `bid.prev_randao == get_randao_mix(parent_state, get_current_epoch(parent_state))`.
241
- const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
242
- if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
243
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
244
- code: ExecutionPayloadBidErrorCode.INVALID_PREV_RANDAO,
245
- builderIndex: bid.builderIndex,
246
- bidPrevRandao: toHex(bid.prevRandao),
247
- expectedPrevRandao: toHex(randaoMix),
248
- });
249
- }
250
-
251
- // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
252
- const signatureSet = createSingleSignatureSetFromComponents(
253
- builder.pubkey,
254
- getExecutionPayloadBidSigningRoot(chain.config, bid),
255
- signedExecutionPayloadBid.signature
256
- );
257
- if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
258
- throw new ExecutionPayloadBidError(GossipAction.REJECT, {
259
- code: ExecutionPayloadBidErrorCode.INVALID_SIGNATURE,
260
- builderIndex: bid.builderIndex,
261
- slot: bid.slot,
262
- });
263
- }
264
- }
265
-
266
86
  export async function validateGossipExecutionPayloadBid(
267
87
  chain: IBeaconChain,
268
88
  signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid
@@ -31,7 +31,10 @@ export async function validateGossipInclusionList(
31
31
  signedInclusionList: heze.SignedInclusionList
32
32
  ): Promise<InclusionListValidationResult> {
33
33
  const {slot, validatorIndex, transactions, dependentRoot} = signedInclusionList.message;
34
- const totalBytes = transactions.reduce((total, transaction) => total + transaction.byteLength, 0);
34
+ let totalBytes = 0;
35
+ for (let i = 0; i < transactions.length; i++) {
36
+ totalBytes += transactions.getSize(i);
37
+ }
35
38
 
36
39
  // A pre-heze slot has no inclusion list committee to validate against
37
40
  if (!isForkPostHeze(chain.config.getForkName(slot))) {
@@ -75,7 +78,13 @@ export async function validateGossipInclusionList(
75
78
  }
76
79
 
77
80
  // [REJECT] Every transaction in message.transactions is non-empty
78
- const emptyTransactionIndex = transactions.findIndex((transaction) => transaction.byteLength === 0);
81
+ let emptyTransactionIndex = -1;
82
+ for (let i = 0; i < transactions.length; i++) {
83
+ if (transactions.getSize(i) === 0) {
84
+ emptyTransactionIndex = i;
85
+ break;
86
+ }
87
+ }
79
88
  if (emptyTransactionIndex !== -1) {
80
89
  throw new InclusionListError(GossipAction.REJECT, {
81
90
  code: InclusionListErrorCode.EMPTY_TRANSACTION,
package/src/db/beacon.ts CHANGED
@@ -30,6 +30,9 @@ import {
30
30
  VoluntaryExitRepository,
31
31
  } from "./repositories/index.js";
32
32
 
33
+ /** Below this size, seeks over leftover legacy column data stay cheap */
34
+ const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
35
+
33
36
  export type BeaconDbOpts = {
34
37
  dataColumnDir: string;
35
38
  logger: Logger;
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
67
70
  readonly dataColumns: IDataColumnStore;
68
71
  lastLegacyArchiveSlot: Slot | null = null;
69
72
  private readonly flatFileStore: FlatFileStore;
73
+ private readonly logger: Logger;
70
74
 
71
75
  constructor(
72
76
  config: ChainForkConfig,
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
100
104
  this.syncCommittee = new SyncCommitteeRepository(config, db);
101
105
  this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
102
106
 
107
+ this.logger = opts.logger;
103
108
  this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
104
109
  this.dataColumns = new LegacyDataColumnStore(
105
110
  this.flatFileStore,
@@ -125,11 +130,50 @@ export class BeaconDb implements IBeaconDb {
125
130
  this.flatFileStore.setMetrics(flatFileStoreMetrics);
126
131
  }
127
132
 
133
+ /**
134
+ * Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
135
+ * rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
136
+ * makes the compaction overlap those files and rewrite them.
137
+ */
138
+ async compactLegacyDataColumns(
139
+ prunedBeforeSlot: Slot,
140
+ minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
141
+ ): Promise<void> {
142
+ const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
143
+ const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
144
+ const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
145
+ const sizeBytes = await this.db.approximateSize(start, end);
146
+ if (sizeBytes < minSizeBytes) return;
147
+
148
+ // Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
149
+ const upperSentinel =
150
+ prunedBeforeSlot > 0
151
+ ? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
152
+ : encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
153
+ const sentinels = [start, upperSentinel];
154
+
155
+ this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
156
+ const startTime = Date.now();
157
+ await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
158
+ await this.db.compactRange(start, end);
159
+ await this.db.batchDelete(sentinels);
160
+ await this.db.compactRange(start, end);
161
+ this.logger.verbose("Compacted legacy data columns", {
162
+ durationMs: Date.now() - startTime,
163
+ sizeBytes: await this.db.approximateSize(start, end),
164
+ });
165
+ }
166
+
128
167
  async pruneHotDb(): Promise<void> {
129
168
  // Prune all hot blobs
130
169
  await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
131
170
  // New columns are persisted in flat files, so this bucket only contains legacy hot data.
132
171
  await this.deleteBucketData(Bucket.allForks_dataColumnSidecars);
172
+ // Compact even if empty to reclaim tombstones left by previous startups.
173
+ await this.db.compactRange(
174
+ encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0)),
175
+ encodeKey(Bucket.allForks_dataColumnSidecars + 1, Buffer.alloc(0))
176
+ );
133
177
  // Prune all hot blocks
134
178
  // TODO: Enable once it's deemed safe
135
179
  // await this.block.batchDelete(await this.block.keys());
@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
13
13
  type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
14
14
  type LegacyArchiveDataColumnStore = Pick<
15
15
  DataColumnSidecarArchiveRepository,
16
- "values" | "getManyBinary" | "keys" | "deleteMany"
16
+ "values" | "getManyBinary" | "pruneBefore"
17
17
  >;
18
18
  type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
19
19
 
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
86
86
 
87
87
  async pruneBefore(slot: Slot): Promise<Slot[]> {
88
88
  const flatFileSlots = await this.flatFiles.pruneBefore(slot);
89
- const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
90
- const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
91
- if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
89
+ const slots = await this.legacyArchive.pruneBefore(slot);
92
90
  return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
93
91
  }
94
92
  }
@@ -68,6 +68,8 @@ export interface IBeaconDb {
68
68
  readonly lastLegacyArchiveSlot: Slot | null;
69
69
  init(): Promise<void>;
70
70
 
71
+ compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
72
+
71
73
  pruneHotDb(): Promise<void>;
72
74
 
73
75
  deleteDeprecatedEth1Data(): Promise<void>;
@@ -15,6 +15,8 @@ const SLOT_BYTE_SIZE = 8;
15
15
  * Indexed data by `slot` + `columnIndex`
16
16
  */
17
17
  export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
18
+ private prunedBeforeSlot: Slot | null = null;
19
+
18
20
  constructor(config: ChainForkConfig, db: Db) {
19
21
  const bucket = Bucket.allForks_dataColumnSidecarsArchive;
20
22
  // Type won't be used since we select it dynamically based on fork
@@ -70,4 +72,22 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot,
70
72
  encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
71
73
  ]);
72
74
  }
75
+
76
+ /**
77
+ * Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
78
+ * compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
79
+ */
80
+ async pruneBefore(slot: Slot): Promise<Slot[]> {
81
+ if (this.prunedBeforeSlot === null) {
82
+ await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
83
+ } else if (slot <= this.prunedBeforeSlot) {
84
+ return [];
85
+ }
86
+
87
+ const keys = await this.keys({gte: {prefix: this.prunedBeforeSlot ?? 0, id: 0}, lt: {prefix: slot, id: 0}});
88
+ const slots = [...new Set(keys.map(({prefix}) => prefix))];
89
+ if (slots.length > 0) await this.deleteMany(slots);
90
+ this.prunedBeforeSlot = slot;
91
+ return slots;
92
+ }
73
93
  }
@@ -7,7 +7,7 @@ import {VersionedHashes} from "./interface.js";
7
7
  * Skip unrelated fields without decoding or copying them, including untrusted nested access lists.
8
8
  * The EL remains responsible for the other transaction validity checks.
9
9
  */
10
- export function isValidBlobVersionedHashes(transactions: Uint8Array[], expected: VersionedHashes): boolean {
10
+ export function isValidBlobVersionedHashes(transactions: Iterable<Uint8Array>, expected: VersionedHashes): boolean {
11
11
  let index = 0;
12
12
  for (const transaction of transactions) {
13
13
  if (transaction.length === 0) return false;
@@ -367,7 +367,6 @@ export class ExecutionEngineHttp implements IExecutionEngine {
367
367
  this.payloadIdCache.add(
368
368
  {
369
369
  headBlockHash,
370
- finalizedBlockHash,
371
370
  ...payloadAttributesRpc,
372
371
  inclusionListTransactions: payloadAttributes.inclusionListTransactions,
373
372
  },
@@ -17,8 +17,8 @@ import {EngineRestHttpClient} from "./restHttpClient.js";
17
17
  import {RestEngineTransport, formatClientVersionHeader} from "./restTransport.js";
18
18
  import {ExecutionEngineMockJsonRpcClient, JsonRpcBackend} from "./utils.js";
19
19
 
20
- export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts, engineApiModes};
21
20
  export type {EngineApiMode};
21
+ export {ExecutionEngineDisabled, ExecutionEngineHttp, defaultExecutionEngineHttpOpts, engineApiModes};
22
22
 
23
23
  export type ExecutionEngineOpts =
24
24
  | ({mode?: "http"} & ExecutionEngineHttpOpts)
@@ -25,7 +25,7 @@ import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
25
25
  import {ExecutionPayloadBodyV2} from "./types.js";
26
26
  import {DATA} from "./utils.js";
27
27
 
28
- export {PayloadIdCache, type PayloadId, type WithdrawalV1};
28
+ export {type PayloadId, PayloadIdCache, type WithdrawalV1};
29
29
  export type ClientVersion = routes.node.ClientVersion;
30
30
  export const ClientCode = routes.node.ClientCode;
31
31
 
@@ -1,4 +1,5 @@
1
1
  import crypto from "node:crypto";
2
+ import {SerializedList} from "@chainsafe/ssz";
2
3
  import {ChainConfig} from "@lodestar/config";
3
4
  import {
4
5
  BLOB_TX_TYPE,
@@ -352,14 +353,14 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
352
353
  executionPayload.gasUsed = Math.floor(0.5 * INTEROP_GAS_LIMIT);
353
354
  executionPayload.timestamp = payloadAttributes.timestamp;
354
355
  executionPayload.blockHash = crypto.randomBytes(32);
355
- executionPayload.transactions = [];
356
+ const transactions: Uint8Array[] = [];
356
357
 
357
358
  // Between 0 and 4 transactions for all forks
358
359
  const eip1559TxCount = Math.round(4 * Math.random());
359
360
  for (let i = 0; i < eip1559TxCount; i++) {
360
361
  const tx = crypto.randomBytes(512);
361
362
  tx[0] = TX_TYPE_EIP1559;
362
- executionPayload.transactions.push(tx);
363
+ transactions.push(tx);
363
364
  }
364
365
 
365
366
  const commitments: deneb.KZGCommitment[] = [];
@@ -373,7 +374,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
373
374
  const blob = generateRandomBlob();
374
375
  const commitment = kzg.blobToKzgCommitment(blob);
375
376
  const {proofs: cellProofs} = kzg.computeCellsAndKzgProofs(blob);
376
- executionPayload.transactions.push(transactionForKzgCommitment(commitment));
377
+ transactions.push(transactionForKzgCommitment(commitment));
377
378
  commitments.push(commitment);
378
379
  blobs.push(blob);
379
380
  proofs.push(...cellProofs);
@@ -385,12 +386,13 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
385
386
  const blob = generateRandomBlob();
386
387
  const commitment = kzg.blobToKzgCommitment(blob);
387
388
  const proof = kzg.computeBlobKzgProof(blob, commitment);
388
- executionPayload.transactions.push(transactionForKzgCommitment(commitment));
389
+ transactions.push(transactionForKzgCommitment(commitment));
389
390
  commitments.push(commitment);
390
391
  blobs.push(blob);
391
392
  proofs.push(proof);
392
393
  }
393
394
  }
395
+ executionPayload.transactions = SerializedList.fromArray(transactions);
394
396
 
395
397
  if (ForkSeq[fork] >= ForkSeq.capella) {
396
398
  (executionPayload as ExecutionPayload<ForkPostCapella>).withdrawals = ssz.capella.Withdrawals.defaultValue();
@@ -19,10 +19,9 @@ export type WithdrawalV1 = {
19
19
  amount: QUANTITY;
20
20
  };
21
21
 
22
- type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit<
23
- PayloadAttributesRpc,
24
- "withdrawals" | "inclusionListTransactions"
25
- > & {inclusionListTransactions?: bellatrix.Transactions};
22
+ type FcuAttributes = {headBlockHash: DATA} & Omit<PayloadAttributesRpc, "withdrawals" | "inclusionListTransactions"> & {
23
+ inclusionListTransactions?: bellatrix.Transactions;
24
+ };
26
25
 
27
26
  export class PayloadIdCache {
28
27
  private readonly payloadIdByFcuAttributes = new Map<
@@ -30,8 +29,8 @@ export class PayloadIdCache {
30
29
  {payloadId: PayloadId; fullKey: string; inclusionListTransactions?: bellatrix.Transactions}
31
30
  >();
32
31
 
33
- getFullKey({headBlockHash, finalizedBlockHash, timestamp, prevRandao, suggestedFeeRecipient}: FcuAttributes): string {
34
- return `${headBlockHash}-${finalizedBlockHash}-${timestamp}-${prevRandao}-${suggestedFeeRecipient}`;
32
+ getFullKey({headBlockHash, timestamp, prevRandao, suggestedFeeRecipient}: FcuAttributes): string {
33
+ return `${headBlockHash}-${timestamp}-${prevRandao}-${suggestedFeeRecipient}`;
35
34
  }
36
35
  getKey({timestamp}: Pick<FcuAttributes, "timestamp">): string {
37
36
  return timestamp;
@@ -69,5 +68,5 @@ function isSameInclusionList(a?: bellatrix.Transactions, b?: bellatrix.Transacti
69
68
  if (a === undefined || b === undefined) {
70
69
  return a === b;
71
70
  }
72
- return a.length === b.length && a.every((transaction, i) => byteArrayEquals(transaction, b[i]));
71
+ return byteArrayEquals(a.bytes, b.bytes);
73
72
  }
@@ -1,4 +1,4 @@
1
- import {Type} from "@chainsafe/ssz";
1
+ import {SerializedList, Type} from "@chainsafe/ssz";
2
2
  import {
3
3
  BUILDER_DEPOSIT_REQUEST_TYPE,
4
4
  BUILDER_EXIT_REQUEST_TYPE,
@@ -9,6 +9,7 @@ import {
9
9
  DEPOSIT_REQUEST_TYPE,
10
10
  FIELD_ELEMENTS_PER_BLOB,
11
11
  ForkName,
12
+ ForkPostElectra,
12
13
  ForkSeq,
13
14
  WITHDRAWAL_REQUEST_TYPE,
14
15
  } from "@lodestar/params";
@@ -23,6 +24,7 @@ import {
23
24
  deneb,
24
25
  gloas,
25
26
  ssz,
27
+ sszTypesFor,
26
28
  } from "@lodestar/types";
27
29
  import {BlobAndProof} from "@lodestar/types/deneb";
28
30
  import {BlobAndProofV2} from "@lodestar/types/fulu";
@@ -181,7 +183,7 @@ export type ExecutionPayloadBodyRpc = {
181
183
  };
182
184
 
183
185
  export type ExecutionPayloadBody = {
184
- transactions: bellatrix.Transaction[];
186
+ transactions: bellatrix.Transactions;
185
187
  withdrawals: capella.Withdrawals | null;
186
188
  };
187
189
 
@@ -316,7 +318,7 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload
316
318
  extraData: bytesToData(data.extraData),
317
319
  baseFeePerGas: numToQuantity(data.baseFeePerGas),
318
320
  blockHash: bytesToData(data.blockHash),
319
- transactions: data.transactions.map((tran) => bytesToData(tran)),
321
+ transactions: Array.from(data.transactions, (tran) => bytesToData(tran)),
320
322
  };
321
323
 
322
324
  // Capella adds withdrawals to the ExecutionPayload
@@ -400,7 +402,7 @@ export function parseExecutionPayload(
400
402
  extraData: dataToBytes(data.extraData, null),
401
403
  baseFeePerGas: quantityToBigint(data.baseFeePerGas),
402
404
  blockHash: dataToBytes(data.blockHash, 32),
403
- transactions: data.transactions.map((tran) => dataToBytes(tran, null)),
405
+ transactions: SerializedList.fromArray(data.transactions.map((tran) => dataToBytes(tran, null))),
404
406
  };
405
407
 
406
408
  if (ForkSeq[fork] >= ForkSeq.capella) {
@@ -474,11 +476,11 @@ export function serializeBeaconBlockRoot(data: Root): DATA {
474
476
  }
475
477
 
476
478
  export function serializeInclusionList(data: bellatrix.Transactions): InclusionListRpc {
477
- return data.map((transaction) => bytesToData(transaction));
479
+ return Array.from(data, (transaction) => bytesToData(transaction));
478
480
  }
479
481
 
480
482
  export function deserializeInclusionList(data: InclusionListRpc): bellatrix.Transactions {
481
- return data.map((transaction) => dataToBytes(transaction, null));
483
+ return SerializedList.fromArray(data.map((transaction) => dataToBytes(transaction, null)));
482
484
  }
483
485
 
484
486
  export function deserializePayloadAttributes(data: PayloadAttributesRpc): PayloadAttributes {
@@ -628,7 +630,7 @@ export function deserializeExecutionRequestsFromBytes(fork: ForkName, serialized
628
630
 
629
631
  switch (currentRequestType) {
630
632
  case DEPOSIT_REQUEST_TYPE: {
631
- result.deposits = ssz.electra.DepositRequests.deserialize(requests);
633
+ result.deposits = sszTypesFor(fork as ForkPostElectra).DepositRequests.deserialize(requests);
632
634
  break;
633
635
  }
634
636
  case WITHDRAWAL_REQUEST_TYPE: {
@@ -670,7 +672,7 @@ export function deserializeExecutionRequests(fork: ForkName, serialized: Executi
670
672
  export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null {
671
673
  return data
672
674
  ? {
673
- transactions: data.transactions.map((tran) => dataToBytes(tran, null)),
675
+ transactions: SerializedList.fromArray(data.transactions.map((tran) => dataToBytes(tran, null))),
674
676
  withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null,
675
677
  }
676
678
  : null;
@@ -693,7 +695,7 @@ export function serializeExecutionPayloadBodyV2(data: ExecutionPayloadBodyV2 | n
693
695
  export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null): ExecutionPayloadBodyRpc | null {
694
696
  return data
695
697
  ? {
696
- transactions: data.transactions.map((tran) => bytesToData(tran)),
698
+ transactions: Array.from(data.transactions, (tran) => bytesToData(tran)),
697
699
  withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null,
698
700
  }
699
701
  : null;
package/src/index.ts CHANGED
@@ -6,6 +6,8 @@ export {persistAnchorState} from "./chain/index.js";
6
6
  export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
7
7
  export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
8
8
  export {BeaconDb, type IBeaconDb} from "./db/index.js";
9
+ // Export execution engine to make it usable by builder
10
+ export {initializeExecutionEngine} from "./execution/index.js";
9
11
  // Export metrics utilities to de-duplicate validator metrics
10
12
  export {
11
13
  type HttpMetricsServer,
@@ -1474,6 +1474,11 @@ export function createLodestarMetrics(
1474
1474
  help: "Total number of InsertOutcome as a result of adding an execution payload bid from gossip to the pool",
1475
1475
  labelNames: ["insertOutcome"],
1476
1476
  }),
1477
+ apiInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({
1478
+ name: "lodestar_oppool_execution_payload_bid_pool_api_insert_outcome_total",
1479
+ help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
1480
+ labelNames: ["insertOutcome"],
1481
+ }),
1477
1482
  apiValidationTime: register.histogram({
1478
1483
  name: "lodestar_api_execution_payload_bid_validation_time_seconds",
1479
1484
  help: "Time elapsed for signed execution payload bid validation - api path",
@@ -101,7 +101,7 @@ export class MonitoringService {
101
101
  if (this.monitoringInterval) {
102
102
  clearTimeout(this.monitoringInterval);
103
103
  }
104
- if (this.pendingRequest) {
104
+ if (this.pendingRequest !== undefined) {
105
105
  this.fetchAbortController?.abort(FetchAbortReason.Close);
106
106
  await this.pendingRequest;
107
107
  }
@@ -855,7 +855,9 @@ export class Network implements INetwork {
855
855
  };
856
856
 
857
857
  private onTargetGroupCountUpdated = (count: number): void => {
858
- this.core.setTargetGroupCount(count);
858
+ this.core.setTargetGroupCount(count).catch((e) => {
859
+ this.logger.debug("Error setting target custody group count", {count}, e as Error);
860
+ });
859
861
  };
860
862
 
861
863
  private onPublishDataColumns = async (sidecars: DataColumnSidecar[]): Promise<void> => {