@lodestar/beacon-node 1.51.0-dev.b3beff2246 → 1.51.0-dev.bb16085c5e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/lib/api/impl/api.js +1 -1
  2. package/lib/api/impl/api.js.map +1 -1
  3. package/lib/api/impl/beacon/blocks/index.d.ts +2 -1
  4. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  5. package/lib/api/impl/beacon/blocks/index.js +29 -34
  6. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  7. package/lib/api/impl/beacon/index.d.ts +2 -1
  8. package/lib/api/impl/beacon/index.d.ts.map +1 -1
  9. package/lib/api/impl/beacon/index.js +2 -2
  10. package/lib/api/impl/beacon/index.js.map +1 -1
  11. package/lib/api/impl/beacon/pool/index.d.ts.map +1 -1
  12. package/lib/api/impl/beacon/pool/index.js +5 -2
  13. package/lib/api/impl/beacon/pool/index.js.map +1 -1
  14. package/lib/api/impl/beacon/state/utils.d.ts +1 -1
  15. package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
  16. package/lib/api/impl/validator/index.d.ts.map +1 -1
  17. package/lib/api/impl/validator/index.js +4 -1
  18. package/lib/api/impl/validator/index.js.map +1 -1
  19. package/lib/api/options.d.ts +2 -0
  20. package/lib/api/options.d.ts.map +1 -1
  21. package/lib/api/options.js +1 -0
  22. package/lib/api/options.js.map +1 -1
  23. package/lib/chain/GetBlobsTracker.d.ts +11 -0
  24. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  25. package/lib/chain/GetBlobsTracker.js +85 -27
  26. package/lib/chain/GetBlobsTracker.js.map +1 -1
  27. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  28. package/lib/chain/archiveStore/archiveStore.js +1 -0
  29. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  30. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +4 -0
  31. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  32. package/lib/chain/archiveStore/utils/archiveBlocks.js +61 -97
  33. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  34. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  35. package/lib/chain/blocks/importBlock.js +6 -5
  36. package/lib/chain/blocks/importBlock.js.map +1 -1
  37. package/lib/chain/blocks/importExecutionPayload.d.ts +4 -1
  38. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  39. package/lib/chain/blocks/importExecutionPayload.js +9 -3
  40. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  41. package/lib/chain/blocks/index.js +2 -2
  42. package/lib/chain/blocks/index.js.map +1 -1
  43. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -5
  44. package/lib/chain/blocks/payloadEnvelopeProcessor.js +7 -7
  45. package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -1
  46. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  47. package/lib/chain/blocks/verifyBlock.js +1 -1
  48. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  49. package/lib/chain/blocks/verifyBlocksDataAvailability.d.ts.map +1 -1
  50. package/lib/chain/blocks/verifyBlocksDataAvailability.js +8 -1
  51. package/lib/chain/blocks/verifyBlocksDataAvailability.js.map +1 -1
  52. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +3 -3
  53. package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -1
  54. package/lib/chain/blocks/verifyPayloadsDataAvailability.js +13 -6
  55. package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -1
  56. package/lib/chain/bls/multithread/index.js +5 -5
  57. package/lib/chain/bls/multithread/index.js.map +1 -1
  58. package/lib/chain/bls/multithread/types.d.ts +1 -1
  59. package/lib/chain/bls/multithread/types.d.ts.map +1 -1
  60. package/lib/chain/bls/multithread/worker.js +1 -1
  61. package/lib/chain/bls/multithread/worker.js.map +1 -1
  62. package/lib/chain/bls/singleThread.js +1 -1
  63. package/lib/chain/bls/singleThread.js.map +1 -1
  64. package/lib/chain/bls/utils.js +1 -1
  65. package/lib/chain/bls/utils.js.map +1 -1
  66. package/lib/chain/chain.d.ts +1 -1
  67. package/lib/chain/chain.d.ts.map +1 -1
  68. package/lib/chain/chain.js +4 -4
  69. package/lib/chain/chain.js.map +1 -1
  70. package/lib/chain/errors/blockError.d.ts +1 -1
  71. package/lib/chain/errors/blockError.d.ts.map +1 -1
  72. package/lib/chain/errors/blockError.js +1 -1
  73. package/lib/chain/errors/blockError.js.map +1 -1
  74. package/lib/chain/interface.d.ts +2 -3
  75. package/lib/chain/interface.d.ts.map +1 -1
  76. package/lib/chain/interface.js.map +1 -1
  77. package/lib/chain/opPools/aggregatedAttestationPool.d.ts.map +1 -1
  78. package/lib/chain/opPools/aggregatedAttestationPool.js +1 -1
  79. package/lib/chain/opPools/aggregatedAttestationPool.js.map +1 -1
  80. package/lib/chain/opPools/attestationPool.d.ts +1 -1
  81. package/lib/chain/opPools/attestationPool.d.ts.map +1 -1
  82. package/lib/chain/opPools/attestationPool.js +1 -1
  83. package/lib/chain/opPools/attestationPool.js.map +1 -1
  84. package/lib/chain/opPools/inclusionListStore.d.ts.map +1 -1
  85. package/lib/chain/opPools/inclusionListStore.js +2 -2
  86. package/lib/chain/opPools/inclusionListStore.js.map +1 -1
  87. package/lib/chain/opPools/payloadAttestationPool.d.ts.map +1 -1
  88. package/lib/chain/opPools/payloadAttestationPool.js +1 -1
  89. package/lib/chain/opPools/payloadAttestationPool.js.map +1 -1
  90. package/lib/chain/opPools/syncCommitteeMessagePool.d.ts.map +1 -1
  91. package/lib/chain/opPools/syncCommitteeMessagePool.js +1 -1
  92. package/lib/chain/opPools/syncCommitteeMessagePool.js.map +1 -1
  93. package/lib/chain/opPools/syncContributionAndProofPool.d.ts.map +1 -1
  94. package/lib/chain/opPools/syncContributionAndProofPool.js +1 -1
  95. package/lib/chain/opPools/syncContributionAndProofPool.js.map +1 -1
  96. package/lib/chain/opPools/utils.d.ts +1 -1
  97. package/lib/chain/opPools/utils.d.ts.map +1 -1
  98. package/lib/chain/opPools/utils.js +1 -1
  99. package/lib/chain/opPools/utils.js.map +1 -1
  100. package/lib/chain/options.d.ts +0 -1
  101. package/lib/chain/options.d.ts.map +1 -1
  102. package/lib/chain/options.js +0 -1
  103. package/lib/chain/options.js.map +1 -1
  104. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  105. package/lib/chain/produceBlock/produceBlockBody.js +0 -1
  106. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  107. package/lib/chain/regen/interface.d.ts +0 -1
  108. package/lib/chain/regen/interface.d.ts.map +1 -1
  109. package/lib/chain/regen/interface.js +0 -1
  110. package/lib/chain/regen/interface.js.map +1 -1
  111. package/lib/chain/validation/executionPayloadBid.d.ts +0 -15
  112. package/lib/chain/validation/executionPayloadBid.d.ts.map +1 -1
  113. package/lib/chain/validation/executionPayloadBid.js +0 -153
  114. package/lib/chain/validation/executionPayloadBid.js.map +1 -1
  115. package/lib/chain/validation/inclusionList.d.ts.map +1 -1
  116. package/lib/chain/validation/inclusionList.js +11 -2
  117. package/lib/chain/validation/inclusionList.js.map +1 -1
  118. package/lib/db/beacon.d.ts +7 -0
  119. package/lib/db/beacon.d.ts.map +1 -1
  120. package/lib/db/beacon.js +34 -0
  121. package/lib/db/beacon.js.map +1 -1
  122. package/lib/db/dataColumnStore.d.ts +1 -1
  123. package/lib/db/dataColumnStore.d.ts.map +1 -1
  124. package/lib/db/dataColumnStore.js +1 -4
  125. package/lib/db/dataColumnStore.js.map +1 -1
  126. package/lib/db/interface.d.ts +1 -0
  127. package/lib/db/interface.d.ts.map +1 -1
  128. package/lib/db/repositories/dataColumnSidecarArchive.d.ts +6 -0
  129. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  130. package/lib/db/repositories/dataColumnSidecarArchive.js +19 -0
  131. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  132. package/lib/execution/engine/blobVersionedHashes.d.ts +1 -1
  133. package/lib/execution/engine/blobVersionedHashes.d.ts.map +1 -1
  134. package/lib/execution/engine/blobVersionedHashes.js.map +1 -1
  135. package/lib/execution/engine/http.d.ts.map +1 -1
  136. package/lib/execution/engine/http.js +0 -1
  137. package/lib/execution/engine/http.js.map +1 -1
  138. package/lib/execution/engine/index.d.ts +1 -1
  139. package/lib/execution/engine/index.d.ts.map +1 -1
  140. package/lib/execution/engine/index.js +1 -1
  141. package/lib/execution/engine/index.js.map +1 -1
  142. package/lib/execution/engine/interface.d.ts +1 -1
  143. package/lib/execution/engine/interface.d.ts.map +1 -1
  144. package/lib/execution/engine/interface.js.map +1 -1
  145. package/lib/execution/engine/mock.d.ts.map +1 -1
  146. package/lib/execution/engine/mock.js +6 -4
  147. package/lib/execution/engine/mock.js.map +1 -1
  148. package/lib/execution/engine/payloadIdCache.d.ts +1 -2
  149. package/lib/execution/engine/payloadIdCache.d.ts.map +1 -1
  150. package/lib/execution/engine/payloadIdCache.js +3 -3
  151. package/lib/execution/engine/payloadIdCache.js.map +1 -1
  152. package/lib/execution/engine/sszTypes.d.ts +38 -38
  153. package/lib/execution/engine/types.d.ts +1 -1
  154. package/lib/execution/engine/types.d.ts.map +1 -1
  155. package/lib/execution/engine/types.js +9 -8
  156. package/lib/execution/engine/types.js.map +1 -1
  157. package/lib/index.d.ts +1 -0
  158. package/lib/index.d.ts.map +1 -1
  159. package/lib/index.js +2 -0
  160. package/lib/index.js.map +1 -1
  161. package/lib/metrics/metrics/lodestar.d.ts +3 -0
  162. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  163. package/lib/metrics/metrics/lodestar.js +5 -0
  164. package/lib/metrics/metrics/lodestar.js.map +1 -1
  165. package/lib/monitoring/service.js +1 -1
  166. package/lib/monitoring/service.js.map +1 -1
  167. package/lib/network/gossip/topic.d.ts +6 -6
  168. package/lib/network/network.d.ts.map +1 -1
  169. package/lib/network/network.js +3 -1
  170. package/lib/network/network.js.map +1 -1
  171. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  172. package/lib/network/processor/gossipHandlers.js +33 -23
  173. package/lib/network/processor/gossipHandlers.js.map +1 -1
  174. package/lib/node/nodejs.d.ts +1 -1
  175. package/lib/node/nodejs.d.ts.map +1 -1
  176. package/lib/node/nodejs.js +10 -2
  177. package/lib/node/nodejs.js.map +1 -1
  178. package/lib/node/options.d.ts +1 -1
  179. package/lib/node/options.d.ts.map +1 -1
  180. package/lib/node/options.js +1 -1
  181. package/lib/node/options.js.map +1 -1
  182. package/lib/sync/interface.d.ts.map +1 -1
  183. package/lib/sync/interface.js.map +1 -1
  184. package/lib/sync/unknownBlock.d.ts.map +1 -1
  185. package/lib/sync/unknownBlock.js +42 -18
  186. package/lib/sync/unknownBlock.js.map +1 -1
  187. package/lib/util/queue/itemQueue.js +1 -1
  188. package/lib/util/queue/itemQueue.js.map +1 -1
  189. package/lib/util/sszBytes.js +2 -2
  190. package/lib/util/sszBytes.js.map +1 -1
  191. package/package.json +17 -17
  192. package/src/api/impl/api.ts +1 -1
  193. package/src/api/impl/beacon/blocks/index.ts +57 -62
  194. package/src/api/impl/beacon/index.ts +3 -1
  195. package/src/api/impl/beacon/pool/index.ts +5 -1
  196. package/src/api/impl/beacon/state/utils.ts +1 -1
  197. package/src/api/impl/validator/index.ts +4 -1
  198. package/src/api/options.ts +3 -0
  199. package/src/chain/GetBlobsTracker.ts +97 -38
  200. package/src/chain/archiveStore/archiveStore.ts +1 -0
  201. package/src/chain/archiveStore/utils/archiveBlocks.ts +76 -130
  202. package/src/chain/blocks/importBlock.ts +8 -7
  203. package/src/chain/blocks/importExecutionPayload.ts +14 -3
  204. package/src/chain/blocks/index.ts +6 -2
  205. package/src/chain/blocks/payloadEnvelopeProcessor.ts +7 -7
  206. package/src/chain/blocks/verifyBlock.ts +1 -0
  207. package/src/chain/blocks/verifyBlocksDataAvailability.ts +10 -1
  208. package/src/chain/blocks/verifyPayloadsDataAvailability.ts +16 -6
  209. package/src/chain/bls/multithread/index.ts +5 -5
  210. package/src/chain/bls/multithread/types.ts +1 -1
  211. package/src/chain/bls/multithread/worker.ts +1 -5
  212. package/src/chain/bls/singleThread.ts +1 -1
  213. package/src/chain/bls/utils.ts +1 -1
  214. package/src/chain/chain.ts +5 -5
  215. package/src/chain/errors/blockError.ts +1 -1
  216. package/src/chain/interface.ts +2 -3
  217. package/src/chain/opPools/aggregatedAttestationPool.ts +1 -1
  218. package/src/chain/opPools/attestationPool.ts +1 -1
  219. package/src/chain/opPools/inclusionListStore.ts +3 -3
  220. package/src/chain/opPools/payloadAttestationPool.ts +1 -1
  221. package/src/chain/opPools/syncCommitteeMessagePool.ts +1 -1
  222. package/src/chain/opPools/syncContributionAndProofPool.ts +1 -1
  223. package/src/chain/opPools/utils.ts +1 -1
  224. package/src/chain/options.ts +0 -2
  225. package/src/chain/produceBlock/produceBlockBody.ts +0 -1
  226. package/src/chain/regen/interface.ts +0 -1
  227. package/src/chain/validation/executionPayloadBid.ts +0 -180
  228. package/src/chain/validation/inclusionList.ts +11 -2
  229. package/src/db/beacon.ts +44 -0
  230. package/src/db/dataColumnStore.ts +2 -4
  231. package/src/db/interface.ts +2 -0
  232. package/src/db/repositories/dataColumnSidecarArchive.ts +20 -0
  233. package/src/execution/engine/blobVersionedHashes.ts +1 -1
  234. package/src/execution/engine/http.ts +0 -1
  235. package/src/execution/engine/index.ts +1 -1
  236. package/src/execution/engine/interface.ts +1 -1
  237. package/src/execution/engine/mock.ts +6 -4
  238. package/src/execution/engine/payloadIdCache.ts +6 -7
  239. package/src/execution/engine/types.ts +11 -9
  240. package/src/index.ts +2 -0
  241. package/src/metrics/metrics/lodestar.ts +5 -0
  242. package/src/monitoring/service.ts +1 -1
  243. package/src/network/network.ts +3 -1
  244. package/src/network/processor/gossipHandlers.ts +41 -25
  245. package/src/node/nodejs.ts +24 -7
  246. package/src/node/options.ts +4 -5
  247. package/src/sync/interface.ts +1 -0
  248. package/src/sync/unknownBlock.ts +47 -28
  249. package/src/util/queue/itemQueue.ts +1 -1
  250. package/src/util/sszBytes.ts +2 -2
@@ -1,4 +1,4 @@
1
- import {Type} from "@chainsafe/ssz";
1
+ import {SerializedList, Type} from "@chainsafe/ssz";
2
2
  import {
3
3
  BUILDER_DEPOSIT_REQUEST_TYPE,
4
4
  BUILDER_EXIT_REQUEST_TYPE,
@@ -9,6 +9,7 @@ import {
9
9
  DEPOSIT_REQUEST_TYPE,
10
10
  FIELD_ELEMENTS_PER_BLOB,
11
11
  ForkName,
12
+ ForkPostElectra,
12
13
  ForkSeq,
13
14
  WITHDRAWAL_REQUEST_TYPE,
14
15
  } from "@lodestar/params";
@@ -23,6 +24,7 @@ import {
23
24
  deneb,
24
25
  gloas,
25
26
  ssz,
27
+ sszTypesFor,
26
28
  } from "@lodestar/types";
27
29
  import {BlobAndProof} from "@lodestar/types/deneb";
28
30
  import {BlobAndProofV2} from "@lodestar/types/fulu";
@@ -181,7 +183,7 @@ export type ExecutionPayloadBodyRpc = {
181
183
  };
182
184
 
183
185
  export type ExecutionPayloadBody = {
184
- transactions: bellatrix.Transaction[];
186
+ transactions: bellatrix.Transactions;
185
187
  withdrawals: capella.Withdrawals | null;
186
188
  };
187
189
 
@@ -316,7 +318,7 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload
316
318
  extraData: bytesToData(data.extraData),
317
319
  baseFeePerGas: numToQuantity(data.baseFeePerGas),
318
320
  blockHash: bytesToData(data.blockHash),
319
- transactions: data.transactions.map((tran) => bytesToData(tran)),
321
+ transactions: Array.from(data.transactions, (tran) => bytesToData(tran)),
320
322
  };
321
323
 
322
324
  // Capella adds withdrawals to the ExecutionPayload
@@ -400,7 +402,7 @@ export function parseExecutionPayload(
400
402
  extraData: dataToBytes(data.extraData, null),
401
403
  baseFeePerGas: quantityToBigint(data.baseFeePerGas),
402
404
  blockHash: dataToBytes(data.blockHash, 32),
403
- transactions: data.transactions.map((tran) => dataToBytes(tran, null)),
405
+ transactions: SerializedList.fromArray(data.transactions.map((tran) => dataToBytes(tran, null))),
404
406
  };
405
407
 
406
408
  if (ForkSeq[fork] >= ForkSeq.capella) {
@@ -474,11 +476,11 @@ export function serializeBeaconBlockRoot(data: Root): DATA {
474
476
  }
475
477
 
476
478
  export function serializeInclusionList(data: bellatrix.Transactions): InclusionListRpc {
477
- return data.map((transaction) => bytesToData(transaction));
479
+ return Array.from(data, (transaction) => bytesToData(transaction));
478
480
  }
479
481
 
480
482
  export function deserializeInclusionList(data: InclusionListRpc): bellatrix.Transactions {
481
- return data.map((transaction) => dataToBytes(transaction, null));
483
+ return SerializedList.fromArray(data.map((transaction) => dataToBytes(transaction, null)));
482
484
  }
483
485
 
484
486
  export function deserializePayloadAttributes(data: PayloadAttributesRpc): PayloadAttributes {
@@ -628,7 +630,7 @@ export function deserializeExecutionRequestsFromBytes(fork: ForkName, serialized
628
630
 
629
631
  switch (currentRequestType) {
630
632
  case DEPOSIT_REQUEST_TYPE: {
631
- result.deposits = ssz.electra.DepositRequests.deserialize(requests);
633
+ result.deposits = sszTypesFor(fork as ForkPostElectra).DepositRequests.deserialize(requests);
632
634
  break;
633
635
  }
634
636
  case WITHDRAWAL_REQUEST_TYPE: {
@@ -670,7 +672,7 @@ export function deserializeExecutionRequests(fork: ForkName, serialized: Executi
670
672
  export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null {
671
673
  return data
672
674
  ? {
673
- transactions: data.transactions.map((tran) => dataToBytes(tran, null)),
675
+ transactions: SerializedList.fromArray(data.transactions.map((tran) => dataToBytes(tran, null))),
674
676
  withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null,
675
677
  }
676
678
  : null;
@@ -693,7 +695,7 @@ export function serializeExecutionPayloadBodyV2(data: ExecutionPayloadBodyV2 | n
693
695
  export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null): ExecutionPayloadBodyRpc | null {
694
696
  return data
695
697
  ? {
696
- transactions: data.transactions.map((tran) => bytesToData(tran)),
698
+ transactions: Array.from(data.transactions, (tran) => bytesToData(tran)),
697
699
  withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null,
698
700
  }
699
701
  : null;
package/src/index.ts CHANGED
@@ -6,6 +6,8 @@ export {persistAnchorState} from "./chain/index.js";
6
6
  export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
7
7
  export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
8
8
  export {BeaconDb, type IBeaconDb} from "./db/index.js";
9
+ // Export execution engine to make it usable by builder
10
+ export {initializeExecutionEngine} from "./execution/index.js";
9
11
  // Export metrics utilities to de-duplicate validator metrics
10
12
  export {
11
13
  type HttpMetricsServer,
@@ -1474,6 +1474,11 @@ export function createLodestarMetrics(
1474
1474
  help: "Total number of InsertOutcome as a result of adding an execution payload bid from gossip to the pool",
1475
1475
  labelNames: ["insertOutcome"],
1476
1476
  }),
1477
+ apiInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({
1478
+ name: "lodestar_oppool_execution_payload_bid_pool_api_insert_outcome_total",
1479
+ help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
1480
+ labelNames: ["insertOutcome"],
1481
+ }),
1477
1482
  apiValidationTime: register.histogram({
1478
1483
  name: "lodestar_api_execution_payload_bid_validation_time_seconds",
1479
1484
  help: "Time elapsed for signed execution payload bid validation - api path",
@@ -101,7 +101,7 @@ export class MonitoringService {
101
101
  if (this.monitoringInterval) {
102
102
  clearTimeout(this.monitoringInterval);
103
103
  }
104
- if (this.pendingRequest) {
104
+ if (this.pendingRequest !== undefined) {
105
105
  this.fetchAbortController?.abort(FetchAbortReason.Close);
106
106
  await this.pendingRequest;
107
107
  }
@@ -855,7 +855,9 @@ export class Network implements INetwork {
855
855
  };
856
856
 
857
857
  private onTargetGroupCountUpdated = (count: number): void => {
858
- this.core.setTargetGroupCount(count);
858
+ this.core.setTargetGroupCount(count).catch((e) => {
859
+ this.logger.debug("Error setting target custody group count", {count}, e as Error);
860
+ });
859
861
  };
860
862
 
861
863
  private onPublishDataColumns = async (sidecars: DataColumnSidecar[]): Promise<void> => {
@@ -24,6 +24,7 @@ import {
24
24
  deneb,
25
25
  fulu,
26
26
  gloas,
27
+ isGloasBeaconBlock,
27
28
  isGloasDataColumnSidecar,
28
29
  ssz,
29
30
  sszTypesFor,
@@ -150,6 +151,31 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
150
151
  */
151
152
  function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): SequentialGossipHandlers {
152
153
  const {chain, config, metrics, logger, core} = modules;
154
+ const incompletePayloadWaits = new WeakSet<PayloadEnvelopeInput>();
155
+
156
+ function scheduleIncompletePayloadEnvelope(payloadInput: PayloadEnvelopeInput, peer: string): void {
157
+ if (payloadInput.isComplete() || incompletePayloadWaits.has(payloadInput)) {
158
+ return;
159
+ }
160
+ incompletePayloadWaits.add(payloadInput);
161
+
162
+ const fork = config.getForkName(payloadInput.slot);
163
+ const cutoffTimeMs = getCutoffTimeMs(chain, payloadInput.slot, config.getPayloadDueMs(fork));
164
+ payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch(() => {
165
+ if (
166
+ payloadInput.isComplete() ||
167
+ chain.seenPayloadEnvelopeInputCache.get(payloadInput.blockRootHex) !== payloadInput
168
+ ) {
169
+ return;
170
+ }
171
+ chain.logger.debug("Payload envelope incomplete at payload deadline", payloadInput.getLogMeta());
172
+ chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
173
+ payloadInput,
174
+ peer,
175
+ source: BlockInputSource.gossip,
176
+ });
177
+ });
178
+ }
153
179
 
154
180
  /**
155
181
  * Add a gossip block to the seen caches. The proposer signature of the block MUST be verified.
@@ -679,8 +705,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
679
705
 
680
706
  // Immediately attempt fetch of data columns from execution engine as the bid contains kzg commitments
681
707
  // which is all the information we need so there is no reason to delay until execution payload arrives
682
- // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
683
- // (for example later in the slot). Do not couple retries to incoming gossip columns.
684
708
  // Columns fetched here feed payloadInput.addColumn, which resolves waitForAllData for any
685
709
  // in-flight importExecutionPayload. No processExecutionPayload trigger needed from this path.
686
710
  chain.getBlobsTracker.triggerGetBlobs(payloadInput);
@@ -692,11 +716,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
692
716
 
693
717
  if (e instanceof BlockError) {
694
718
  switch (e.type.code) {
695
- case BlockErrorCode.DATA_UNAVAILABLE: {
696
- // Error is quite frequent and not critical
719
+ case BlockErrorCode.DATA_UNAVAILABLE:
720
+ // Data did not arrive in time, BlockInputSync keeps pulling it and retries the import
697
721
  logLevel = LogLevel.debug;
698
722
  break;
699
- }
700
723
  // ALREADY_KNOWN should not happen with ignoreIfKnown=true above
701
724
  // PARENT_BLOCK_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above
702
725
  case BlockErrorCode.ALREADY_KNOWN:
@@ -764,7 +787,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
764
787
  // roots per proposer and slot, which bounds full imports over gossip to one alternate
765
788
  chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root) &&
766
789
  // A sibling with an unknown parent cannot be imported, sync fetches it if its branch becomes relevant
767
- chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null
790
+ chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null &&
791
+ (!isGloasBeaconBlock(signedBlock.message) ||
792
+ chain.forkChoice.getBlockHexAndBlockHash(
793
+ toRootHex(signedBlock.message.parentRoot),
794
+ toRootHex(signedBlock.message.body.signedExecutionPayloadBid.message.parentBlockHash)
795
+ ) !== null)
768
796
  ) {
769
797
  const blockInput = cacheSignatureVerifiedBlock(
770
798
  signedBlock,
@@ -902,25 +930,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
902
930
  // envelope arrival (gossip or API). An in-flight importExecutionPayload is awaiting
903
931
  // payloadInput.waitForAllData(); addColumn above will resolve it once hasAllData flips.
904
932
 
905
- if (!payloadInput.isComplete()) {
906
- const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
907
- // do not await here to not delay gossip validation
908
- payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch((_e) => {
909
- chain.logger.debug(
910
- "Waited for envelope and data after receiving gossip column. Cut-off reached so emitting incompletePayloadEnvelope",
911
- {
912
- dataColumnIndex: index,
913
- ...payloadInputMeta,
914
- }
915
- );
916
- // TODO GLOAS: UnknownBlockSync to handle this event
917
- chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
918
- payloadInput,
919
- peer: peerIdStr,
920
- source: BlockInputSource.gossip,
921
- });
922
- });
923
- }
933
+ scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
924
934
  } catch (e) {
925
935
  logger.debug(
926
936
  "Error handling gossip data column",
@@ -1330,6 +1340,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
1330
1340
  peerIdStr,
1331
1341
  });
1332
1342
 
1343
+ scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
1344
+
1333
1345
  chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
1334
1346
  slot,
1335
1347
  builderIndex: envelope.builderIndex,
@@ -1343,6 +1355,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
1343
1355
 
1344
1356
  if (e instanceof PayloadError) {
1345
1357
  switch (e.type.code) {
1358
+ case PayloadErrorCode.DATA_UNAVAILABLE:
1359
+ // Columns did not arrive in time, BlockInputSync keeps pulling them and retries the import
1360
+ logLevel = LogLevel.debug;
1361
+ break;
1346
1362
  // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
1347
1363
  // already verified the block is in fork choice
1348
1364
  case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
@@ -1,23 +1,29 @@
1
1
  import {setMaxListeners} from "node:events";
2
2
  import {PrivateKey} from "@libp2p/interface";
3
3
  import {Registry} from "prom-client";
4
+ import {hasher} from "@chainsafe/persistent-merkle-tree";
5
+ import {BeaconApiMethods} from "@lodestar/api/beacon/server";
6
+ import {BeaconConfig} from "@lodestar/config";
7
+ import type {LoggerNode} from "@lodestar/logger/node";
4
8
  import {
5
9
  init as initNativeMetrics,
6
10
  registerLocalValidator as registerNativeLocalValidator,
7
11
  scrapeMetrics as scrapeNativeMetrics,
8
12
  unregisterLocalValidator as unregisterNativeLocalValidator,
9
- } from "@chainsafe/lodestar-z/metrics";
10
- import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
11
- import {hasher} from "@chainsafe/persistent-merkle-tree";
12
- import {BeaconApiMethods} from "@lodestar/api/beacon/server";
13
- import {BeaconConfig} from "@lodestar/config";
14
- import type {LoggerNode} from "@lodestar/logger/node";
13
+ } from "@lodestar/native/metrics";
14
+ import {type PubkeyCache} from "@lodestar/native/pubkeys";
15
15
  import {ZERO_HASH_HEX} from "@lodestar/params";
16
- import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
16
+ import {
17
+ IBeaconStateView,
18
+ computeStartSlotAtEpoch,
19
+ isStatePostBellatrix,
20
+ isStatePostGloas,
21
+ } from "@lodestar/state-transition";
17
22
  import {Slot} from "@lodestar/types";
18
23
  import {sleep, toRootHex} from "@lodestar/utils";
19
24
  import {ProcessShutdownCallback} from "@lodestar/validator";
20
25
  import {BeaconRestApiServer, getApi} from "../api/index.js";
26
+ import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
21
27
  import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
22
28
  import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
23
29
  import {IBeaconDb} from "../db/index.js";
@@ -212,6 +218,17 @@ export class BeaconNode {
212
218
 
213
219
  const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
214
220
 
221
+ // Before any lookup into the legacy column buckets, which can be slow until compacted
222
+ const columnsMinEpoch =
223
+ opts.chain.archiveDataEpochs === Infinity
224
+ ? null
225
+ : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
226
+ await db.compactLegacyDataColumns(
227
+ columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
228
+ ? computeStartSlotAtEpoch(columnsMinEpoch)
229
+ : 0
230
+ );
231
+
215
232
  await db.init();
216
233
 
217
234
  // Prune hot db repos
@@ -19,16 +19,15 @@ import {SyncOptions, defaultSyncOptions} from "../sync/options.js";
19
19
 
20
20
  // Re-export so the CLI doesn't need to depend on lodestar-api
21
21
  export {allNamespaces} from "../api/rest/index.js";
22
-
22
+ export type {EngineApiMode};
23
23
  // Re-export to use as default values in CLI args
24
24
  export {
25
- defaultExecutionEngineHttpOpts,
26
- defaultExecutionBuilderHttpOpts,
27
- engineApiModes,
28
25
  ArchiveMode,
29
26
  DEFAULT_ARCHIVE_MODE,
27
+ defaultExecutionBuilderHttpOpts,
28
+ defaultExecutionEngineHttpOpts,
29
+ engineApiModes,
30
30
  };
31
- export type {EngineApiMode};
32
31
 
33
32
  export interface IBeaconNodeOptions {
34
33
  api: ApiOptions;
@@ -7,6 +7,7 @@ import {IBeaconDb} from "../db/index.js";
7
7
  import {Metrics} from "../metrics/index.js";
8
8
  import {INetwork} from "../network/index.js";
9
9
  import {SyncChainDebugState} from "./range/chain.js";
10
+
10
11
  export type {SyncChainDebugState};
11
12
 
12
13
  export type SyncingStatus = routes.node.SyncingStatus;
@@ -300,10 +300,19 @@ export class BlockInputSync {
300
300
 
301
301
  private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
302
302
  try {
303
- this.addByPayloadInput(data.payloadInput, data.peer);
304
- this.triggerUnknownBlockSearch();
305
- this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
306
- this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
303
+ const isNewRoot = this.addByPayloadInput(data.payloadInput, data.peer);
304
+ const pendingPayload = this.pendingPayloads.get(data.payloadInput.blockRootHex);
305
+ if (
306
+ pendingPayload &&
307
+ pendingPayload.status !== PendingPayloadInputStatus.fetching &&
308
+ pendingPayload.status !== PendingPayloadInputStatus.processing
309
+ ) {
310
+ this.triggerUnknownBlockSearch();
311
+ }
312
+ if (isNewRoot) {
313
+ this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
314
+ this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
315
+ }
307
316
  } catch (e) {
308
317
  this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e as Error);
309
318
  }
@@ -599,43 +608,44 @@ export class BlockInputSync {
599
608
  return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
600
609
  }
601
610
 
602
- private addByPayloadInput = (
603
- payloadInput: PayloadEnvelopeInput,
604
- peerIdStr?: PeerIdStr,
605
- envelope?: gloas.SignedExecutionPayloadEnvelope
606
- ): void => {
611
+ private addByPayloadInput = (payloadInput: PayloadEnvelopeInput, peerIdStr?: PeerIdStr): boolean => {
607
612
  if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
608
613
  this.logger.debug("Ignoring payload input for known bad block root", {
609
614
  slot: payloadInput.slot,
610
615
  root: payloadInput.blockRootHex,
611
616
  peerIdStr,
612
617
  });
613
- return;
618
+ return false;
614
619
  }
615
620
 
616
- const pendingPayload = this.toPendingPayloadInput(
617
- payloadInput,
618
- this.pendingPayloads.get(payloadInput.blockRootHex),
619
- envelope
620
- );
621
+ let pendingPayload = this.pendingPayloads.get(payloadInput.blockRootHex);
622
+ const isNewRoot = pendingPayload === undefined;
623
+ // Envelope-only entries must go through reconciliation to validate their speculative envelope.
624
+ if (
625
+ !pendingPayload ||
626
+ pendingPayload.status === PendingPayloadInputStatus.pending ||
627
+ pendingPayload.status === PendingPayloadInputStatus.downloaded
628
+ ) {
629
+ pendingPayload = this.toPendingPayloadInput(payloadInput, pendingPayload);
630
+ this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
631
+
632
+ this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
633
+ slot: payloadInput.slot,
634
+ root: payloadInput.blockRootHex,
635
+ delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
636
+ });
637
+ }
621
638
 
622
639
  if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
623
640
  pendingPayload.peerIdStrings.add(peerIdStr);
624
641
  }
625
642
 
626
- this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
627
-
628
- this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
629
- slot: payloadInput.slot,
630
- root: payloadInput.blockRootHex,
631
- delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
632
- });
633
-
634
643
  const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
635
644
  if (prunedItemCount > 0) {
636
645
  this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.capacity}, prunedItemCount);
637
646
  this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
638
647
  }
648
+ return isNewRoot;
639
649
  };
640
650
 
641
651
  private onPeerConnected = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
@@ -1267,6 +1277,9 @@ export class BlockInputSync {
1267
1277
  }
1268
1278
 
1269
1279
  const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
1280
+ if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
1281
+ return;
1282
+ }
1270
1283
  if (!payloadInput) {
1271
1284
  this.logger.debug("PayloadEnvelopeInput not yet reloadable for imported block, will retry", {
1272
1285
  root: rootHex,
@@ -1278,6 +1291,9 @@ export class BlockInputSync {
1278
1291
  const validationResult = await wrapError(
1279
1292
  validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
1280
1293
  );
1294
+ if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
1295
+ return;
1296
+ }
1281
1297
  if (validationResult.err) {
1282
1298
  this.logger.debug(
1283
1299
  "Pending payload envelope failed validation after block import, refetching by root",
@@ -1345,14 +1361,13 @@ export class BlockInputSync {
1345
1361
  payload.status = PendingPayloadInputStatus.fetching;
1346
1362
 
1347
1363
  const res = await wrapError(this.fetchPayloadInput(payload));
1364
+ if (this.pendingPayloads.get(rootHex) !== payload) {
1365
+ this.logger.verbose("Dropping downloaded payload, entry changed during fetch", logCtx);
1366
+ return;
1367
+ }
1348
1368
  if (!res.err) {
1349
1369
  const pendingPayload = res.result;
1350
1370
  const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
1351
- // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
1352
- if (!this.pendingPayloads.has(resultRootHex)) {
1353
- this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
1354
- return;
1355
- }
1356
1371
  this.pendingPayloads.set(resultRootHex, pendingPayload);
1357
1372
  const result = isPendingPayloadEnvelope(pendingPayload)
1358
1373
  ? DownloadResult.WaitingForBlock
@@ -1423,12 +1438,16 @@ export class BlockInputSync {
1423
1438
  this.logger.debug("Processing downloaded payload", logCtx);
1424
1439
 
1425
1440
  const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
1441
+ // The executionPayload event emitted during import usually removes the entry before this point
1426
1442
  if (!res.err) {
1427
1443
  this.logger.debug("Processed payload from unknown sync", logCtx);
1428
1444
  this.pendingPayloads.delete(rootHex);
1429
1445
  this.triggerUnknownBlockSearch();
1430
1446
  return;
1431
1447
  }
1448
+ if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
1449
+ return;
1450
+ }
1432
1451
 
1433
1452
  if (res.err instanceof PayloadError) {
1434
1453
  switch (res.err.type.code) {
@@ -69,7 +69,7 @@ export class JobItemQueue<Args extends any[], R> {
69
69
  if (this.jobs.length === 1 && this.opts.noYieldIfOneItem) {
70
70
  void this.runJob();
71
71
  } else if (this.runningJobs < this.opts.maxConcurrency) {
72
- callInNextEventLoop(this.runJob);
72
+ callInNextEventLoop(() => void this.runJob());
73
73
  }
74
74
  });
75
75
  }
@@ -854,7 +854,7 @@ export function getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized(
854
854
 
855
855
  const start = bidMsgStart + commitmentsRange.start;
856
856
  const end = bidMsgStart + commitmentsRange.end;
857
- return Math.round(((end > blockBytes.byteLength ? blockBytes.byteLength : end) - start) / commitmentSize);
857
+ return Math.round((Math.min(end, blockBytes.byteLength) - start) / commitmentSize);
858
858
  }
859
859
 
860
860
  const {SignedBeaconBlock, BeaconBlock, BeaconBlockBody, KZGCommitment} = ssz[forkName as ForkPostDeneb];
@@ -880,5 +880,5 @@ export function getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized(
880
880
  const end = messageRange.end + bodyRange.end + kzgCommitmentsRange.end;
881
881
  const start = messageRange.start + bodyRange.start + kzgCommitmentsRange.start;
882
882
 
883
- return Math.round(((end > blockBytes.byteLength ? blockBytes.byteLength : end) - start) / commitmentSize);
883
+ return Math.round((Math.min(end, blockBytes.byteLength) - start) / commitmentSize);
884
884
  }