@lodestar/beacon-node 1.42.0-dev.84f3b9f030 → 1.42.0-dev.89ddaaeed2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +11 -11
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/debug/index.js.map +1 -1
  5. package/lib/chain/ColumnReconstructionTracker.d.ts +2 -1
  6. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  7. package/lib/chain/ColumnReconstructionTracker.js +5 -5
  8. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  9. package/lib/chain/GetBlobsTracker.d.ts +2 -1
  10. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  11. package/lib/chain/GetBlobsTracker.js +14 -12
  12. package/lib/chain/GetBlobsTracker.js.map +1 -1
  13. package/lib/chain/blocks/blockInput/blockInput.d.ts +5 -5
  14. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  15. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  16. package/lib/chain/blocks/blockInput/types.d.ts +4 -4
  17. package/lib/chain/blocks/blockInput/types.d.ts.map +1 -1
  18. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  19. package/lib/chain/blocks/importBlock.js +13 -1
  20. package/lib/chain/blocks/importBlock.js.map +1 -1
  21. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +14 -6
  22. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  23. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +33 -2
  24. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  25. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +2 -1
  26. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  27. package/lib/chain/chain.d.ts +3 -2
  28. package/lib/chain/chain.d.ts.map +1 -1
  29. package/lib/chain/chain.js +55 -20
  30. package/lib/chain/chain.js.map +1 -1
  31. package/lib/chain/emitter.d.ts +29 -7
  32. package/lib/chain/emitter.d.ts.map +1 -1
  33. package/lib/chain/emitter.js +12 -3
  34. package/lib/chain/emitter.js.map +1 -1
  35. package/lib/chain/errors/dataColumnSidecarError.d.ts +31 -1
  36. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  37. package/lib/chain/errors/dataColumnSidecarError.js +7 -0
  38. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  39. package/lib/chain/interface.d.ts +4 -2
  40. package/lib/chain/interface.d.ts.map +1 -1
  41. package/lib/chain/seenCache/seenGossipBlockInput.d.ts +1 -1
  42. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  43. package/lib/chain/seenCache/seenGossipBlockInput.js +2 -2
  44. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  45. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +1 -1
  46. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  47. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +2 -2
  48. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  49. package/lib/chain/validation/dataColumnSidecar.d.ts +11 -4
  50. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  51. package/lib/chain/validation/dataColumnSidecar.js +184 -5
  52. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  53. package/lib/db/buckets.d.ts +2 -2
  54. package/lib/db/buckets.d.ts.map +1 -1
  55. package/lib/db/buckets.js +2 -2
  56. package/lib/db/buckets.js.map +1 -1
  57. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  58. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  59. package/lib/db/repositories/dataColumnSidecar.d.ts.map +1 -1
  60. package/lib/db/repositories/dataColumnSidecar.js +4 -2
  61. package/lib/db/repositories/dataColumnSidecar.js.map +1 -1
  62. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  63. package/lib/db/repositories/dataColumnSidecarArchive.js +4 -2
  64. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  65. package/lib/metrics/metrics/lodestar.d.ts +20 -0
  66. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  67. package/lib/metrics/metrics/lodestar.js +33 -0
  68. package/lib/metrics/metrics/lodestar.js.map +1 -1
  69. package/lib/network/interface.d.ts +3 -2
  70. package/lib/network/interface.d.ts.map +1 -1
  71. package/lib/network/libp2p/index.d.ts.map +1 -1
  72. package/lib/network/libp2p/index.js +19 -13
  73. package/lib/network/libp2p/index.js.map +1 -1
  74. package/lib/network/network.d.ts +3 -2
  75. package/lib/network/network.d.ts.map +1 -1
  76. package/lib/network/network.js +3 -0
  77. package/lib/network/network.js.map +1 -1
  78. package/lib/network/options.d.ts.map +1 -1
  79. package/lib/network/options.js +7 -2
  80. package/lib/network/options.js.map +1 -1
  81. package/lib/network/processor/extractSlotRootFns.d.ts +1 -1
  82. package/lib/network/processor/extractSlotRootFns.d.ts.map +1 -1
  83. package/lib/network/processor/extractSlotRootFns.js +25 -5
  84. package/lib/network/processor/extractSlotRootFns.js.map +1 -1
  85. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  86. package/lib/network/processor/gossipHandlers.js +242 -66
  87. package/lib/network/processor/gossipHandlers.js.map +1 -1
  88. package/lib/network/processor/index.d.ts +11 -1
  89. package/lib/network/processor/index.d.ts.map +1 -1
  90. package/lib/network/processor/index.js +232 -22
  91. package/lib/network/processor/index.js.map +1 -1
  92. package/lib/network/reqresp/types.d.ts +3 -3
  93. package/lib/network/reqresp/types.d.ts.map +1 -1
  94. package/lib/network/reqresp/types.js +9 -3
  95. package/lib/network/reqresp/types.js.map +1 -1
  96. package/lib/sync/unknownBlock.js +2 -2
  97. package/lib/sync/unknownBlock.js.map +1 -1
  98. package/lib/sync/utils/downloadByRange.d.ts +3 -3
  99. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  100. package/lib/sync/utils/downloadByRange.js +4 -2
  101. package/lib/sync/utils/downloadByRange.js.map +1 -1
  102. package/lib/sync/utils/downloadByRoot.d.ts +3 -3
  103. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  104. package/lib/sync/utils/downloadByRoot.js +10 -5
  105. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  106. package/lib/util/blobs.d.ts +3 -3
  107. package/lib/util/blobs.d.ts.map +1 -1
  108. package/lib/util/blobs.js +21 -10
  109. package/lib/util/blobs.js.map +1 -1
  110. package/lib/util/dataColumns.d.ts +18 -11
  111. package/lib/util/dataColumns.d.ts.map +1 -1
  112. package/lib/util/dataColumns.js +51 -17
  113. package/lib/util/dataColumns.js.map +1 -1
  114. package/lib/util/execution.d.ts +6 -2
  115. package/lib/util/execution.d.ts.map +1 -1
  116. package/lib/util/execution.js +49 -25
  117. package/lib/util/execution.js.map +1 -1
  118. package/lib/util/sszBytes.d.ts +25 -1
  119. package/lib/util/sszBytes.d.ts.map +1 -1
  120. package/lib/util/sszBytes.js +189 -2
  121. package/lib/util/sszBytes.js.map +1 -1
  122. package/package.json +15 -15
  123. package/src/api/impl/beacon/blocks/index.ts +17 -14
  124. package/src/api/impl/debug/index.ts +2 -2
  125. package/src/chain/ColumnReconstructionTracker.ts +6 -5
  126. package/src/chain/GetBlobsTracker.ts +14 -12
  127. package/src/chain/blocks/blockInput/blockInput.ts +8 -8
  128. package/src/chain/blocks/blockInput/types.ts +4 -4
  129. package/src/chain/blocks/importBlock.ts +18 -1
  130. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +53 -12
  131. package/src/chain/blocks/payloadEnvelopeInput/types.ts +2 -1
  132. package/src/chain/chain.ts +63 -24
  133. package/src/chain/emitter.ts +25 -7
  134. package/src/chain/errors/dataColumnSidecarError.ts +32 -1
  135. package/src/chain/interface.ts +4 -2
  136. package/src/chain/seenCache/seenGossipBlockInput.ts +2 -2
  137. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +2 -2
  138. package/src/chain/validation/dataColumnSidecar.ts +230 -7
  139. package/src/db/buckets.ts +2 -2
  140. package/src/db/repositories/dataColumnSidecar.ts +4 -2
  141. package/src/db/repositories/dataColumnSidecarArchive.ts +4 -2
  142. package/src/metrics/metrics/lodestar.ts +34 -0
  143. package/src/network/interface.ts +3 -2
  144. package/src/network/libp2p/index.ts +21 -15
  145. package/src/network/network.ts +7 -4
  146. package/src/network/options.ts +7 -2
  147. package/src/network/processor/extractSlotRootFns.ts +32 -6
  148. package/src/network/processor/gossipHandlers.ts +305 -79
  149. package/src/network/processor/index.ts +302 -22
  150. package/src/network/reqresp/types.ts +13 -5
  151. package/src/sync/unknownBlock.ts +3 -3
  152. package/src/sync/utils/downloadByRange.ts +9 -7
  153. package/src/sync/utils/downloadByRoot.ts +16 -12
  154. package/src/util/blobs.ts +35 -15
  155. package/src/util/dataColumns.ts +69 -25
  156. package/src/util/execution.ts +49 -30
  157. package/src/util/sszBytes.ts +245 -3
@@ -10,11 +10,13 @@ import {
10
10
  getBlockHeaderProposerSignatureSetByHeaderSlot,
11
11
  getBlockHeaderProposerSignatureSetByParentStateSlot,
12
12
  } from "@lodestar/state-transition";
13
- import {DataColumnSidecar, Root, Slot, SubnetID, fulu, ssz} from "@lodestar/types";
13
+ import {DataColumnSidecar, Root, Slot, SubnetID, fulu, gloas, ssz} from "@lodestar/types";
14
14
  import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils";
15
15
  import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
16
16
  import {Metrics} from "../../metrics/metrics.js";
17
+ import {getDataColumnSidecarSlot} from "../../util/dataColumns.js";
17
18
  import {kzg} from "../../util/kzg.js";
19
+ import {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
18
20
  import {
19
21
  DataColumnSidecarErrorCode,
20
22
  DataColumnSidecarGossipError,
@@ -26,7 +28,7 @@ import {RegenCaller} from "../regen/interface.js";
26
28
 
27
29
  // SPEC FUNCTION
28
30
  // https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/p2p-interface.md#data_column_sidecar_subnet_id
29
- export async function validateGossipDataColumnSidecar(
31
+ export async function validateGossipFuluDataColumnSidecar(
30
32
  chain: IBeaconChain,
31
33
  dataColumnSidecar: fulu.DataColumnSidecar,
32
34
  gossipSubnet: SubnetID,
@@ -36,7 +38,7 @@ export async function validateGossipDataColumnSidecar(
36
38
  const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
37
39
 
38
40
  // 1) [REJECT] The sidecar is valid as verified by verify_data_column_sidecar
39
- verifyDataColumnSidecar(chain.config, dataColumnSidecar);
41
+ verifyFuluDataColumnSidecar(chain.config, dataColumnSidecar);
40
42
 
41
43
  // 2) [REJECT] The sidecar is for the correct subnet -- i.e. compute_subnet_for_data_column_sidecar(sidecar.index) == subnet_id
42
44
  if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
@@ -203,11 +205,75 @@ export async function validateGossipDataColumnSidecar(
203
205
  // -- Handled in seenGossipBlockInput
204
206
  }
205
207
 
208
+ // SPEC FUNCTION
209
+ // https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/p2p-interface.md#data_column_sidecar_subnet_id
210
+ export async function validateGossipGloasDataColumnSidecar(
211
+ chain: IBeaconChain,
212
+ payloadInput: PayloadEnvelopeInput,
213
+ dataColumnSidecar: gloas.DataColumnSidecar,
214
+ gossipSubnet: SubnetID,
215
+ metrics: Metrics | null
216
+ ): Promise<void> {
217
+ const blockRootHex = toRootHex(dataColumnSidecar.beaconBlockRoot);
218
+ const block = chain.forkChoice.getBlockHexDefaultStatus(blockRootHex);
219
+
220
+ // [IGNORE] A valid block for the sidecar's `slot` has been seen.
221
+ if (block === null) {
222
+ throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
223
+ code: DataColumnSidecarErrorCode.BLOCK_UNKNOWN,
224
+ blockRoot: blockRootHex,
225
+ slot: dataColumnSidecar.slot,
226
+ });
227
+ }
228
+
229
+ // [REJECT] The sidecar slot matches the slot of the block with root beacon_block_root.
230
+ if (block.slot !== dataColumnSidecar.slot) {
231
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
232
+ code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
233
+ columnIndex: dataColumnSidecar.index,
234
+ expected: block.slot,
235
+ actual: dataColumnSidecar.slot,
236
+ });
237
+ }
238
+
239
+ // [REJECT] The sidecar must pass verify_data_column_sidecar against the block commitments
240
+ const kzgCommitments = payloadInput.getBlobKzgCommitments();
241
+ verifyGloasDataColumnSidecar(dataColumnSidecar, kzgCommitments);
242
+
243
+ // [REJECT] The sidecar must be on the correct subnet
244
+ if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
245
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
246
+ code: DataColumnSidecarErrorCode.INVALID_SUBNET,
247
+ columnIndex: dataColumnSidecar.index,
248
+ gossipSubnet,
249
+ });
250
+ }
251
+
252
+ // [REJECT] The sidecar kzg proofs must verify
253
+ const kzgProofTimer = metrics?.peerDas.dataColumnSidecarKzgProofsVerificationTime.startTimer();
254
+ try {
255
+ await verifyDataColumnSidecarKzgProofs(
256
+ kzgCommitments,
257
+ Array.from({length: dataColumnSidecar.column.length}, () => dataColumnSidecar.index),
258
+ dataColumnSidecar.column,
259
+ dataColumnSidecar.kzgProofs
260
+ );
261
+ } catch {
262
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
263
+ code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
264
+ slot: dataColumnSidecar.slot,
265
+ columnIndex: dataColumnSidecar.index,
266
+ });
267
+ } finally {
268
+ kzgProofTimer?.();
269
+ }
270
+ }
271
+
206
272
  /**
207
273
  * SPEC FUNCTION
208
274
  * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/p2p-interface.md#verify_data_column_sidecar
209
275
  */
210
- function verifyDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: fulu.DataColumnSidecar): void {
276
+ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: fulu.DataColumnSidecar): void {
211
277
  if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
212
278
  throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
213
279
  code: DataColumnSidecarErrorCode.INVALID_INDEX,
@@ -250,6 +316,41 @@ function verifyDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: ful
250
316
  }
251
317
  }
252
318
 
319
+ /**
320
+ * SPEC FUNCTION
321
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/p2p-interface.md#modified-verify_data_column_sidecar
322
+ */
323
+ function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar, kzgCommitments: Uint8Array[]): void {
324
+ const slot = getDataColumnSidecarSlot(dataColumnSidecar);
325
+ if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
326
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
327
+ code: DataColumnSidecarErrorCode.INVALID_INDEX,
328
+ slot,
329
+ columnIndex: dataColumnSidecar.index,
330
+ });
331
+ }
332
+
333
+ if (dataColumnSidecar.column.length === 0) {
334
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
335
+ code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
336
+ slot,
337
+ columnIndex: dataColumnSidecar.index,
338
+ });
339
+ }
340
+
341
+ if (
342
+ dataColumnSidecar.column.length !== kzgCommitments.length ||
343
+ dataColumnSidecar.column.length !== dataColumnSidecar.kzgProofs.length
344
+ ) {
345
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
346
+ code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS,
347
+ columnLength: dataColumnSidecar.column.length,
348
+ commitmentsLength: kzgCommitments.length,
349
+ proofsLength: dataColumnSidecar.kzgProofs.length,
350
+ });
351
+ }
352
+ }
353
+
253
354
  /**
254
355
  * SPEC FUNCTION
255
356
  * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/p2p-interface.md#verify_data_column_sidecar_kzg_proofs
@@ -287,19 +388,19 @@ export function verifyDataColumnSidecarInclusionProof(dataColumnSidecar: fulu.Da
287
388
  }
288
389
 
289
390
  /**
290
- * Validate a subset of data column sidecars in a block
391
+ * Validate a subset of fulu data column sidecars against a block
291
392
  *
292
393
  * Requires the block to be known to the node
293
394
  *
294
395
  * NOTE: chain is optional to skip signature verification. Helpful for testing purposes and so that can control whether
295
396
  * signature gets checked depending on the reqresp method that is being checked
296
397
  */
297
- export async function validateBlockDataColumnSidecars(
398
+ export async function validateFuluBlockDataColumnSidecars(
298
399
  chain: IBeaconChain | null,
299
400
  blockSlot: Slot,
300
401
  blockRoot: Root,
301
402
  blockBlobCount: number,
302
- dataColumnSidecars: fulu.DataColumnSidecars,
403
+ dataColumnSidecars: fulu.DataColumnSidecar[],
303
404
  metrics?: BeaconMetrics["peerDas"] | null
304
405
  ): Promise<void> {
305
406
  metrics?.dataColumnSidecarProcessingRequests.inc(dataColumnSidecars.length);
@@ -467,6 +568,128 @@ export async function validateBlockDataColumnSidecars(
467
568
  "DataColumnSidecar has invalid KZG proof batch"
468
569
  );
469
570
  }
571
+
572
+ metrics?.dataColumnSidecarProcessingSuccesses.inc();
573
+ } finally {
574
+ verificationTimer?.();
575
+ }
576
+ }
577
+
578
+ /**
579
+ * Validate a subset of gloas data column sidecars against a block
580
+ * Gloas sidecars don't carry signed block headers, kzg commitments, or inclusion proofs
581
+ */
582
+ export async function validateGloasBlockDataColumnSidecars(
583
+ blockSlot: Slot,
584
+ blockRoot: Root,
585
+ blockKzgCommitments: Uint8Array[],
586
+ dataColumnSidecars: gloas.DataColumnSidecar[],
587
+ metrics?: BeaconMetrics["peerDas"] | null
588
+ ): Promise<void> {
589
+ metrics?.dataColumnSidecarProcessingRequests.inc(dataColumnSidecars.length);
590
+ const verificationTimer = metrics?.dataColumnSidecarGossipVerificationTime.startTimer();
591
+ try {
592
+ if (dataColumnSidecars.length === 0) {
593
+ return;
594
+ }
595
+
596
+ if (blockKzgCommitments.length === 0) {
597
+ throw new DataColumnSidecarValidationError(
598
+ {
599
+ code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT,
600
+ slot: blockSlot,
601
+ expected: 0,
602
+ actual: dataColumnSidecars.length,
603
+ },
604
+ "Block has no blob commitments but data column sidecars were provided"
605
+ );
606
+ }
607
+
608
+ const commitments: Uint8Array[] = [];
609
+ const cellIndices: number[] = [];
610
+ const cells: Uint8Array[] = [];
611
+ const proofs: Uint8Array[] = [];
612
+ for (const columnSidecar of dataColumnSidecars) {
613
+ if (columnSidecar.slot !== blockSlot) {
614
+ throw new DataColumnSidecarValidationError({
615
+ code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
616
+ columnIndex: columnSidecar.index,
617
+ expected: blockSlot,
618
+ actual: columnSidecar.slot,
619
+ });
620
+ }
621
+
622
+ if (!byteArrayEquals(columnSidecar.beaconBlockRoot, blockRoot)) {
623
+ throw new DataColumnSidecarValidationError({
624
+ code: DataColumnSidecarErrorCode.INCORRECT_BLOCK,
625
+ slot: blockSlot,
626
+ columnIndex: columnSidecar.index,
627
+ expected: toRootHex(blockRoot),
628
+ actual: toRootHex(columnSidecar.beaconBlockRoot),
629
+ });
630
+ }
631
+
632
+ if (columnSidecar.index >= NUMBER_OF_COLUMNS) {
633
+ throw new DataColumnSidecarValidationError(
634
+ {
635
+ code: DataColumnSidecarErrorCode.INVALID_INDEX,
636
+ slot: blockSlot,
637
+ columnIndex: columnSidecar.index,
638
+ },
639
+ "DataColumnSidecar has invalid index"
640
+ );
641
+ }
642
+
643
+ if (columnSidecar.column.length !== blockKzgCommitments.length) {
644
+ throw new DataColumnSidecarValidationError({
645
+ code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT,
646
+ slot: blockSlot,
647
+ columnIndex: columnSidecar.index,
648
+ expected: blockKzgCommitments.length,
649
+ actual: columnSidecar.column.length,
650
+ });
651
+ }
652
+
653
+ if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) {
654
+ throw new DataColumnSidecarValidationError({
655
+ code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT,
656
+ slot: blockSlot,
657
+ columnIndex: columnSidecar.index,
658
+ expected: columnSidecar.column.length,
659
+ actual: columnSidecar.kzgProofs.length,
660
+ });
661
+ }
662
+
663
+ commitments.push(...blockKzgCommitments);
664
+ cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index));
665
+ cells.push(...columnSidecar.column);
666
+ proofs.push(...columnSidecar.kzgProofs);
667
+ }
668
+
669
+ let reason: string | undefined;
670
+ // batch verification for the cases: downloadByRange and downloadByRoot
671
+ const kzgVerificationTimer = metrics?.kzgVerificationDataColumnBatchTime.startTimer();
672
+ try {
673
+ const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs);
674
+ if (!valid) {
675
+ reason = "Invalid KZG proof batch";
676
+ }
677
+ } catch (e) {
678
+ reason = (e as Error).message;
679
+ } finally {
680
+ kzgVerificationTimer?.();
681
+ }
682
+ if (reason !== undefined) {
683
+ throw new DataColumnSidecarValidationError(
684
+ {
685
+ code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH,
686
+ slot: blockSlot,
687
+ reason,
688
+ },
689
+ "DataColumnSidecar has invalid KZG proof batch"
690
+ );
691
+ }
692
+
470
693
  metrics?.dataColumnSidecarProcessingSuccesses.inc();
471
694
  } finally {
472
695
  verificationTimer?.();
package/src/db/buckets.ts CHANGED
@@ -67,8 +67,8 @@ export enum Bucket {
67
67
  // lightClient_bestLightClientUpdate = 55, // SyncPeriod -> LightClientUpdate // DEPRECATED on v1.5.0
68
68
  lightClient_bestLightClientUpdate = 56, // SyncPeriod -> [Slot, LightClientUpdate]
69
69
 
70
- fulu_dataColumnSidecars = 57, // FULU BeaconBlockRoot -> DataColumnSidecars
71
- fulu_dataColumnSidecarsArchive = 58, // FULU BeaconBlockSlot -> DataColumnSidecars
70
+ allForks_dataColumnSidecars = 57, // BeaconBlockRoot -> DataColumnSidecars
71
+ allForks_dataColumnSidecarsArchive = 58, // BeaconBlockSlot -> DataColumnSidecars
72
72
 
73
73
  gloas_executionPayloadEnvelope = 59, // GLOAS BeaconBlockRoot -> SignedExecutionPayloadEnvelope
74
74
  gloas_executionPayloadEnvelopeArchive = 60, // GLOAS Slot -> SignedExecutionPayloadEnvelope
@@ -18,8 +18,10 @@ type BlockRoot = Root;
18
18
  */
19
19
  export class DataColumnSidecarRepository extends PrefixedRepository<BlockRoot, ColumnIndex, DataColumnSidecar> {
20
20
  constructor(config: ChainForkConfig, db: Db) {
21
- const bucket = Bucket.fulu_dataColumnSidecars;
22
- super(config, db, bucket, ssz.fulu.DataColumnSidecar, getBucketNameByValue(bucket));
21
+ const bucket = Bucket.allForks_dataColumnSidecars;
22
+ // Type won't be used since we select it dynamically based on fork
23
+ const type = ssz.fulu.DataColumnSidecar;
24
+ super(config, db, bucket, type, getBucketNameByValue(bucket));
23
25
  }
24
26
 
25
27
  /**
@@ -16,8 +16,10 @@ const SLOT_BYTE_SIZE = 8;
16
16
  */
17
17
  export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
18
18
  constructor(config: ChainForkConfig, db: Db) {
19
- const bucket = Bucket.fulu_dataColumnSidecarsArchive;
20
- super(config, db, bucket, ssz.fulu.DataColumnSidecar, getBucketNameByValue(bucket));
19
+ const bucket = Bucket.allForks_dataColumnSidecarsArchive;
20
+ // Type won't be used since we select it dynamically based on fork
21
+ const type = ssz.fulu.DataColumnSidecar;
22
+ super(config, db, bucket, type, getBucketNameByValue(bucket));
21
23
  }
22
24
 
23
25
  /**
@@ -1746,6 +1746,40 @@ export function createLodestarMetrics(
1746
1746
  }),
1747
1747
  },
1748
1748
 
1749
+ // some gossip messages need to wait for payload to be processed before they can be processed
1750
+ awaitingPayloadGossipMessages: {
1751
+ queue: register.gauge<{topic: GossipType}>({
1752
+ name: "lodestar_awaiting_payload_gossip_messages_total",
1753
+ help: "Total number of gossip messages waiting for payload to be processed",
1754
+ labelNames: ["topic"],
1755
+ }),
1756
+ countPerSlot: register.gauge({
1757
+ name: "lodestar_awaiting_payload_gossip_messages_per_slot_total",
1758
+ help: "Total number of gossip messages waiting for payload to be processed per slot",
1759
+ }),
1760
+ resolve: register.gauge<{topic: GossipType}>({
1761
+ name: "lodestar_awaiting_payload_gossip_messages_resolve_total",
1762
+ help: "Total number of gossip messages are reprocessed",
1763
+ labelNames: ["topic"],
1764
+ }),
1765
+ waitSecBeforeResolve: register.gauge<{topic: GossipType}>({
1766
+ name: "lodestar_awaiting_payload_gossip_messages_wait_time_resolve_seconds",
1767
+ help: "Time to wait for unknown payload in seconds",
1768
+ labelNames: ["topic"],
1769
+ }),
1770
+ // having 2 labels here is not great for performance, however it's rarely happening and having the reason label is important for debugging
1771
+ reject: register.gauge<{reason: ReprocessRejectReason; topic: GossipType}>({
1772
+ name: "lodestar_awaiting_payload_gossip_messages_reject_total",
1773
+ help: "Total number of gossip messages are rejected to reprocess",
1774
+ labelNames: ["reason", "topic"],
1775
+ }),
1776
+ waitSecBeforeReject: register.gauge<{reason: ReprocessRejectReason; topic: GossipType}>({
1777
+ name: "lodestar_awaiting_payload_gossip_messages_wait_time_reject_seconds",
1778
+ help: "Time to wait for unknown payload before being rejected",
1779
+ labelNames: ["reason", "topic"],
1780
+ }),
1781
+ },
1782
+
1749
1783
  lightclientServer: {
1750
1784
  onSyncAggregate: register.gauge<{event: string}>({
1751
1785
  name: "lodestar_lightclient_server_on_sync_aggregate_event_total",
@@ -74,6 +74,7 @@ export interface INetwork extends INetworkCorePublic {
74
74
  shouldAggregate(subnet: SubnetID, slot: Slot): boolean;
75
75
  reStatusPeers(peers: PeerIdStr[]): Promise<void>;
76
76
  searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
77
+ searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
77
78
  // ReqResp
78
79
  sendBeaconBlocksByRange(peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest): Promise<SignedBeaconBlock[]>;
79
80
  sendBeaconBlocksByRoot(peerId: PeerIdStr, request: BeaconBlocksByRootRequest): Promise<SignedBeaconBlock[]>;
@@ -82,11 +83,11 @@ export interface INetwork extends INetworkCorePublic {
82
83
  sendDataColumnSidecarsByRange(
83
84
  peerId: PeerIdStr,
84
85
  request: fulu.DataColumnSidecarsByRangeRequest
85
- ): Promise<fulu.DataColumnSidecar[]>;
86
+ ): Promise<DataColumnSidecar[]>;
86
87
  sendDataColumnSidecarsByRoot(
87
88
  peerId: PeerIdStr,
88
89
  request: DataColumnSidecarsByRootRequest
89
- ): Promise<fulu.DataColumnSidecar[]>;
90
+ ): Promise<DataColumnSidecar[]>;
90
91
  sendExecutionPayloadEnvelopesByRange(
91
92
  peerId: PeerIdStr,
92
93
  request: gloas.ExecutionPayloadEnvelopesByRangeRequest
@@ -44,6 +44,8 @@ export async function createNodeJsLibp2p(
44
44
  ): Promise<Libp2p> {
45
45
  const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs;
46
46
  const disconnectThreshold = networkOpts.disconnectThreshold ?? defaultNetworkOptions.disconnectThreshold;
47
+ const tcpEnabled = networkOpts.tcp ?? defaultNetworkOptions.tcp;
48
+ const quicEnabled = networkOpts.quic ?? defaultNetworkOptions.quic;
47
49
  const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts;
48
50
 
49
51
  let datastore: undefined | Eth2PeerDataStore = undefined;
@@ -58,7 +60,7 @@ export async function createNodeJsLibp2p(
58
60
  ...(networkOpts.bootMultiaddrs ?? defaultNetworkOptions.bootMultiaddrs ?? []),
59
61
  // Append discv5.bootEnrs to bootMultiaddrs if requested
60
62
  ...(networkOpts.connectToDiscv5Bootnodes
61
- ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? [], networkOpts.quic)
63
+ ? await getDiscv5Multiaddrs(networkOpts.discv5?.bootEnrs ?? [], quicEnabled)
62
64
  : []),
63
65
  ];
64
66
 
@@ -71,7 +73,7 @@ export async function createNodeJsLibp2p(
71
73
  }
72
74
  }
73
75
  const transports: Libp2pInit["transports"] = [];
74
- if (networkOpts.tcp ?? true) {
76
+ if (tcpEnabled) {
75
77
  transports.unshift(
76
78
  tcp({
77
79
  // Reject connections when the server's connection count gets high
@@ -87,22 +89,26 @@ export async function createNodeJsLibp2p(
87
89
  })
88
90
  );
89
91
  }
90
- if (networkOpts.quic) {
92
+ if (quicEnabled) {
91
93
  const quicMultiaddrs = localMultiaddrs.filter((ma) => ma.includes("/quic-v1"));
92
94
  const hasIpv4Quic = quicMultiaddrs.some((ma) => ma.includes("/ip4/"));
93
95
  const hasIpv6Quic = quicMultiaddrs.some((ma) => ma.includes("/ip6/"));
94
- transports.unshift(
95
- quic({
96
- handshakeTimeout: 5_000,
97
- maxIdleTimeout: 10_000,
98
- keepAliveInterval: 5_000,
99
- maxConcurrentStreamLimit: 256,
100
- maxStreamData: 10_000_000,
101
- maxConnectionData: 15_000_000,
102
- ipv4: hasIpv4Quic,
103
- ipv6: hasIpv6Quic,
104
- })
105
- );
96
+ // Only add QUIC transport if at least one QUIC listen address is configured,
97
+ // otherwise the transport constructor will throw
98
+ if (hasIpv4Quic || hasIpv6Quic) {
99
+ transports.unshift(
100
+ quic({
101
+ handshakeTimeout: 5_000,
102
+ maxIdleTimeout: 10_000,
103
+ keepAliveInterval: 5_000,
104
+ maxConcurrentStreamLimit: 256,
105
+ maxStreamData: 10_000_000,
106
+ maxConnectionData: 15_000_000,
107
+ ipv4: hasIpv4Quic,
108
+ ipv6: hasIpv6Quic,
109
+ })
110
+ );
111
+ }
106
112
  }
107
113
 
108
114
  const noiseCrypto = {
@@ -11,7 +11,6 @@ import {computeEpochAtSlot} from "@lodestar/state-transition";
11
11
  import {
12
12
  AttesterSlashing,
13
13
  DataColumnSidecar,
14
- DataColumnSidecars,
15
14
  LightClientBootstrap,
16
15
  LightClientFinalityUpdate,
17
16
  LightClientOptimisticUpdate,
@@ -286,6 +285,10 @@ export class Network implements INetwork {
286
285
  this.networkProcessor.searchUnknownBlock(slotRoot, source, peer);
287
286
  }
288
287
 
288
+ searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
289
+ this.networkProcessor.searchUnknownEnvelope(slotRoot, source, peer);
290
+ }
291
+
289
292
  async reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): Promise<void> {
290
293
  return this.core.reportPeer(peer, action, actionName);
291
294
  }
@@ -621,7 +624,7 @@ export class Network implements INetwork {
621
624
  async sendDataColumnSidecarsByRange(
622
625
  peerId: PeerIdStr,
623
626
  request: fulu.DataColumnSidecarsByRangeRequest
624
- ): Promise<fulu.DataColumnSidecar[]> {
627
+ ): Promise<DataColumnSidecar[]> {
625
628
  return collectMaxResponseTyped(
626
629
  this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRange, [Version.V1], request),
627
630
  request.count * request.columns.length,
@@ -632,7 +635,7 @@ export class Network implements INetwork {
632
635
  async sendDataColumnSidecarsByRoot(
633
636
  peerId: PeerIdStr,
634
637
  request: DataColumnSidecarsByRootRequest
635
- ): Promise<fulu.DataColumnSidecar[]> {
638
+ ): Promise<DataColumnSidecar[]> {
636
639
  return collectMaxResponseTyped(
637
640
  this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRoot, [Version.V1], request),
638
641
  request.reduce((total, {columns}) => total + columns.length, 0),
@@ -812,7 +815,7 @@ export class Network implements INetwork {
812
815
  this.core.setTargetGroupCount(count);
813
816
  };
814
817
 
815
- private onPublishDataColumns = (sidecars: DataColumnSidecars): Promise<number[]> => {
818
+ private onPublishDataColumns = (sidecars: DataColumnSidecar[]): Promise<number[]> => {
816
819
  return promiseAllMaybeAsync(sidecars.map((sidecar) => () => this.publishDataColumnSidecar(sidecar)));
817
820
  };
818
821
 
@@ -55,7 +55,12 @@ export const defaultNetworkOptions: NetworkOptions = {
55
55
  maxPeers: 210, // Allow some room above targetPeers for new inbound peers
56
56
  targetPeers: 200,
57
57
  // In CLI usage this is typically overridden; when unset it serves as a fallback default (e.g. programmatic usage/tests)
58
- localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000", "/ip6/::/tcp/9000"],
58
+ localMultiaddrs: [
59
+ "/ip4/0.0.0.0/udp/9001/quic-v1",
60
+ "/ip6/::/udp/9001/quic-v1",
61
+ "/ip4/0.0.0.0/tcp/9000",
62
+ "/ip6/::/tcp/9000",
63
+ ],
59
64
  bootMultiaddrs: [],
60
65
  /** disabled by default */
61
66
  discv5: null,
@@ -69,7 +74,7 @@ export const defaultNetworkOptions: NetworkOptions = {
69
74
  slotsToSubscribeBeforeAggregatorDuty: 2,
70
75
  // This will enable the light client server by default
71
76
  disableLightClientServer: false,
72
- quic: false,
77
+ quic: true,
73
78
  tcp: true,
74
79
  // specific option for fulu
75
80
  // - this is the same to TARGET_SUBNET_PEERS
@@ -1,23 +1,25 @@
1
- import {ForkName, isForkPostGloas} from "@lodestar/params";
1
+ import {ForkName, ForkSeq} from "@lodestar/params";
2
2
  import {SlotOptionalRoot, SlotRootHex} from "@lodestar/types";
3
3
  import {
4
4
  getBeaconBlockRootFromDataColumnSidecarSerialized,
5
- getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized,
6
5
  getBlockRootFromBeaconAttestationSerialized,
6
+ getBlockRootFromPayloadAttestationMessageSerialized,
7
7
  getBlockRootFromSignedAggregateAndProofSerialized,
8
8
  getSlotFromBeaconAttestationSerialized,
9
9
  getSlotFromBlobSidecarSerialized,
10
10
  getSlotFromDataColumnSidecarSerialized,
11
11
  getSlotFromExecutionPayloadEnvelopeSerialized,
12
+ getSlotFromPayloadAttestationMessageSerialized,
12
13
  getSlotFromSignedAggregateAndProofSerialized,
13
14
  getSlotFromSignedBeaconBlockSerialized,
15
+ getSlotFromSignedExecutionPayloadBidSerialized,
14
16
  } from "../../util/sszBytes.js";
15
17
  import {GossipType} from "../gossip/index.js";
16
18
  import {ExtractSlotRootFns} from "./types.js";
17
19
 
18
20
  /**
19
21
  * Extract the slot and block root of a gossip message form serialized data.
20
- * Not applicable for all topics.
22
+ * Only do it for messages that have a slot and block root, and we want to await the block if the block root is not known.
21
23
  */
22
24
  export function createExtractBlockSlotRootFns(): ExtractSlotRootFns {
23
25
  return {
@@ -57,21 +59,45 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns {
57
59
  },
58
60
  [GossipType.data_column_sidecar]: (data: Uint8Array, fork: ForkName): SlotOptionalRoot | null => {
59
61
  const slot = getSlotFromDataColumnSidecarSerialized(data, fork);
62
+
60
63
  if (slot === null) {
61
64
  return null;
62
65
  }
63
66
 
64
- const root = isForkPostGloas(fork) ? getBeaconBlockRootFromDataColumnSidecarSerialized(data) : null;
67
+ if (ForkSeq[fork] < ForkSeq.gloas) {
68
+ return {slot};
69
+ }
70
+
71
+ const root = getBeaconBlockRootFromDataColumnSidecarSerialized(data);
72
+ // null root means the message is invalid here and will be ignored in gossip handler later
73
+ // returning the slot here helps check the earliest permissable slot in the network processor
65
74
  return root !== null ? {slot, root} : {slot};
66
75
  },
67
- [GossipType.execution_payload]: (data: Uint8Array): SlotRootHex | null => {
76
+ [GossipType.execution_payload]: (data: Uint8Array): SlotOptionalRoot | null => {
68
77
  const slot = getSlotFromExecutionPayloadEnvelopeSerialized(data);
69
- const root = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data);
78
+ // Do not extract the root here; the network processor will extract it in the 2nd round to trigger block search without awaiting.
79
+ if (slot === null) {
80
+ return null;
81
+ }
82
+ return {slot};
83
+ },
84
+ [GossipType.payload_attestation_message]: (data: Uint8Array): SlotRootHex | null => {
85
+ const slot = getSlotFromPayloadAttestationMessageSerialized(data);
86
+ const root = getBlockRootFromPayloadAttestationMessageSerialized(data);
70
87
 
71
88
  if (slot === null || root === null) {
72
89
  return null;
73
90
  }
74
91
  return {slot, root};
75
92
  },
93
+ [GossipType.execution_payload_bid]: (data: Uint8Array): SlotOptionalRoot | null => {
94
+ const slot = getSlotFromSignedExecutionPayloadBidSerialized(data);
95
+
96
+ if (slot === null) {
97
+ return null;
98
+ }
99
+
100
+ return {slot};
101
+ },
76
102
  };
77
103
  }