@lodestar/beacon-node 1.42.0-dev.8961934298 → 1.42.0-dev.8b81ee6c3e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +9 -9
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/debug/index.js.map +1 -1
  5. package/lib/chain/ColumnReconstructionTracker.d.ts +2 -1
  6. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  7. package/lib/chain/ColumnReconstructionTracker.js +5 -5
  8. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  9. package/lib/chain/GetBlobsTracker.d.ts +2 -1
  10. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  11. package/lib/chain/GetBlobsTracker.js +14 -12
  12. package/lib/chain/GetBlobsTracker.js.map +1 -1
  13. package/lib/chain/blocks/blockInput/blockInput.d.ts +5 -5
  14. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  15. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  16. package/lib/chain/blocks/blockInput/types.d.ts +4 -4
  17. package/lib/chain/blocks/blockInput/types.d.ts.map +1 -1
  18. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  19. package/lib/chain/blocks/importBlock.js +13 -1
  20. package/lib/chain/blocks/importBlock.js.map +1 -1
  21. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +14 -6
  22. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  23. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +33 -2
  24. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  25. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +2 -1
  26. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  27. package/lib/chain/chain.d.ts +2 -2
  28. package/lib/chain/chain.d.ts.map +1 -1
  29. package/lib/chain/chain.js +51 -19
  30. package/lib/chain/chain.js.map +1 -1
  31. package/lib/chain/emitter.d.ts +2 -2
  32. package/lib/chain/emitter.d.ts.map +1 -1
  33. package/lib/chain/errors/dataColumnSidecarError.d.ts +31 -1
  34. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  35. package/lib/chain/errors/dataColumnSidecarError.js +7 -0
  36. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  37. package/lib/chain/interface.d.ts +2 -2
  38. package/lib/chain/interface.d.ts.map +1 -1
  39. package/lib/chain/validation/dataColumnSidecar.d.ts +11 -4
  40. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  41. package/lib/chain/validation/dataColumnSidecar.js +184 -5
  42. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  43. package/lib/db/buckets.d.ts +2 -2
  44. package/lib/db/buckets.d.ts.map +1 -1
  45. package/lib/db/buckets.js +2 -2
  46. package/lib/db/buckets.js.map +1 -1
  47. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  48. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  49. package/lib/db/repositories/dataColumnSidecar.d.ts.map +1 -1
  50. package/lib/db/repositories/dataColumnSidecar.js +4 -2
  51. package/lib/db/repositories/dataColumnSidecar.js.map +1 -1
  52. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  53. package/lib/db/repositories/dataColumnSidecarArchive.js +4 -2
  54. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  55. package/lib/network/gossip/topic.d.ts +727 -0
  56. package/lib/network/gossip/topic.d.ts.map +1 -1
  57. package/lib/network/interface.d.ts +2 -2
  58. package/lib/network/interface.d.ts.map +1 -1
  59. package/lib/network/network.d.ts +2 -2
  60. package/lib/network/network.d.ts.map +1 -1
  61. package/lib/network/network.js.map +1 -1
  62. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  63. package/lib/network/processor/gossipHandlers.js +204 -54
  64. package/lib/network/processor/gossipHandlers.js.map +1 -1
  65. package/lib/network/reqresp/types.d.ts +3 -3
  66. package/lib/network/reqresp/types.d.ts.map +1 -1
  67. package/lib/network/reqresp/types.js +9 -3
  68. package/lib/network/reqresp/types.js.map +1 -1
  69. package/lib/sync/utils/downloadByRange.d.ts +3 -3
  70. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  71. package/lib/sync/utils/downloadByRange.js +4 -2
  72. package/lib/sync/utils/downloadByRange.js.map +1 -1
  73. package/lib/sync/utils/downloadByRoot.d.ts +3 -3
  74. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  75. package/lib/sync/utils/downloadByRoot.js +10 -5
  76. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  77. package/lib/util/blobs.d.ts +3 -3
  78. package/lib/util/blobs.d.ts.map +1 -1
  79. package/lib/util/blobs.js +21 -10
  80. package/lib/util/blobs.js.map +1 -1
  81. package/lib/util/dataColumns.d.ts +18 -11
  82. package/lib/util/dataColumns.d.ts.map +1 -1
  83. package/lib/util/dataColumns.js +51 -17
  84. package/lib/util/dataColumns.js.map +1 -1
  85. package/lib/util/execution.d.ts +6 -2
  86. package/lib/util/execution.d.ts.map +1 -1
  87. package/lib/util/execution.js +49 -25
  88. package/lib/util/execution.js.map +1 -1
  89. package/lib/util/sszBytes.d.ts.map +1 -1
  90. package/lib/util/sszBytes.js +34 -1
  91. package/lib/util/sszBytes.js.map +1 -1
  92. package/package.json +15 -15
  93. package/src/api/impl/beacon/blocks/index.ts +15 -12
  94. package/src/api/impl/debug/index.ts +2 -2
  95. package/src/chain/ColumnReconstructionTracker.ts +6 -5
  96. package/src/chain/GetBlobsTracker.ts +14 -12
  97. package/src/chain/blocks/blockInput/blockInput.ts +8 -8
  98. package/src/chain/blocks/blockInput/types.ts +4 -4
  99. package/src/chain/blocks/importBlock.ts +18 -1
  100. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +53 -12
  101. package/src/chain/blocks/payloadEnvelopeInput/types.ts +2 -1
  102. package/src/chain/chain.ts +58 -23
  103. package/src/chain/emitter.ts +2 -2
  104. package/src/chain/errors/dataColumnSidecarError.ts +32 -1
  105. package/src/chain/interface.ts +2 -2
  106. package/src/chain/validation/dataColumnSidecar.ts +230 -7
  107. package/src/db/buckets.ts +2 -2
  108. package/src/db/repositories/dataColumnSidecar.ts +4 -2
  109. package/src/db/repositories/dataColumnSidecarArchive.ts +4 -2
  110. package/src/network/interface.ts +2 -2
  111. package/src/network/network.ts +3 -4
  112. package/src/network/processor/gossipHandlers.ts +261 -67
  113. package/src/network/reqresp/types.ts +13 -5
  114. package/src/sync/utils/downloadByRange.ts +9 -7
  115. package/src/sync/utils/downloadByRoot.ts +16 -12
  116. package/src/util/blobs.ts +35 -15
  117. package/src/util/dataColumns.ts +69 -25
  118. package/src/util/execution.ts +49 -30
  119. package/src/util/sszBytes.ts +42 -2
@@ -1,13 +1,16 @@
1
1
  import {routes} from "@lodestar/api";
2
2
  import {BeaconConfig, ChainForkConfig} from "@lodestar/config";
3
+ import {PayloadStatus} from "@lodestar/fork-choice";
3
4
  import {
4
5
  ForkName,
5
6
  ForkPostDeneb,
6
7
  ForkPostElectra,
8
+ ForkPostGloas,
7
9
  ForkPreElectra,
8
10
  ForkSeq,
9
11
  NUMBER_OF_COLUMNS,
10
12
  isForkPostElectra,
13
+ isForkPostGloas,
11
14
  } from "@lodestar/params";
12
15
  import {computeTimeAtSlot} from "@lodestar/state-transition";
13
16
  import {
@@ -19,6 +22,8 @@ import {
19
22
  UintNum64,
20
23
  deneb,
21
24
  fulu,
25
+ gloas,
26
+ isGloasDataColumnSidecar,
22
27
  ssz,
23
28
  sszTypesFor,
24
29
  } from "@lodestar/types";
@@ -30,7 +35,7 @@ import {
30
35
  IBlockInput,
31
36
  isBlockInputColumns,
32
37
  } from "../../chain/blocks/blockInput/index.js";
33
- import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
38
+ import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
34
39
  import {BlobSidecarValidation} from "../../chain/blocks/types.js";
35
40
  import {ChainEvent} from "../../chain/emitter.js";
36
41
  import {
@@ -51,7 +56,10 @@ import {
51
56
  } from "../../chain/errors/index.js";
52
57
  import {IBeaconChain} from "../../chain/interface.js";
53
58
  import {validateGossipBlobSidecar} from "../../chain/validation/blobSidecar.js";
54
- import {validateGossipDataColumnSidecar} from "../../chain/validation/dataColumnSidecar.js";
59
+ import {
60
+ validateGossipFuluDataColumnSidecar,
61
+ validateGossipGloasDataColumnSidecar,
62
+ } from "../../chain/validation/dataColumnSidecar.js";
55
63
  import {validateGossipExecutionPayloadBid} from "../../chain/validation/executionPayloadBid.js";
56
64
  import {validateGossipExecutionPayloadEnvelope} from "../../chain/validation/executionPayloadEnvelope.js";
57
65
  import {
@@ -74,7 +82,7 @@ import {validateGossipPayloadAttestationMessage} from "../../chain/validation/pa
74
82
  import {OpSource} from "../../chain/validatorMonitor.js";
75
83
  import {Metrics} from "../../metrics/index.js";
76
84
  import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
77
- import {getBlobKzgCommitments} from "../../util/dataColumns.js";
85
+ import {getBlobKzgCommitments, getDataColumnSidecarSlot} from "../../util/dataColumns.js";
78
86
  import {INetworkCore} from "../core/index.js";
79
87
  import {NetworkEventBus} from "../events.js";
80
88
  import {
@@ -324,7 +332,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
324
332
  const blockInput = chain.seenBlockInputCache.get(blockRootHex);
325
333
  if (blockInput && isBlockInputColumns(blockInput) && blockInput.hasColumn(dataColumnSidecar.index)) {
326
334
  metrics?.peerDas.dataColumnSidecarProcessingSkip.inc();
327
- logger.debug("Already have column sidecar, skipping processing", {
335
+ logger.debug("Already have column sidecar in BlockInput, skipping processing", {
328
336
  ...blockInput.getLogMeta(),
329
337
  index: dataColumnSidecar.index,
330
338
  });
@@ -339,10 +347,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
339
347
  const verificationTimer = metrics?.peerDas.dataColumnSidecarGossipVerificationTime.startTimer();
340
348
 
341
349
  const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
350
+ const secFromSlot = chain.clock.secFromSlot(slot);
342
351
  const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
343
352
 
344
353
  try {
345
- await validateGossipDataColumnSidecar(chain, dataColumnSidecar, gossipSubnet, metrics);
354
+ await validateGossipFuluDataColumnSidecar(chain, dataColumnSidecar, gossipSubnet, metrics);
346
355
  const blockInput = chain.seenBlockInputCache.getByColumn({
347
356
  blockRootHex,
348
357
  columnSidecar: dataColumnSidecar,
@@ -372,6 +381,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
372
381
  currentSlot: chain.clock.currentSlot,
373
382
  peerId: peerIdStr,
374
383
  delaySec,
384
+ secFromSlot,
375
385
  gossipSubnet,
376
386
  columnIndex: dataColumnSidecar.index,
377
387
  recvToValLatency,
@@ -401,6 +411,131 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
401
411
  }
402
412
  }
403
413
 
414
+ async function validatePayloadDataColumn(
415
+ dataColumnSidecar: gloas.DataColumnSidecar,
416
+ gossipSubnet: SubnetID,
417
+ peerIdStr: string,
418
+ seenTimestampSec: number
419
+ ): Promise<PayloadEnvelopeInput> {
420
+ metrics?.peerDas.dataColumnSidecarProcessingRequests.inc();
421
+ const slot = dataColumnSidecar.slot;
422
+ const blockRootHex = toRootHex(dataColumnSidecar.beaconBlockRoot);
423
+
424
+ // check to see if payload has already been processed and PayloadEnvelopeInput has been deleted (column received via reqresp or other means)
425
+ if (chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.FULL) !== null) {
426
+ metrics?.peerDas.dataColumnSidecarProcessingSkip.inc();
427
+ logger.debug("Already processed payload for column sidecar, skipping processing", {
428
+ slot,
429
+ blockRoot: blockRootHex,
430
+ index: dataColumnSidecar.index,
431
+ });
432
+ throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
433
+ code: DataColumnSidecarErrorCode.ALREADY_KNOWN,
434
+ columnIndex: dataColumnSidecar.index,
435
+ slot,
436
+ });
437
+ }
438
+
439
+ const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
440
+
441
+ if (!payloadInput) {
442
+ // This should not happen for gossip because the network processor queues `data_column_sidecar`
443
+ // until block import creates the corresponding PayloadEnvelopeInput.
444
+ throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
445
+ code: DataColumnSidecarErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING,
446
+ slot,
447
+ blockRoot: blockRootHex,
448
+ });
449
+ }
450
+
451
+ // [IGNORE] The sidecar is the first sidecar for the tuple
452
+ // (sidecar.beacon_block_root, sidecar.index) with valid kzg proof.
453
+ if (payloadInput.hasColumn(dataColumnSidecar.index)) {
454
+ metrics?.peerDas.dataColumnSidecarProcessingSkip.inc();
455
+ logger.debug("Already have column sidecar in PayloadEnvelopeInput, skipping processing", {
456
+ ...payloadInput.getLogMeta(),
457
+ index: dataColumnSidecar.index,
458
+ });
459
+ throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
460
+ code: DataColumnSidecarErrorCode.ALREADY_KNOWN,
461
+ columnIndex: dataColumnSidecar.index,
462
+ slot,
463
+ });
464
+ }
465
+
466
+ const verificationTimer = metrics?.peerDas.dataColumnSidecarGossipVerificationTime.startTimer();
467
+
468
+ const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
469
+ const secFromSlot = chain.clock.secFromSlot(slot);
470
+ const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
471
+
472
+ try {
473
+ await validateGossipGloasDataColumnSidecar(chain, payloadInput, dataColumnSidecar, gossipSubnet, metrics);
474
+
475
+ const addedColumn = payloadInput.addColumn({
476
+ columnSidecar: dataColumnSidecar,
477
+ source: PayloadEnvelopeInputSource.gossip,
478
+ seenTimestampSec,
479
+ peerIdStr,
480
+ });
481
+
482
+ if (!addedColumn) {
483
+ metrics?.peerDas.dataColumnSidecarProcessingSkip.inc();
484
+ logger.debug("Already have column sidecar in PayloadEnvelopeInput, skipping processing", {
485
+ ...payloadInput.getLogMeta(),
486
+ index: dataColumnSidecar.index,
487
+ });
488
+ throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
489
+ code: DataColumnSidecarErrorCode.ALREADY_KNOWN,
490
+ columnIndex: dataColumnSidecar.index,
491
+ slot,
492
+ });
493
+ }
494
+
495
+ const recvToValidation = Date.now() / 1000 - seenTimestampSec;
496
+ const validationTime = recvToValidation - recvToValLatency;
497
+
498
+ metrics?.peerDas.dataColumnSidecarProcessingSuccesses.inc();
499
+ metrics?.gossipBlob.recvToValidation.observe(recvToValidation);
500
+ metrics?.gossipBlob.validationTime.observe(validationTime);
501
+
502
+ if (chain.emitter.listenerCount(routes.events.EventType.dataColumnSidecar)) {
503
+ chain.emitter.emit(routes.events.EventType.dataColumnSidecar, {
504
+ blockRoot: blockRootHex,
505
+ slot,
506
+ index: dataColumnSidecar.index,
507
+ });
508
+ }
509
+
510
+ logger.debug("Received gossip dataColumn", {
511
+ ...payloadInput.getLogMeta(),
512
+ currentSlot: chain.clock.currentSlot,
513
+ peerId: peerIdStr,
514
+ delaySec,
515
+ secFromSlot,
516
+ gossipSubnet,
517
+ columnIndex: dataColumnSidecar.index,
518
+ recvToValLatency,
519
+ recvToValidation,
520
+ validationTime,
521
+ });
522
+
523
+ return payloadInput;
524
+ } catch (e) {
525
+ if (e instanceof DataColumnSidecarGossipError && e.action === GossipAction.REJECT) {
526
+ chain.persistInvalidSszValue(
527
+ sszTypesFor(payloadInput.forkName as ForkPostGloas).DataColumnSidecar,
528
+ dataColumnSidecar,
529
+ `gossip_reject_slot_${slot}_index_${dataColumnSidecar.index}`
530
+ );
531
+ }
532
+
533
+ throw e;
534
+ } finally {
535
+ verificationTimer?.();
536
+ }
537
+ }
538
+
404
539
  function handleValidBeaconBlock(blockInput: IBlockInput, peerIdStr: string, seenTimestampSec: number): void {
405
540
  const signedBlock = blockInput.getBlock();
406
541
  const slot = signedBlock.message.slot;
@@ -554,78 +689,137 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
554
689
  peerIdStr,
555
690
  seenTimestampSec,
556
691
  }: GossipHandlerParamGeneric<GossipType.data_column_sidecar>) => {
692
+ const {fork} = topic.boundary;
557
693
  const {serializedData} = gossipData;
558
- // TODO GLOAS: handle gloas.DataColumnSidecar
559
- const dataColumnSidecar = sszDeserialize(topic, serializedData) as fulu.DataColumnSidecar;
560
- const dataColumnSlot = dataColumnSidecar.signedBlockHeader.message.slot;
694
+ const dataColumnSidecar = sszDeserialize(topic, serializedData);
695
+ const dataColumnSlot = getDataColumnSidecarSlot(dataColumnSidecar);
561
696
  const index = dataColumnSidecar.index;
562
-
563
- if (config.getForkSeq(dataColumnSlot) < ForkSeq.fulu) {
564
- throw new GossipActionError(GossipAction.REJECT, {code: "PRE_FULU_BLOCK"});
565
- }
566
697
  const delaySec = chain.clock.secFromSlot(dataColumnSlot, seenTimestampSec);
567
- const blockInput = await validateBeaconDataColumn(
568
- dataColumnSidecar,
569
- serializedData,
570
- topic.subnet,
571
- peerIdStr,
572
- seenTimestampSec
573
- );
574
- chain.serializedCache.set(dataColumnSidecar, serializedData);
575
- const blockInputMeta = blockInput.getLogMeta();
576
- const {receivedColumns} = blockInputMeta;
577
- // it's not helpful to track every single column received
578
- // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
579
- switch (receivedColumns) {
580
- case 1:
581
- case config.SAMPLES_PER_SLOT:
582
- case 2 * config.SAMPLES_PER_SLOT:
583
- case NUMBER_OF_COLUMNS / 4:
584
- case NUMBER_OF_COLUMNS / 2:
585
- case NUMBER_OF_COLUMNS:
586
- metrics?.dataColumns.elapsedTimeTillReceived.observe({receivedOrder: receivedColumns}, delaySec);
587
- break;
588
- }
589
698
 
590
- if (!blockInput.hasComputedAllData()) {
591
- // immediately attempt fetch of data columns from execution engine
592
- chain.getBlobsTracker.triggerGetBlobs(blockInput);
593
- // if we've received at least half of the columns, trigger reconstruction of the rest
594
- if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
595
- chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
699
+ if (isForkPostGloas(fork)) {
700
+ if (!isGloasDataColumnSidecar(dataColumnSidecar)) {
701
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
702
+ code: DataColumnSidecarErrorCode.INCORRECT_TYPE,
703
+ slot: dataColumnSlot,
704
+ columnIndex: index,
705
+ fork,
706
+ });
596
707
  }
597
- }
598
708
 
599
- if (!blockInput.hasBlockAndAllData()) {
600
- const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
601
- chain.logger.debug("Received gossip data column, waiting for full data availability", {
602
- msToWait: cutoffTimeMs,
603
- dataColumnIndex: index,
604
- ...blockInputMeta,
605
- });
606
- // do not await here to not delay gossip validation
607
- blockInput.waitForBlockAndAllData(cutoffTimeMs).catch((_e) => {
709
+ // After gloas, data columns are tracked in PayloadEnvelopeInput
710
+ const payloadInput = await validatePayloadDataColumn(
711
+ dataColumnSidecar,
712
+ topic.subnet,
713
+ peerIdStr,
714
+ seenTimestampSec
715
+ );
716
+ chain.serializedCache.set(dataColumnSidecar, serializedData);
717
+
718
+ const payloadInputMeta = payloadInput.getLogMeta();
719
+ const {receivedColumns} = payloadInputMeta;
720
+ // it's not helpful to track every single column received
721
+ // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
722
+ switch (receivedColumns) {
723
+ case 1:
724
+ case config.SAMPLES_PER_SLOT:
725
+ case 2 * config.SAMPLES_PER_SLOT:
726
+ case NUMBER_OF_COLUMNS / 4:
727
+ case NUMBER_OF_COLUMNS / 2:
728
+ case NUMBER_OF_COLUMNS:
729
+ metrics?.dataColumns.elapsedTimeTillReceived.observe({receivedOrder: receivedColumns}, delaySec);
730
+ break;
731
+ }
732
+
733
+ if (!payloadInput.hasComputedAllData()) {
734
+ // if we've received at least half of the columns, trigger reconstruction of the rest
735
+ if (receivedColumns >= NUMBER_OF_COLUMNS / 2) {
736
+ chain.columnReconstructionTracker.triggerColumnReconstruction(payloadInput);
737
+ }
738
+
739
+ chain.logger.debug("Received gossip data column, payload envelope input not yet complete", {
740
+ dataColumnIndex: index,
741
+ ...payloadInputMeta,
742
+ });
743
+ }
744
+
745
+ chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
608
746
  chain.logger.debug(
609
- "Waited for data after receiving gossip column. Cut-off reached so attempting to fetch remainder of BlockInput",
610
- {
611
- dataColumnIndex: index,
612
- ...blockInputMeta,
613
- }
747
+ "Error processing execution payload from gossip data column",
748
+ {slot: dataColumnSlot, root: payloadInput.blockRootHex},
749
+ e as Error
614
750
  );
615
- chain.emitter.emit(ChainEvent.incompleteBlockInput, {
616
- blockInput,
617
- peer: peerIdStr,
618
- source: BlockInputSource.gossip,
619
- });
620
751
  });
621
- }
752
+ } else {
753
+ if (config.getForkSeq(dataColumnSlot) < ForkSeq.fulu) {
754
+ throw new GossipActionError(GossipAction.REJECT, {code: "PRE_FULU_BLOCK"});
755
+ }
756
+
757
+ if (isGloasDataColumnSidecar(dataColumnSidecar)) {
758
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
759
+ code: DataColumnSidecarErrorCode.INCORRECT_TYPE,
760
+ slot: dataColumnSlot,
761
+ columnIndex: index,
762
+ fork,
763
+ });
764
+ }
765
+
766
+ // Before gloas, data columns are tracked in BlockInput
767
+ const blockInput = await validateBeaconDataColumn(
768
+ dataColumnSidecar,
769
+ serializedData,
770
+ topic.subnet,
771
+ peerIdStr,
772
+ seenTimestampSec
773
+ );
774
+ chain.serializedCache.set(dataColumnSidecar, serializedData);
775
+ const blockInputMeta = blockInput.getLogMeta();
776
+ const {receivedColumns} = blockInputMeta;
777
+ // it's not helpful to track every single column received
778
+ // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
779
+ switch (receivedColumns) {
780
+ case 1:
781
+ case config.SAMPLES_PER_SLOT:
782
+ case 2 * config.SAMPLES_PER_SLOT:
783
+ case NUMBER_OF_COLUMNS / 4:
784
+ case NUMBER_OF_COLUMNS / 2:
785
+ case NUMBER_OF_COLUMNS:
786
+ metrics?.dataColumns.elapsedTimeTillReceived.observe({receivedOrder: receivedColumns}, delaySec);
787
+ break;
788
+ }
622
789
 
623
- // TODO GLOAS: In Gloas, also add column to PayloadEnvelopeInput and notify the payload processor:
624
- // const payloadInput = chain.seenPayloadEnvelopeInput.get(blockRootHex);
625
- // if (payloadInput) {
626
- // payloadInput.addColumn({columnSidecar, source: BlockInputSource.gossip, seenTimestampSec, peerIdStr});
627
- // chain.processExecutionPayload(payloadInput, {validSignature: true});
628
- // }
790
+ if (!blockInput.hasComputedAllData()) {
791
+ // immediately attempt fetch of data columns from execution engine
792
+ chain.getBlobsTracker.triggerGetBlobs(blockInput);
793
+ // if we've received at least half of the columns, trigger reconstruction of the rest
794
+ if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
795
+ chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
796
+ }
797
+ }
798
+
799
+ if (!blockInput.hasBlockAndAllData()) {
800
+ const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
801
+ chain.logger.debug("Received gossip data column, waiting for full data availability", {
802
+ msToWait: cutoffTimeMs,
803
+ dataColumnIndex: index,
804
+ ...blockInputMeta,
805
+ });
806
+ // do not await here to not delay gossip validation
807
+ blockInput.waitForBlockAndAllData(cutoffTimeMs).catch((_e) => {
808
+ chain.logger.debug(
809
+ "Waited for data after receiving gossip column. Cut-off reached so attempting to fetch remainder of BlockInput",
810
+ {
811
+ dataColumnIndex: index,
812
+ ...blockInputMeta,
813
+ }
814
+ );
815
+ chain.emitter.emit(ChainEvent.incompleteBlockInput, {
816
+ blockInput,
817
+ peer: peerIdStr,
818
+ source: BlockInputSource.gossip,
819
+ });
820
+ });
821
+ }
822
+ }
629
823
  },
630
824
 
631
825
  [GossipType.beacon_aggregate_and_proof]: async ({
@@ -1,8 +1,9 @@
1
1
  import {Type} from "@chainsafe/ssz";
2
2
  import {BeaconConfig} from "@lodestar/config";
3
- import {ForkName, ForkPostAltair, isForkPostAltair} from "@lodestar/params";
3
+ import {ForkName, ForkPostAltair, ForkPostFulu, isForkPostAltair, isForkPostFulu} from "@lodestar/params";
4
4
  import {Protocol, ProtocolHandler, ReqRespRequest} from "@lodestar/reqresp";
5
5
  import {
6
+ DataColumnSidecar,
6
7
  LightClientBootstrap,
7
8
  LightClientFinalityUpdate,
8
9
  LightClientOptimisticUpdate,
@@ -83,8 +84,8 @@ type ResponseBodyByMethod = {
83
84
  [ReqRespMethod.BeaconBlocksByRoot]: SignedBeaconBlock;
84
85
  [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecar;
85
86
  [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecar;
86
- [ReqRespMethod.DataColumnSidecarsByRange]: fulu.DataColumnSidecar;
87
- [ReqRespMethod.DataColumnSidecarsByRoot]: fulu.DataColumnSidecar;
87
+ [ReqRespMethod.DataColumnSidecarsByRange]: DataColumnSidecar;
88
+ [ReqRespMethod.DataColumnSidecarsByRoot]: DataColumnSidecar;
88
89
  [ReqRespMethod.ExecutionPayloadEnvelopesByRoot]: gloas.SignedExecutionPayloadEnvelope;
89
90
  [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: gloas.SignedExecutionPayloadEnvelope;
90
91
 
@@ -146,8 +147,8 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter<
146
147
  [ReqRespMethod.LightClientBootstrap]: (fork) => sszTypesFor(onlyPostAltairFork(fork)).LightClientBootstrap,
147
148
  [ReqRespMethod.LightClientUpdatesByRange]: (fork) => sszTypesFor(onlyPostAltairFork(fork)).LightClientUpdate,
148
149
  [ReqRespMethod.LightClientFinalityUpdate]: (fork) => sszTypesFor(onlyPostAltairFork(fork)).LightClientFinalityUpdate,
149
- [ReqRespMethod.DataColumnSidecarsByRange]: () => ssz.fulu.DataColumnSidecar,
150
- [ReqRespMethod.DataColumnSidecarsByRoot]: () => ssz.fulu.DataColumnSidecar,
150
+ [ReqRespMethod.DataColumnSidecarsByRange]: (fork) => sszTypesFor(onlyPostFuluFork(fork)).DataColumnSidecar,
151
+ [ReqRespMethod.DataColumnSidecarsByRoot]: (fork) => sszTypesFor(onlyPostFuluFork(fork)).DataColumnSidecar,
151
152
  [ReqRespMethod.ExecutionPayloadEnvelopesByRoot]: () => ssz.gloas.SignedExecutionPayloadEnvelope,
152
153
  [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: () => ssz.gloas.SignedExecutionPayloadEnvelope,
153
154
  [ReqRespMethod.LightClientOptimisticUpdate]: (fork) =>
@@ -161,6 +162,13 @@ function onlyPostAltairFork(fork: ForkName): ForkPostAltair {
161
162
  throw Error(`Not a post-altair fork ${fork}`);
162
163
  }
163
164
 
165
+ function onlyPostFuluFork(fork: ForkName): ForkPostFulu {
166
+ if (isForkPostFulu(fork)) {
167
+ return fork;
168
+ }
169
+ throw Error(`Not a post-fulu fork ${fork}`);
170
+ }
171
+
164
172
  export type RequestTypedContainer = {
165
173
  [K in ReqRespMethod]: {method: K; body: RequestBodyByMethod[K]};
166
174
  }[ReqRespMethod];
@@ -11,7 +11,7 @@ import {
11
11
  } from "../../chain/blocks/blockInput/index.js";
12
12
  import {SeenBlockInput} from "../../chain/seenCache/seenGossipBlockInput.js";
13
13
  import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js";
14
- import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js";
14
+ import {validateFuluBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js";
15
15
  import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
16
16
  import {INetwork} from "../../network/index.js";
17
17
  import {getBlobKzgCommitments} from "../../util/dataColumns.js";
@@ -27,7 +27,7 @@ export type DownloadByRangeRequests = {
27
27
  export type DownloadByRangeResponses = {
28
28
  blocks?: SignedBeaconBlock[];
29
29
  blobSidecars?: deneb.BlobSidecars;
30
- columnSidecars?: fulu.DataColumnSidecars;
30
+ columnSidecars?: fulu.DataColumnSidecar[];
31
31
  };
32
32
 
33
33
  export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & {
@@ -58,7 +58,7 @@ export type ValidatedBlobSidecars = {
58
58
 
59
59
  export type ValidatedColumnSidecars = {
60
60
  blockRoot: Uint8Array;
61
- columnSidecars: fulu.DataColumnSidecars;
61
+ columnSidecars: fulu.DataColumnSidecar[];
62
62
  };
63
63
 
64
64
  export type ValidatedResponses = {
@@ -245,7 +245,7 @@ export async function requestByRange({
245
245
  }): Promise<DownloadByRangeResponses> {
246
246
  let blocks: undefined | SignedBeaconBlock[];
247
247
  let blobSidecars: undefined | deneb.BlobSidecars;
248
- let columnSidecars: undefined | fulu.DataColumnSidecars;
248
+ let columnSidecars: undefined | fulu.DataColumnSidecar[];
249
249
 
250
250
  const requests: Promise<unknown>[] = [];
251
251
 
@@ -268,7 +268,7 @@ export async function requestByRange({
268
268
  if (columnsRequest) {
269
269
  requests.push(
270
270
  network.sendDataColumnSidecarsByRange(peerIdStr, columnsRequest).then((columnResponse) => {
271
- columnSidecars = columnResponse;
271
+ columnSidecars = columnResponse as fulu.DataColumnSidecar[];
272
272
  })
273
273
  );
274
274
  }
@@ -615,11 +615,13 @@ export async function validateColumnsByRangeResponse(
615
615
  config: ChainForkConfig,
616
616
  request: fulu.DataColumnSidecarsByRangeRequest,
617
617
  blocks: ValidatedBlock[],
618
- columnSidecars: fulu.DataColumnSidecars,
618
+ columnSidecars: fulu.DataColumnSidecar[],
619
619
  peerDasMetrics?: BeaconMetrics["peerDas"] | null
620
620
  ): Promise<WarnResult<ValidatedColumnSidecars[], DownloadByRangeError>> {
621
621
  const warnings: DownloadByRangeError[] = [];
622
622
 
623
+ // TODO GLOAS: Extend by range column sync to support gloas.DataColumnSidecar and
624
+ // validate against the block bid commitments instead of the fulu signed header shape
623
625
  const seenColumns = new Map<Slot, Map<number, fulu.DataColumnSidecar>>();
624
626
  let currentSlot = -1;
625
627
  let currentIndex = -1;
@@ -767,7 +769,7 @@ export async function validateColumnsByRangeResponse(
767
769
  }
768
770
 
769
771
  validationPromises.push(
770
- validateBlockDataColumnSidecars(
772
+ validateFuluBlockDataColumnSidecars(
771
773
  null, // do not pass chain here so we do not validate header signature
772
774
  slot,
773
775
  blockRoot,
@@ -8,7 +8,7 @@ import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types
8
8
  import {ChainEventEmitter} from "../../chain/emitter.js";
9
9
  import {IBeaconChain} from "../../chain/interface.js";
10
10
  import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js";
11
- import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js";
11
+ import {validateFuluBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js";
12
12
  import {INetwork} from "../../network/interface.js";
13
13
  import {PeerSyncMeta} from "../../network/peers/peersData.js";
14
14
  import {prettyPrintPeerIdStr} from "../../network/util.js";
@@ -52,7 +52,7 @@ export type FetchByRootAndValidateColumnsProps = FetchByRootCoreProps & {
52
52
  export type FetchByRootResponses = {
53
53
  block: SignedBeaconBlock;
54
54
  blobSidecars?: deneb.BlobSidecars;
55
- columnSidecars?: fulu.DataColumnSidecars;
55
+ columnSidecars?: fulu.DataColumnSidecar[];
56
56
  };
57
57
 
58
58
  export type DownloadByRootProps = FetchByRootCoreProps & {
@@ -213,7 +213,7 @@ export async function fetchByRoot({
213
213
  }: FetchByRootProps): Promise<WarnResult<FetchByRootResponses, DownloadByRootError>> {
214
214
  let block: SignedBeaconBlock;
215
215
  let blobSidecars: deneb.BlobSidecars | undefined;
216
- let columnSidecarResult: WarnResult<fulu.DataColumnSidecars, DownloadByRootError> | undefined;
216
+ let columnSidecarResult: WarnResult<fulu.DataColumnSidecar[], DownloadByRootError> | undefined;
217
217
  const {peerId: peerIdStr} = peerMeta;
218
218
 
219
219
  if (isPendingBlockInput(cacheItem)) {
@@ -376,7 +376,7 @@ export async function fetchAndValidateColumns({
376
376
  block,
377
377
  blockRoot,
378
378
  missing,
379
- }: FetchByRootAndValidateColumnsProps): Promise<WarnResult<fulu.DataColumnSidecars, DownloadByRootError>> {
379
+ }: FetchByRootAndValidateColumnsProps): Promise<WarnResult<fulu.DataColumnSidecar[], DownloadByRootError>> {
380
380
  const {peerId: peerIdStr} = peerMeta;
381
381
  const slot = block.message.slot;
382
382
  const blobCount = getBlobKzgCommitments(forkName, block).length;
@@ -387,9 +387,11 @@ export async function fetchAndValidateColumns({
387
387
  const blockRootHex = toRootHex(blockRoot);
388
388
  const peerColumns = new Set(peerMeta.custodyColumns ?? []);
389
389
  const requestedColumns = missing.filter((c) => peerColumns.has(c));
390
- const columnSidecars = await network.sendDataColumnSidecarsByRoot(peerIdStr, [
390
+ // TODO GLOAS: Extend by root column sync to support gloas.DataColumnSidecar and
391
+ // validate against block bid commitments instead of the fulu signed header shape
392
+ const columnSidecars = (await network.sendDataColumnSidecarsByRoot(peerIdStr, [
391
393
  {blockRoot, columns: requestedColumns},
392
- ]);
394
+ ])) as fulu.DataColumnSidecar[];
393
395
 
394
396
  const warnings: DownloadByRootError[] = [];
395
397
 
@@ -440,7 +442,8 @@ export async function fetchAndValidateColumns({
440
442
  );
441
443
  }
442
444
 
443
- await validateBlockDataColumnSidecars(chain, slot, blockRoot, blobCount, columnSidecars, chain?.metrics?.peerDas);
445
+ // TODO GLOAS: Swap to fork-aware column validation once post-gloas by-root sync is implemented
446
+ await validateFuluBlockDataColumnSidecars(chain, slot, blockRoot, blobCount, columnSidecars, chain?.metrics?.peerDas);
444
447
 
445
448
  return {result: columnSidecars, warnings: warnings.length > 0 ? warnings : null};
446
449
  }
@@ -451,11 +454,12 @@ export async function fetchColumnsByRoot({
451
454
  peerMeta,
452
455
  blockRoot,
453
456
  missing,
454
- }: Pick<
455
- FetchByRootAndValidateColumnsProps,
456
- "network" | "peerMeta" | "blockRoot" | "missing"
457
- >): Promise<fulu.DataColumnSidecars> {
458
- return await network.sendDataColumnSidecarsByRoot(peerMeta.peerId, [{blockRoot, columns: missing}]);
457
+ }: Pick<FetchByRootAndValidateColumnsProps, "network" | "peerMeta" | "blockRoot" | "missing">): Promise<
458
+ fulu.DataColumnSidecar[]
459
+ > {
460
+ return (await network.sendDataColumnSidecarsByRoot(peerMeta.peerId, [
461
+ {blockRoot, columns: missing},
462
+ ])) as fulu.DataColumnSidecar[];
459
463
  }
460
464
 
461
465
  export enum DownloadByRootErrorCode {