@lodestar/beacon-node 1.40.0-dev.3be9500fa9 → 1.40.0-dev.63c5c3e7f7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +8 -18
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/config/constants.d.ts +3 -0
  5. package/lib/api/impl/config/constants.d.ts.map +1 -1
  6. package/lib/api/impl/config/constants.js +5 -1
  7. package/lib/api/impl/config/constants.js.map +1 -1
  8. package/lib/api/impl/debug/index.d.ts +1 -1
  9. package/lib/api/impl/debug/index.d.ts.map +1 -1
  10. package/lib/api/impl/debug/index.js +3 -6
  11. package/lib/api/impl/debug/index.js.map +1 -1
  12. package/lib/api/impl/lodestar/index.d.ts.map +1 -1
  13. package/lib/api/impl/lodestar/index.js +5 -2
  14. package/lib/api/impl/lodestar/index.js.map +1 -1
  15. package/lib/api/impl/validator/index.d.ts.map +1 -1
  16. package/lib/api/impl/validator/index.js +9 -8
  17. package/lib/api/impl/validator/index.js.map +1 -1
  18. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  19. package/lib/chain/archiveStore/utils/archiveBlocks.js +4 -0
  20. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  21. package/lib/chain/blocks/blockInput/blockInput.d.ts +2 -0
  22. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  23. package/lib/chain/blocks/blockInput/blockInput.js +6 -0
  24. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  25. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  26. package/lib/chain/blocks/importBlock.js +2 -6
  27. package/lib/chain/blocks/importBlock.js.map +1 -1
  28. package/lib/chain/blocks/index.d.ts.map +1 -1
  29. package/lib/chain/blocks/index.js +0 -14
  30. package/lib/chain/blocks/index.js.map +1 -1
  31. package/lib/chain/blocks/types.d.ts +0 -2
  32. package/lib/chain/blocks/types.d.ts.map +1 -1
  33. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  34. package/lib/chain/blocks/verifyBlock.js +0 -7
  35. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  36. package/lib/chain/blocks/writeBlockInputToDb.d.ts +1 -4
  37. package/lib/chain/blocks/writeBlockInputToDb.d.ts.map +1 -1
  38. package/lib/chain/blocks/writeBlockInputToDb.js +20 -28
  39. package/lib/chain/blocks/writeBlockInputToDb.js.map +1 -1
  40. package/lib/chain/chain.d.ts +14 -2
  41. package/lib/chain/chain.d.ts.map +1 -1
  42. package/lib/chain/chain.js +158 -10
  43. package/lib/chain/chain.js.map +1 -1
  44. package/lib/chain/interface.d.ts +14 -1
  45. package/lib/chain/interface.d.ts.map +1 -1
  46. package/lib/chain/interface.js.map +1 -1
  47. package/lib/chain/prepareNextSlot.js +6 -4
  48. package/lib/chain/prepareNextSlot.js.map +1 -1
  49. package/lib/chain/produceBlock/produceBlockBody.d.ts +3 -2
  50. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  51. package/lib/chain/produceBlock/produceBlockBody.js +5 -3
  52. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  53. package/lib/chain/regen/interface.d.ts +2 -8
  54. package/lib/chain/regen/interface.d.ts.map +1 -1
  55. package/lib/chain/regen/interface.js +0 -1
  56. package/lib/chain/regen/interface.js.map +1 -1
  57. package/lib/chain/regen/queued.d.ts +1 -2
  58. package/lib/chain/regen/queued.d.ts.map +1 -1
  59. package/lib/chain/regen/queued.js +2 -16
  60. package/lib/chain/regen/queued.js.map +1 -1
  61. package/lib/chain/regen/regen.d.ts +3 -7
  62. package/lib/chain/regen/regen.d.ts.map +1 -1
  63. package/lib/chain/regen/regen.js +3 -16
  64. package/lib/chain/regen/regen.js.map +1 -1
  65. package/lib/chain/validation/blobSidecar.js +1 -1
  66. package/lib/chain/validation/blobSidecar.js.map +1 -1
  67. package/lib/chain/validation/dataColumnSidecar.js +1 -1
  68. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  69. package/lib/chain/validatorMonitor.d.ts +2 -0
  70. package/lib/chain/validatorMonitor.d.ts.map +1 -1
  71. package/lib/chain/validatorMonitor.js +42 -3
  72. package/lib/chain/validatorMonitor.js.map +1 -1
  73. package/lib/metrics/metrics/lodestar.d.ts +7 -0
  74. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  75. package/lib/metrics/metrics/lodestar.js +24 -0
  76. package/lib/metrics/metrics/lodestar.js.map +1 -1
  77. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  78. package/lib/network/processor/gossipHandlers.js +0 -3
  79. package/lib/network/processor/gossipHandlers.js.map +1 -1
  80. package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts.map +1 -1
  81. package/lib/network/reqresp/handlers/beaconBlocksByRange.js +2 -4
  82. package/lib/network/reqresp/handlers/beaconBlocksByRange.js.map +1 -1
  83. package/lib/network/reqresp/handlers/beaconBlocksByRoot.d.ts +1 -2
  84. package/lib/network/reqresp/handlers/beaconBlocksByRoot.d.ts.map +1 -1
  85. package/lib/network/reqresp/handlers/beaconBlocksByRoot.js +5 -26
  86. package/lib/network/reqresp/handlers/beaconBlocksByRoot.js.map +1 -1
  87. package/lib/network/reqresp/handlers/blobSidecarsByRoot.d.ts +1 -2
  88. package/lib/network/reqresp/handlers/blobSidecarsByRoot.d.ts.map +1 -1
  89. package/lib/network/reqresp/handlers/blobSidecarsByRoot.js +5 -7
  90. package/lib/network/reqresp/handlers/blobSidecarsByRoot.js.map +1 -1
  91. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
  92. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +1 -2
  93. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
  94. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts.map +1 -1
  95. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +1 -5
  96. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
  97. package/lib/network/reqresp/handlers/index.js +2 -2
  98. package/lib/network/reqresp/handlers/index.js.map +1 -1
  99. package/lib/sync/range/chain.d.ts.map +1 -1
  100. package/lib/sync/range/chain.js +0 -1
  101. package/lib/sync/range/chain.js.map +1 -1
  102. package/lib/sync/range/range.d.ts.map +1 -1
  103. package/lib/sync/range/range.js +0 -3
  104. package/lib/sync/range/range.js.map +1 -1
  105. package/lib/sync/unknownBlock.d.ts.map +1 -1
  106. package/lib/sync/unknownBlock.js +0 -3
  107. package/lib/sync/unknownBlock.js.map +1 -1
  108. package/package.json +15 -15
  109. package/src/api/impl/beacon/blocks/index.ts +8 -18
  110. package/src/api/impl/config/constants.ts +8 -0
  111. package/src/api/impl/debug/index.ts +2 -6
  112. package/src/api/impl/lodestar/index.ts +6 -3
  113. package/src/api/impl/validator/index.ts +12 -11
  114. package/src/chain/archiveStore/utils/archiveBlocks.ts +4 -0
  115. package/src/chain/blocks/blockInput/blockInput.ts +8 -0
  116. package/src/chain/blocks/importBlock.ts +2 -6
  117. package/src/chain/blocks/index.ts +0 -19
  118. package/src/chain/blocks/types.ts +0 -2
  119. package/src/chain/blocks/verifyBlock.ts +0 -8
  120. package/src/chain/blocks/writeBlockInputToDb.ts +24 -30
  121. package/src/chain/chain.ts +180 -20
  122. package/src/chain/interface.ts +16 -0
  123. package/src/chain/prepareNextSlot.ts +6 -6
  124. package/src/chain/produceBlock/produceBlockBody.ts +7 -5
  125. package/src/chain/regen/interface.ts +1 -12
  126. package/src/chain/regen/queued.ts +3 -23
  127. package/src/chain/regen/regen.ts +4 -24
  128. package/src/chain/validation/blobSidecar.ts +1 -1
  129. package/src/chain/validation/dataColumnSidecar.ts +1 -1
  130. package/src/chain/validatorMonitor.ts +52 -3
  131. package/src/metrics/metrics/lodestar.ts +25 -0
  132. package/src/network/processor/gossipHandlers.ts +0 -3
  133. package/src/network/reqresp/handlers/beaconBlocksByRange.ts +2 -4
  134. package/src/network/reqresp/handlers/beaconBlocksByRoot.ts +5 -32
  135. package/src/network/reqresp/handlers/blobSidecarsByRoot.ts +5 -9
  136. package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +5 -2
  137. package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +1 -5
  138. package/src/network/reqresp/handlers/index.ts +2 -2
  139. package/src/sync/range/chain.ts +0 -1
  140. package/src/sync/range/range.ts +0 -3
  141. package/src/sync/unknownBlock.ts +0 -3
@@ -37,14 +37,19 @@ import {
37
37
  UintNum64,
38
38
  ValidatorIndex,
39
39
  Wei,
40
+ deneb,
41
+ fulu,
40
42
  isBlindedBeaconBlock,
41
43
  phase0,
42
44
  rewards,
45
+ ssz,
46
+ sszTypesFor,
43
47
  } from "@lodestar/types";
44
48
  import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
45
49
  import {ProcessShutdownCallback} from "@lodestar/validator";
46
50
  import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
47
51
  import {IBeaconDb} from "../db/index.js";
52
+ import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.ts";
48
53
  import {BuilderStatus} from "../execution/builder/http.js";
49
54
  import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
50
55
  import {Metrics} from "../metrics/index.js";
@@ -55,12 +60,15 @@ import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumn
55
60
  import {callInNextEventLoop} from "../util/eventLoop.js";
56
61
  import {ensureDir, writeIfNotExist} from "../util/file.js";
57
62
  import {isOptimisticBlock} from "../util/forkChoice.js";
63
+ import {JobItemQueue} from "../util/queue/itemQueue.ts";
58
64
  import {SerializedCache} from "../util/serializedCache.js";
65
+ import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.ts";
59
66
  import {ArchiveStore} from "./archiveStore/archiveStore.js";
60
67
  import {CheckpointBalancesCache} from "./balancesCache.js";
61
68
  import {BeaconProposerCache} from "./beaconProposerCache.js";
62
- import {IBlockInput} from "./blocks/blockInput/index.js";
69
+ import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js";
63
70
  import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
71
+ import {persistBlockInputs} from "./blocks/writeBlockInputToDb.ts";
64
72
  import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
65
73
  import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
66
74
  import {ChainEvent, ChainEventEmitter} from "./emitter.js";
@@ -113,6 +121,11 @@ import {ValidatorMonitor} from "./validatorMonitor.js";
113
121
  */
114
122
  const DEFAULT_MAX_CACHED_PRODUCED_RESULTS = 4;
115
123
 
124
+ /**
125
+ * The maximum number of pending unfinalized block writes to the database before backpressure is applied.
126
+ */
127
+ const DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES = 32;
128
+
116
129
  export class BeaconChain implements IBeaconChain {
117
130
  readonly genesisTime: UintNum64;
118
131
  readonly genesisValidatorsRoot: Root;
@@ -136,6 +149,7 @@ export class BeaconChain implements IBeaconChain {
136
149
  readonly lightClientServer?: LightClientServer;
137
150
  readonly reprocessController: ReprocessController;
138
151
  readonly archiveStore: ArchiveStore;
152
+ readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput[]], void>;
139
153
 
140
154
  // Ops pool
141
155
  readonly attestationPool: AttestationPool;
@@ -405,6 +419,15 @@ export class BeaconChain implements IBeaconChain {
405
419
  signal
406
420
  );
407
421
 
422
+ this.unfinalizedBlockWrites = new JobItemQueue(
423
+ persistBlockInputs.bind(this),
424
+ {
425
+ maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES,
426
+ signal,
427
+ },
428
+ metrics?.unfinalizedBlockWritesQueue
429
+ );
430
+
408
431
  // always run PrepareNextSlotScheduler except for fork_choice spec tests
409
432
  if (!opts?.disablePrepareNextSlot) {
410
433
  new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal);
@@ -430,6 +453,12 @@ export class BeaconChain implements IBeaconChain {
430
453
  async close(): Promise<void> {
431
454
  await this.archiveStore.close();
432
455
  await this.bls.close();
456
+
457
+ // Since we don't persist unfinalized fork-choice,
458
+ // we can abort any ongoing unfinalized block writes.
459
+ // TODO: persist fork choice to disk and allow unfinalized block writes to complete.
460
+ this.unfinalizedBlockWrites.dropAllJobs();
461
+
433
462
  this.abortController.abort();
434
463
  }
435
464
 
@@ -501,7 +530,7 @@ export class BeaconChain implements IBeaconChain {
501
530
  // only use regen queue if necessary, it'll cache in checkpointStateCache if regen gets through epoch transition
502
531
  const head = this.forkChoice.getHead();
503
532
  const startSlot = computeStartSlotAtEpoch(epoch);
504
- return this.regen.getBlockSlotState(head.blockRoot, startSlot, {dontTransferCache: true}, regenCaller);
533
+ return this.regen.getBlockSlotState(head, startSlot, {dontTransferCache: true}, regenCaller);
505
534
  }
506
535
 
507
536
  async getStateBySlot(
@@ -519,12 +548,7 @@ export class BeaconChain implements IBeaconChain {
519
548
  if (opts?.allowRegen) {
520
549
  // Find closest canonical block to slot, then trigger regen
521
550
  const block = this.forkChoice.getCanonicalBlockClosestLteSlot(slot) ?? finalizedBlock;
522
- const state = await this.regen.getBlockSlotState(
523
- block.blockRoot,
524
- slot,
525
- {dontTransferCache: true},
526
- RegenCaller.restApi
527
- );
551
+ const state = await this.regen.getBlockSlotState(block, slot, {dontTransferCache: true}, RegenCaller.restApi);
528
552
  return {
529
553
  state,
530
554
  executionOptimistic: isOptimisticBlock(block),
@@ -652,6 +676,13 @@ export class BeaconChain implements IBeaconChain {
652
676
  // Unfinalized slot, attempt to find in fork-choice
653
677
  const block = this.forkChoice.getCanonicalBlockAtSlot(slot);
654
678
  if (block) {
679
+ // Block found in fork-choice.
680
+ // It may be in the block input cache, awaiting full DA reconstruction, check there first
681
+ // Otherwise (most likely), check the hot db
682
+ const blockInput = this.seenBlockInputCache.get(block.blockRoot);
683
+ if (blockInput?.hasBlock()) {
684
+ return {block: blockInput.getBlock(), executionOptimistic: isOptimisticBlock(block), finalized: false};
685
+ }
655
686
  const data = await this.db.block.get(fromHex(block.blockRoot));
656
687
  if (data) {
657
688
  return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
@@ -671,6 +702,13 @@ export class BeaconChain implements IBeaconChain {
671
702
  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null> {
672
703
  const block = this.forkChoice.getBlockHex(root);
673
704
  if (block) {
705
+ // Block found in fork-choice.
706
+ // It may be in the block input cache, awaiting full DA reconstruction, check there first
707
+ // Otherwise (most likely), check the hot db
708
+ const blockInput = this.seenBlockInputCache.get(block.blockRoot);
709
+ if (blockInput?.hasBlock()) {
710
+ return {block: blockInput.getBlock(), executionOptimistic: isOptimisticBlock(block), finalized: false};
711
+ }
674
712
  const data = await this.db.block.get(fromHex(root));
675
713
  if (data) {
676
714
  return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false};
@@ -683,10 +721,137 @@ export class BeaconChain implements IBeaconChain {
683
721
  return data && {block: data, executionOptimistic: false, finalized: true};
684
722
  }
685
723
 
724
+ async getSerializedBlockByRoot(
725
+ root: string
726
+ ): Promise<{block: Uint8Array; executionOptimistic: boolean; finalized: boolean; slot: Slot} | null> {
727
+ const block = this.forkChoice.getBlockHex(root);
728
+ if (block) {
729
+ // Block found in fork-choice.
730
+ // It may be in the block input cache, awaiting full DA reconstruction, check there first
731
+ // Otherwise (most likely), check the hot db
732
+ const blockInput = this.seenBlockInputCache.get(block.blockRoot);
733
+ if (blockInput?.hasBlock()) {
734
+ const signedBlock = blockInput.getBlock();
735
+ const serialized = this.serializedCache.get(signedBlock);
736
+ if (serialized) {
737
+ return {
738
+ block: serialized,
739
+ executionOptimistic: isOptimisticBlock(block),
740
+ finalized: false,
741
+ slot: blockInput.slot,
742
+ };
743
+ }
744
+ return {
745
+ block: sszTypesFor(blockInput.forkName).SignedBeaconBlock.serialize(signedBlock),
746
+ executionOptimistic: isOptimisticBlock(block),
747
+ finalized: false,
748
+ slot: blockInput.slot,
749
+ };
750
+ }
751
+ const data = await this.db.block.getBinary(fromHex(root));
752
+ if (data) {
753
+ const slot = getSlotFromSignedBeaconBlockSerialized(data);
754
+ if (slot === null) throw new Error(`Invalid block data stored in DB for root: ${root}`);
755
+ return {block: data, executionOptimistic: isOptimisticBlock(block), finalized: false, slot};
756
+ }
757
+ // If block is not found in hot db, try cold db since there could be an archive cycle happening
758
+ // TODO: Add a lock to the archiver to have deterministic behavior on where are blocks
759
+ }
760
+
761
+ const data = await this.db.blockArchive.getBinaryEntryByRoot(fromHex(root));
762
+ return data && {block: data.value, executionOptimistic: false, finalized: true, slot: data.key};
763
+ }
764
+
765
+ async getBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<deneb.BlobSidecars | null> {
766
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
767
+ if (blockInput) {
768
+ if (!isBlockInputBlobs(blockInput)) {
769
+ throw new Error(`Expected block input to have blobs: slot=${blockSlot} root=${blockRootHex}`);
770
+ }
771
+ if (!blockInput.hasAllData()) {
772
+ return null;
773
+ }
774
+ return blockInput.getBlobs();
775
+ }
776
+ const unfinalizedBlobSidecars = (await this.db.blobSidecars.get(fromHex(blockRootHex)))?.blobSidecars ?? null;
777
+ if (unfinalizedBlobSidecars) {
778
+ return unfinalizedBlobSidecars;
779
+ }
780
+ return (await this.db.blobSidecarsArchive.get(blockSlot))?.blobSidecars ?? null;
781
+ }
782
+
783
+ async getSerializedBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null> {
784
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
785
+ if (blockInput) {
786
+ if (!isBlockInputBlobs(blockInput)) {
787
+ throw new Error(`Expected block input to have blobs: slot=${blockSlot} root=${blockRootHex}`);
788
+ }
789
+ if (!blockInput.hasAllData()) {
790
+ return null;
791
+ }
792
+ return ssz.deneb.BlobSidecars.serialize(blockInput.getBlobs());
793
+ }
794
+ const unfinalizedBlobSidecarsWrapper = await this.db.blobSidecars.getBinary(fromHex(blockRootHex));
795
+ if (unfinalizedBlobSidecarsWrapper) {
796
+ return unfinalizedBlobSidecarsWrapper.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX);
797
+ }
798
+ const finalizedBlobSidecarsWrapper = await this.db.blobSidecarsArchive.getBinary(blockSlot);
799
+ if (finalizedBlobSidecarsWrapper) {
800
+ return finalizedBlobSidecarsWrapper.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX);
801
+ }
802
+ return null;
803
+ }
804
+
805
+ async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<fulu.DataColumnSidecars> {
806
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
807
+ if (blockInput) {
808
+ if (!isBlockInputColumns(blockInput)) {
809
+ throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
810
+ }
811
+ return blockInput.getAllColumns();
812
+ }
813
+ const sidecarsUnfinalized = await this.db.dataColumnSidecar.values(fromHex(blockRootHex));
814
+ if (sidecarsUnfinalized.length > 0) {
815
+ return sidecarsUnfinalized;
816
+ }
817
+ const sidecarsFinalized = await this.db.dataColumnSidecarArchive.values(blockSlot);
818
+ return sidecarsFinalized;
819
+ }
820
+
821
+ async getSerializedDataColumnSidecars(
822
+ blockSlot: Slot,
823
+ blockRootHex: string,
824
+ indices: number[]
825
+ ): Promise<(Uint8Array | undefined)[]> {
826
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
827
+ if (blockInput) {
828
+ if (!isBlockInputColumns(blockInput)) {
829
+ throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
830
+ }
831
+ return indices.map((index) => {
832
+ const sidecar = blockInput.getColumn(index);
833
+ if (!sidecar) {
834
+ return undefined;
835
+ }
836
+ const serialized = this.serializedCache.get(sidecar);
837
+ if (serialized) {
838
+ return serialized;
839
+ }
840
+ return ssz.fulu.DataColumnSidecar.serialize(sidecar);
841
+ });
842
+ }
843
+ const sidecarsUnfinalized = await this.db.dataColumnSidecar.getManyBinary(fromHex(blockRootHex), indices);
844
+ if (sidecarsUnfinalized.some((sidecar) => sidecar != null)) {
845
+ return sidecarsUnfinalized;
846
+ }
847
+ const sidecarsFinalized = await this.db.dataColumnSidecarArchive.getManyBinary(blockSlot, indices);
848
+ return sidecarsFinalized;
849
+ }
850
+
686
851
  async produceCommonBlockBody(blockAttributes: BlockAttributes): Promise<CommonBlockBody> {
687
- const {slot, parentBlockRoot} = blockAttributes;
852
+ const {slot, parentBlock} = blockAttributes;
688
853
  const state = await this.regen.getBlockSlotState(
689
- toRootHex(parentBlockRoot),
854
+ parentBlock,
690
855
  slot,
691
856
  {dontTransferCache: true},
692
857
  RegenCaller.produceBlock
@@ -723,7 +888,7 @@ export class BeaconChain implements IBeaconChain {
723
888
  slot,
724
889
  feeRecipient,
725
890
  commonBlockBodyPromise,
726
- parentBlockRoot,
891
+ parentBlock,
727
892
  }: BlockAttributes & {commonBlockBodyPromise: Promise<CommonBlockBody>}
728
893
  ): Promise<{
729
894
  block: AssembledBlockType<T>;
@@ -732,7 +897,7 @@ export class BeaconChain implements IBeaconChain {
732
897
  shouldOverrideBuilder?: boolean;
733
898
  }> {
734
899
  const state = await this.regen.getBlockSlotState(
735
- toRootHex(parentBlockRoot),
900
+ parentBlock,
736
901
  slot,
737
902
  {dontTransferCache: true},
738
903
  RegenCaller.produceBlock
@@ -749,7 +914,7 @@ export class BeaconChain implements IBeaconChain {
749
914
  graffiti,
750
915
  slot,
751
916
  feeRecipient,
752
- parentBlockRoot,
917
+ parentBlock,
753
918
  proposerIndex,
754
919
  proposerPubKey,
755
920
  commonBlockBodyPromise,
@@ -772,7 +937,7 @@ export class BeaconChain implements IBeaconChain {
772
937
  const block = {
773
938
  slot,
774
939
  proposerIndex,
775
- parentRoot: parentBlockRoot,
940
+ parentRoot: fromHex(parentBlock.blockRoot),
776
941
  stateRoot: ZERO_HASH,
777
942
  body,
778
943
  } as AssembledBlockType<T>;
@@ -968,12 +1133,7 @@ export class BeaconChain implements IBeaconChain {
968
1133
  // thanks to one epoch look ahead, we don't need to dial up to attEpoch
969
1134
  const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
970
1135
  this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
971
- state = await this.regen.getBlockSlotState(
972
- attHeadBlock.blockRoot,
973
- targetSlot,
974
- {dontTransferCache: true},
975
- regenCaller
976
- );
1136
+ state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
977
1137
  } else if (blockEpoch > attEpoch) {
978
1138
  // should not happen, handled inside attestation verification code
979
1139
  throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
@@ -22,6 +22,8 @@ import {
22
22
  Wei,
23
23
  altair,
24
24
  capella,
25
+ deneb,
26
+ fulu,
25
27
  phase0,
26
28
  rewards,
27
29
  } from "@lodestar/types";
@@ -193,12 +195,26 @@ export interface IBeaconChain {
193
195
  getCanonicalBlockAtSlot(
194
196
  slot: Slot
195
197
  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null>;
198
+ /**
199
+ * Get local block by root, does not fetch from the network
200
+ */
201
+ getSerializedBlockByRoot(
202
+ root: RootHex
203
+ ): Promise<{block: Uint8Array; executionOptimistic: boolean; finalized: boolean; slot: Slot} | null>;
196
204
  /**
197
205
  * Get local block by root, does not fetch from the network
198
206
  */
199
207
  getBlockByRoot(
200
208
  root: RootHex
201
209
  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null>;
210
+ getBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<deneb.BlobSidecars | null>;
211
+ getSerializedBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
212
+ getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<fulu.DataColumnSidecars>;
213
+ getSerializedDataColumnSidecars(
214
+ blockSlot: Slot,
215
+ blockRootHex: string,
216
+ indices: number[]
217
+ ): Promise<(Uint8Array | undefined)[]>;
202
218
 
203
219
  produceCommonBlockBody(blockAttributes: BlockAttributes): Promise<CommonBlockBody>;
204
220
  produceBlock(blockAttributes: BlockAttributes & {commonBlockBodyPromise: Promise<CommonBlockBody>}): Promise<{
@@ -80,9 +80,8 @@ export class PrepareNextSlotScheduler {
80
80
  await sleep(this.config.getSlotComponentDurationMs(PREPARE_NEXT_SLOT_BPS), this.signal);
81
81
 
82
82
  // calling updateHead() here before we produce a block to reduce reorg possibility
83
- const {slot: headSlot, blockRoot: headRoot} = this.chain.recomputeForkChoiceHead(
84
- ForkchoiceCaller.prepareNextSlot
85
- );
83
+ const headBlock = this.chain.recomputeForkChoiceHead(ForkchoiceCaller.prepareNextSlot);
84
+ const {slot: headSlot, blockRoot: headRoot} = headBlock;
86
85
 
87
86
  // PS: previously this was comparing slots, but that gave no leway on the skipped
88
87
  // slots on epoch bounday. Making it more fluid.
@@ -112,7 +111,7 @@ export class PrepareNextSlotScheduler {
112
111
  // Pre Bellatrix: we only do precompute state transition for the last slot of epoch
113
112
  // For Bellatrix, we always do the `processSlots()` to prepare payload for the next slot
114
113
  const prepareState = await this.chain.regen.getBlockSlotState(
115
- headRoot,
114
+ headBlock,
116
115
  prepareSlot,
117
116
  // the slot 0 of next epoch will likely use this Previous Root Checkpoint state for state transition so we transfer cache here
118
117
  // the resulting state with cache will be cached in Checkpoint State Cache which is used for the upcoming block processing
@@ -129,7 +128,8 @@ export class PrepareNextSlotScheduler {
129
128
 
130
129
  if (feeRecipient) {
131
130
  // If we are proposing next slot, we need to predict if we can proposer-boost-reorg or not
132
- const {slot: proposerHeadSlot, blockRoot: proposerHeadRoot} = this.chain.predictProposerHead(clockSlot);
131
+ const proposerHead = this.chain.predictProposerHead(clockSlot);
132
+ const {slot: proposerHeadSlot, blockRoot: proposerHeadRoot} = proposerHead;
133
133
 
134
134
  // If we predict we can reorg, update prepareState with proposer head block
135
135
  if (proposerHeadRoot !== headRoot || proposerHeadSlot !== headSlot) {
@@ -141,7 +141,7 @@ export class PrepareNextSlotScheduler {
141
141
  });
142
142
  this.metrics?.weakHeadDetected.inc();
143
143
  updatedPrepareState = (await this.chain.regen.getBlockSlotState(
144
- proposerHeadRoot,
144
+ proposerHead,
145
145
  prepareSlot,
146
146
  // only transfer cache if epoch transition because that's the state we will use to stateTransition() the 1st block of epoch
147
147
  {dontTransferCache: !isEpochTransition},
@@ -1,5 +1,5 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
- import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
2
+ import {ProtoBlock, getSafeExecutionBlockHash} from "@lodestar/fork-choice";
3
3
  import {
4
4
  ForkName,
5
5
  ForkPostBellatrix,
@@ -43,7 +43,7 @@ import {
43
43
  electra,
44
44
  fulu,
45
45
  } from "@lodestar/types";
46
- import {Logger, sleep, toHex, toPubkeyHex, toRootHex} from "@lodestar/utils";
46
+ import {Logger, fromHex, sleep, toHex, toPubkeyHex, toRootHex} from "@lodestar/utils";
47
47
  import {ZERO_HASH_HEX} from "../../constants/index.js";
48
48
  import {numToQuantity} from "../../execution/engine/utils.js";
49
49
  import {
@@ -86,7 +86,7 @@ export type BlockAttributes = {
86
86
  randaoReveal: BLSSignature;
87
87
  graffiti: Bytes32;
88
88
  slot: Slot;
89
- parentBlockRoot: Root;
89
+ parentBlock: ProtoBlock;
90
90
  feeRecipient?: string;
91
91
  };
92
92
 
@@ -155,13 +155,14 @@ export async function produceBlockBody<T extends BlockType>(
155
155
  const {
156
156
  slot: blockSlot,
157
157
  feeRecipient: requestedFeeRecipient,
158
- parentBlockRoot,
158
+ parentBlock,
159
159
  proposerIndex,
160
160
  proposerPubKey,
161
161
  commonBlockBodyPromise,
162
162
  } = blockAttr;
163
163
  let executionPayloadValue: Wei;
164
164
  let blockBody: AssembledBodyType<T>;
165
+ const parentBlockRoot = fromHex(parentBlock.blockRoot);
165
166
  // even though shouldOverrideBuilder is relevant for the engine response, for simplicity of typing
166
167
  // we just return it undefined for the builder which anyway doesn't get consumed downstream
167
168
  let shouldOverrideBuilder: boolean | undefined;
@@ -637,7 +638,7 @@ export async function produceCommonBlockBody<T extends BlockType>(
637
638
  this: BeaconChain,
638
639
  blockType: T,
639
640
  currentState: CachedBeaconStateAllForks,
640
- {randaoReveal, graffiti, slot, parentBlockRoot}: BlockAttributes
641
+ {randaoReveal, graffiti, slot, parentBlock}: BlockAttributes
641
642
  ): Promise<CommonBlockBody> {
642
643
  const stepsMetrics =
643
644
  blockType === BlockType.Full
@@ -691,6 +692,7 @@ export async function produceCommonBlockBody<T extends BlockType>(
691
692
 
692
693
  const endSyncAggregate = stepsMetrics?.startTimer();
693
694
  if (ForkSeq[fork] >= ForkSeq.altair) {
695
+ const parentBlockRoot = fromHex(parentBlock.blockRoot);
694
696
  const previousSlot = slot - 1;
695
697
  const syncAggregate = this.syncContributionAndProofPool.getAggregate(previousSlot, parentBlockRoot);
696
698
  this.metrics?.production.producedSyncAggregateParticipants.observe(
@@ -26,7 +26,6 @@ export enum RegenFnName {
26
26
  getBlockSlotState = "getBlockSlotState",
27
27
  getState = "getState",
28
28
  getPreState = "getPreState",
29
- getCheckpointState = "getCheckpointState",
30
29
  }
31
30
 
32
31
  export type StateRegenerationOpts = {
@@ -63,21 +62,11 @@ export interface IStateRegeneratorInternal {
63
62
  rCaller: RegenCaller
64
63
  ): Promise<CachedBeaconStateAllForks>;
65
64
 
66
- /**
67
- * Return a valid checkpoint state
68
- * This will always return a state with `state.slot % SLOTS_PER_EPOCH === 0`
69
- */
70
- getCheckpointState(
71
- cp: phase0.Checkpoint,
72
- opts: StateRegenerationOpts,
73
- rCaller: RegenCaller
74
- ): Promise<CachedBeaconStateAllForks>;
75
-
76
65
  /**
77
66
  * Return the state of `blockRoot` processed to slot `slot`
78
67
  */
79
68
  getBlockSlotState(
80
- blockRoot: RootHex,
69
+ block: ProtoBlock,
81
70
  slot: Slot,
82
71
  opts: StateRegenerationOpts,
83
72
  rCaller: RegenCaller
@@ -5,7 +5,7 @@ import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
5
5
  import {Logger, toRootHex} from "@lodestar/utils";
6
6
  import {Metrics} from "../../metrics/index.js";
7
7
  import {JobItemQueue} from "../../util/queue/index.js";
8
- import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js";
8
+ import {CheckpointHex} from "../stateCache/index.js";
9
9
  import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js";
10
10
  import {RegenError, RegenErrorCode} from "./errors.js";
11
11
  import {
@@ -220,24 +220,6 @@ export class QueuedStateRegenerator implements IStateRegenerator {
220
220
  return this.jobQueue.push({key: "getPreState", args: [block, opts, rCaller]});
221
221
  }
222
222
 
223
- async getCheckpointState(
224
- cp: phase0.Checkpoint,
225
- opts: StateRegenerationOpts,
226
- rCaller: RegenCaller
227
- ): Promise<CachedBeaconStateAllForks> {
228
- this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState});
229
-
230
- // First attempt to fetch the state from cache before queueing
231
- const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp));
232
- if (checkpointState) {
233
- return checkpointState;
234
- }
235
-
236
- // The state is not immediately available in the caches, enqueue the job
237
- this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState});
238
- return this.jobQueue.push({key: "getCheckpointState", args: [cp, opts, rCaller]});
239
- }
240
-
241
223
  /**
242
224
  * Get state of provided `blockRoot` and dial forward to `slot`
243
225
  * Use this api with care because we don't want the queue to be busy
@@ -245,7 +227,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
245
227
  * @returns
246
228
  */
247
229
  async getBlockSlotState(
248
- blockRoot: RootHex,
230
+ block: ProtoBlock,
249
231
  slot: Slot,
250
232
  opts: StateRegenerationOpts,
251
233
  rCaller: RegenCaller
@@ -253,7 +235,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
253
235
  this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getBlockSlotState});
254
236
 
255
237
  // The state is not immediately available in the caches, enqueue the job
256
- return this.jobQueue.push({key: "getBlockSlotState", args: [blockRoot, slot, opts, rCaller]});
238
+ return this.jobQueue.push({key: "getBlockSlotState", args: [block, slot, opts, rCaller]});
257
239
  }
258
240
 
259
241
  async getState(stateRoot: RootHex, rCaller: RegenCaller): Promise<CachedBeaconStateAllForks> {
@@ -281,8 +263,6 @@ export class QueuedStateRegenerator implements IStateRegenerator {
281
263
  switch (regenRequest.key) {
282
264
  case "getPreState":
283
265
  return await this.regen.getPreState(...regenRequest.args);
284
- case "getCheckpointState":
285
- return await this.regen.getCheckpointState(...regenRequest.args);
286
266
  case "getBlockSlotState":
287
267
  return await this.regen.getBlockSlotState(...regenRequest.args);
288
268
  case "getState":
@@ -11,7 +11,7 @@ import {
11
11
  processSlots,
12
12
  stateTransition,
13
13
  } from "@lodestar/state-transition";
14
- import {BeaconBlock, RootHex, SignedBeaconBlock, Slot, phase0} from "@lodestar/types";
14
+ import {BeaconBlock, RootHex, SignedBeaconBlock, Slot} from "@lodestar/types";
15
15
  import {Logger, fromHex, toRootHex} from "@lodestar/utils";
16
16
  import {IBeaconDb} from "../../db/index.js";
17
17
  import {Metrics} from "../../metrics/index.js";
@@ -74,45 +74,24 @@ export class StateRegenerator implements IStateRegeneratorInternal {
74
74
  // We may have the checkpoint state with parent root inside the checkpoint state cache
75
75
  // through gossip validation.
76
76
  if (parentEpoch < blockEpoch) {
77
- return this.getCheckpointState({root: block.parentRoot, epoch: blockEpoch}, opts, regenCaller, allowDiskReload);
77
+ return this.getBlockSlotState(parentBlock, block.slot, opts, regenCaller, allowDiskReload);
78
78
  }
79
79
 
80
80
  // Otherwise, get the state normally.
81
81
  return this.getState(parentBlock.stateRoot, regenCaller, allowDiskReload);
82
82
  }
83
83
 
84
- /**
85
- * Get state after block `cp.root` dialed forward to first slot of `cp.epoch`
86
- */
87
- async getCheckpointState(
88
- cp: phase0.Checkpoint,
89
- opts: StateRegenerationOpts,
90
- regenCaller: RegenCaller,
91
- allowDiskReload = false
92
- ): Promise<CachedBeaconStateAllForks> {
93
- const checkpointStartSlot = computeStartSlotAtEpoch(cp.epoch);
94
- return this.getBlockSlotState(toRootHex(cp.root), checkpointStartSlot, opts, regenCaller, allowDiskReload);
95
- }
96
-
97
84
  /**
98
85
  * Get state after block `blockRoot` dialed forward to `slot`
99
86
  * - allowDiskReload should be used with care, as it will cause the state to be reloaded from disk
100
87
  */
101
88
  async getBlockSlotState(
102
- blockRoot: RootHex,
89
+ block: ProtoBlock,
103
90
  slot: Slot,
104
91
  opts: StateRegenerationOpts,
105
92
  regenCaller: RegenCaller,
106
93
  allowDiskReload = false
107
94
  ): Promise<CachedBeaconStateAllForks> {
108
- const block = this.modules.forkChoice.getBlockHex(blockRoot);
109
- if (!block) {
110
- throw new RegenError({
111
- code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
112
- blockRoot,
113
- });
114
- }
115
-
116
95
  if (slot < block.slot) {
117
96
  throw new RegenError({
118
97
  code: RegenErrorCode.SLOT_BEFORE_BLOCK_SLOT,
@@ -121,6 +100,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
121
100
  });
122
101
  }
123
102
 
103
+ const {blockRoot} = block;
124
104
  const {checkpointStateCache} = this.modules;
125
105
  const epoch = computeEpochAtSlot(slot);
126
106
  const latestCheckpointStateCtx = allowDiskReload
@@ -124,7 +124,7 @@ export async function validateGossipBlobSidecar(
124
124
  // [IGNORE] The block's parent (defined by block.parent_root) has been seen (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved).
125
125
  // [REJECT] The block's parent (defined by block.parent_root) passes validation.
126
126
  const blockState = await chain.regen
127
- .getBlockSlotState(parentRoot, blobSlot, {dontTransferCache: true}, RegenCaller.validateGossipBlock)
127
+ .getBlockSlotState(parentBlock, blobSlot, {dontTransferCache: true}, RegenCaller.validateGossipBlock)
128
128
  .catch(() => {
129
129
  throw new BlobSidecarGossipError(GossipAction.IGNORE, {
130
130
  code: BlobSidecarErrorCode.PARENT_UNKNOWN,
@@ -106,7 +106,7 @@ export async function validateGossipDataColumnSidecar(
106
106
  // this is something we should change this in the future to make the code airtight to the spec.
107
107
  // 7) [REJECT] The sidecar's block's parent passes validation.
108
108
  const blockState = await chain.regen
109
- .getBlockSlotState(parentRoot, blockHeader.slot, {dontTransferCache: true}, RegenCaller.validateGossipDataColumn)
109
+ .getBlockSlotState(parentBlock, blockHeader.slot, {dontTransferCache: true}, RegenCaller.validateGossipDataColumn)
110
110
  .catch(() => {
111
111
  throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
112
112
  code: DataColumnSidecarErrorCode.PARENT_UNKNOWN,