@lodestar/beacon-node 1.41.0-dev.b90dff673d → 1.41.0-dev.bb16850567

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +9 -0
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/beacon/state/utils.d.ts +2 -2
  5. package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
  6. package/lib/api/impl/beacon/state/utils.js.map +1 -1
  7. package/lib/api/impl/validator/index.d.ts.map +1 -1
  8. package/lib/api/impl/validator/index.js +5 -1
  9. package/lib/api/impl/validator/index.js.map +1 -1
  10. package/lib/chain/archiveStore/archiveStore.d.ts +0 -1
  11. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  12. package/lib/chain/archiveStore/archiveStore.js +0 -9
  13. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  14. package/lib/chain/archiveStore/interface.d.ts +4 -4
  15. package/lib/chain/archiveStore/interface.d.ts.map +1 -1
  16. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts +4 -4
  17. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts.map +1 -1
  18. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js +4 -1
  19. package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js.map +1 -1
  20. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  21. package/lib/chain/archiveStore/utils/archiveBlocks.js +38 -0
  22. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  23. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  24. package/lib/chain/blocks/importBlock.js +12 -8
  25. package/lib/chain/blocks/importBlock.js.map +1 -1
  26. package/lib/chain/blocks/verifyBlocksSignatures.js +1 -1
  27. package/lib/chain/blocks/verifyBlocksSignatures.js.map +1 -1
  28. package/lib/chain/chain.d.ts +3 -3
  29. package/lib/chain/chain.d.ts.map +1 -1
  30. package/lib/chain/chain.js +20 -9
  31. package/lib/chain/chain.js.map +1 -1
  32. package/lib/chain/interface.d.ts +2 -2
  33. package/lib/chain/interface.d.ts.map +1 -1
  34. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  35. package/lib/chain/prepareNextSlot.js +6 -2
  36. package/lib/chain/prepareNextSlot.js.map +1 -1
  37. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  38. package/lib/chain/produceBlock/produceBlockBody.js +9 -1
  39. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  40. package/lib/chain/regen/errors.d.ts +11 -1
  41. package/lib/chain/regen/errors.d.ts.map +1 -1
  42. package/lib/chain/regen/errors.js +2 -0
  43. package/lib/chain/regen/errors.js.map +1 -1
  44. package/lib/chain/regen/interface.d.ts +12 -6
  45. package/lib/chain/regen/interface.d.ts.map +1 -1
  46. package/lib/chain/regen/queued.d.ts +11 -6
  47. package/lib/chain/regen/queued.d.ts.map +1 -1
  48. package/lib/chain/regen/queued.js +40 -8
  49. package/lib/chain/regen/queued.js.map +1 -1
  50. package/lib/chain/regen/regen.d.ts +5 -0
  51. package/lib/chain/regen/regen.d.ts.map +1 -1
  52. package/lib/chain/regen/regen.js +33 -6
  53. package/lib/chain/regen/regen.js.map +1 -1
  54. package/lib/chain/stateCache/datastore/db.d.ts +4 -5
  55. package/lib/chain/stateCache/datastore/db.d.ts.map +1 -1
  56. package/lib/chain/stateCache/datastore/db.js +32 -10
  57. package/lib/chain/stateCache/datastore/db.js.map +1 -1
  58. package/lib/chain/stateCache/datastore/file.d.ts +1 -1
  59. package/lib/chain/stateCache/datastore/file.d.ts.map +1 -1
  60. package/lib/chain/stateCache/datastore/file.js +5 -5
  61. package/lib/chain/stateCache/datastore/file.js.map +1 -1
  62. package/lib/chain/stateCache/datastore/types.d.ts +1 -1
  63. package/lib/chain/stateCache/datastore/types.d.ts.map +1 -1
  64. package/lib/chain/stateCache/fifoBlockStateCache.d.ts +7 -4
  65. package/lib/chain/stateCache/fifoBlockStateCache.d.ts.map +1 -1
  66. package/lib/chain/stateCache/fifoBlockStateCache.js +8 -3
  67. package/lib/chain/stateCache/fifoBlockStateCache.js.map +1 -1
  68. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +33 -14
  69. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  70. package/lib/chain/stateCache/persistentCheckpointsCache.js +217 -119
  71. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  72. package/lib/chain/stateCache/types.d.ts +15 -8
  73. package/lib/chain/stateCache/types.d.ts.map +1 -1
  74. package/lib/chain/stateCache/types.js.map +1 -1
  75. package/lib/chain/validation/dataColumnSidecar.d.ts +2 -1
  76. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  77. package/lib/chain/validation/dataColumnSidecar.js +124 -107
  78. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  79. package/lib/chain/validation/voluntaryExit.d.ts.map +1 -1
  80. package/lib/chain/validation/voluntaryExit.js +2 -2
  81. package/lib/chain/validation/voluntaryExit.js.map +1 -1
  82. package/lib/metrics/metrics/beacon.d.ts +2 -1
  83. package/lib/metrics/metrics/beacon.d.ts.map +1 -1
  84. package/lib/metrics/metrics/beacon.js +9 -3
  85. package/lib/metrics/metrics/beacon.js.map +1 -1
  86. package/lib/metrics/metrics/lodestar.d.ts +5 -5
  87. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  88. package/lib/metrics/metrics/lodestar.js +16 -14
  89. package/lib/metrics/metrics/lodestar.js.map +1 -1
  90. package/lib/network/discv5/utils.d.ts +1 -1
  91. package/lib/network/discv5/utils.d.ts.map +1 -1
  92. package/lib/network/discv5/utils.js +5 -4
  93. package/lib/network/discv5/utils.js.map +1 -1
  94. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  95. package/lib/network/gossip/gossipsub.js +9 -6
  96. package/lib/network/gossip/gossipsub.js.map +1 -1
  97. package/lib/network/libp2p/index.d.ts +1 -1
  98. package/lib/network/libp2p/index.d.ts.map +1 -1
  99. package/lib/network/libp2p/index.js +35 -17
  100. package/lib/network/libp2p/index.js.map +1 -1
  101. package/lib/network/metadata.d.ts +1 -0
  102. package/lib/network/metadata.d.ts.map +1 -1
  103. package/lib/network/metadata.js +1 -0
  104. package/lib/network/metadata.js.map +1 -1
  105. package/lib/network/options.d.ts +2 -0
  106. package/lib/network/options.d.ts.map +1 -1
  107. package/lib/network/options.js +3 -0
  108. package/lib/network/options.js.map +1 -1
  109. package/lib/network/peers/discover.d.ts +2 -0
  110. package/lib/network/peers/discover.d.ts.map +1 -1
  111. package/lib/network/peers/discover.js +41 -10
  112. package/lib/network/peers/discover.js.map +1 -1
  113. package/lib/sync/range/range.d.ts.map +1 -1
  114. package/lib/sync/range/range.js +1 -0
  115. package/lib/sync/range/range.js.map +1 -1
  116. package/lib/sync/utils/downloadByRange.d.ts +6 -3
  117. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  118. package/lib/sync/utils/downloadByRange.js +6 -5
  119. package/lib/sync/utils/downloadByRange.js.map +1 -1
  120. package/lib/sync/utils/downloadByRoot.js +1 -1
  121. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  122. package/lib/util/dataColumns.d.ts.map +1 -1
  123. package/lib/util/dataColumns.js +5 -1
  124. package/lib/util/dataColumns.js.map +1 -1
  125. package/lib/util/execution.d.ts.map +1 -1
  126. package/lib/util/execution.js +17 -8
  127. package/lib/util/execution.js.map +1 -1
  128. package/package.json +28 -27
  129. package/src/api/impl/beacon/blocks/index.ts +11 -0
  130. package/src/api/impl/beacon/state/utils.ts +2 -2
  131. package/src/api/impl/validator/index.ts +7 -3
  132. package/src/chain/archiveStore/archiveStore.ts +0 -10
  133. package/src/chain/archiveStore/interface.ts +4 -4
  134. package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +8 -5
  135. package/src/chain/archiveStore/utils/archiveBlocks.ts +59 -1
  136. package/src/chain/blocks/importBlock.ts +12 -7
  137. package/src/chain/blocks/verifyBlocksSignatures.ts +1 -1
  138. package/src/chain/chain.ts +27 -14
  139. package/src/chain/interface.ts +2 -2
  140. package/src/chain/prepareNextSlot.ts +6 -2
  141. package/src/chain/produceBlock/produceBlockBody.ts +8 -1
  142. package/src/chain/regen/errors.ts +6 -1
  143. package/src/chain/regen/interface.ts +12 -6
  144. package/src/chain/regen/queued.ts +48 -12
  145. package/src/chain/regen/regen.ts +37 -7
  146. package/src/chain/stateCache/datastore/db.ts +33 -10
  147. package/src/chain/stateCache/datastore/file.ts +6 -5
  148. package/src/chain/stateCache/datastore/types.ts +3 -2
  149. package/src/chain/stateCache/fifoBlockStateCache.ts +10 -4
  150. package/src/chain/stateCache/persistentCheckpointsCache.ts +248 -139
  151. package/src/chain/stateCache/types.ts +18 -8
  152. package/src/chain/validation/dataColumnSidecar.ts +146 -126
  153. package/src/chain/validation/voluntaryExit.ts +2 -1
  154. package/src/metrics/metrics/beacon.ts +9 -3
  155. package/src/metrics/metrics/lodestar.ts +16 -14
  156. package/src/network/discv5/utils.ts +5 -4
  157. package/src/network/gossip/gossipsub.ts +12 -7
  158. package/src/network/libp2p/index.ts +40 -18
  159. package/src/network/metadata.ts +1 -0
  160. package/src/network/options.ts +5 -1
  161. package/src/network/peers/discover.ts +46 -11
  162. package/src/sync/range/range.ts +1 -0
  163. package/src/sync/utils/downloadByRange.ts +12 -3
  164. package/src/sync/utils/downloadByRoot.ts +1 -1
  165. package/src/util/dataColumns.ts +6 -2
  166. package/src/util/execution.ts +23 -12
  167. package/lib/chain/archiveStore/utils/archivePayloads.d.ts +0 -7
  168. package/lib/chain/archiveStore/utils/archivePayloads.d.ts.map +0 -1
  169. package/lib/chain/archiveStore/utils/archivePayloads.js +0 -10
  170. package/lib/chain/archiveStore/utils/archivePayloads.js.map +0 -1
  171. package/src/chain/archiveStore/utils/archivePayloads.ts +0 -15
@@ -7,6 +7,7 @@ import {
7
7
  ForkChoiceErrorCode,
8
8
  NotReorgedReason,
9
9
  getSafeExecutionBlockHash,
10
+ isGloasBlock,
10
11
  } from "@lodestar/fork-choice";
11
12
  import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
12
13
  import {
@@ -30,7 +31,7 @@ import type {BeaconChain} from "../chain.js";
30
31
  import {ChainEvent, ReorgEventData} from "../emitter.js";
31
32
  import {ForkchoiceCaller} from "../forkChoice/index.js";
32
33
  import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
33
- import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
34
+ import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js";
34
35
  import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
35
36
  import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
36
37
  import {getCheckpointFromState} from "./utils/checkpoint.js";
@@ -116,7 +117,11 @@ export async function importBlock(
116
117
 
117
118
  // This adds the state necessary to process the next block
118
119
  // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
119
- this.regen.processState(blockRootHex, postState);
120
+ // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true
121
+ // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
122
+ const payloadPresent = !isGloasBlock(blockSummary);
123
+ // processState manages both block state and payload state variants together for memory/disk management
124
+ this.regen.processBlockState(blockRootHex, postState);
120
125
 
121
126
  this.metrics?.importBlock.bySource.inc({source: source.source});
122
127
  this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex});
@@ -456,12 +461,12 @@ export async function importBlock(
456
461
  // Cache state to preserve epoch transition work
457
462
  const checkpointState = postState;
458
463
  const cp = getCheckpointFromState(checkpointState);
459
- this.regen.addCheckpointState(cp, checkpointState);
464
+ this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
460
465
  // consumers should not mutate state ever
461
466
  this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
462
467
 
463
468
  // Note: in-lined code from previos handler of ChainEvent.checkpoint
464
- this.logger.verbose("Checkpoint processed", toCheckpointHex(cp));
469
+ this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent));
465
470
 
466
471
  const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length;
467
472
  this.metrics?.currentActiveValidators.set(activeValidatorsCount);
@@ -479,7 +484,7 @@ export async function importBlock(
479
484
  const justifiedEpoch = justifiedCheckpoint.epoch;
480
485
  const preJustifiedEpoch = parentBlockSummary.justifiedEpoch;
481
486
  if (justifiedEpoch > preJustifiedEpoch) {
482
- this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint));
487
+ this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent));
483
488
  this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
484
489
  this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
485
490
  }
@@ -493,7 +498,7 @@ export async function importBlock(
493
498
  state: toRootHex(checkpointState.hashTreeRoot()),
494
499
  executionOptimistic: false,
495
500
  });
496
- this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
501
+ this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent));
497
502
  this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
498
503
  }
499
504
  }
@@ -554,7 +559,7 @@ export async function importBlock(
554
559
 
555
560
  if (isBlockInputColumns(blockInput)) {
556
561
  for (const {source} of blockInput.getSampledColumnsWithSource()) {
557
- this.metrics?.importBlock.columnsBySource.inc({source});
562
+ this.metrics?.dataColumns.bySource.inc({source});
558
563
  }
559
564
  } else if (isBlockInputBlobs(blockInput)) {
560
565
  for (const {source} of blockInput.getAllBlobsWithSource()) {
@@ -41,7 +41,7 @@ export async function verifyBlocksSignatures(
41
41
  : //
42
42
  // Verify signatures per block to track which block is invalid
43
43
  bls.verifySignatureSets(
44
- getBlockSignatureSets(config, currentSyncCommitteeIndexed, block, indexedAttestationsByBlock[i], {
44
+ getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], {
45
45
  skipProposerSignature: opts.validProposerSignature,
46
46
  })
47
47
  );
@@ -3,11 +3,12 @@ import {PrivateKey} from "@libp2p/interface";
3
3
  import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
4
4
  import {BeaconConfig} from "@lodestar/config";
5
5
  import {
6
- CheckpointWithHex,
7
6
  CheckpointWithPayloadStatus,
8
7
  IForkChoice,
8
+ PayloadStatus,
9
9
  ProtoBlock,
10
10
  UpdateHeadOpt,
11
+ getCheckpointPayloadStatus,
11
12
  } from "@lodestar/fork-choice";
12
13
  import {LoggerNode} from "@lodestar/logger/node";
13
14
  import {
@@ -126,7 +127,7 @@ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datasto
126
127
  import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
127
128
  import {CPStateDatastore} from "./stateCache/datastore/types.js";
128
129
  import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
129
- import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
130
+ import {PersistentCheckpointStateCache, fcCheckpointToHexPayload} from "./stateCache/persistentCheckpointsCache.js";
130
131
  import {CheckpointStateCache} from "./stateCache/types.js";
131
132
  import {ValidatorMonitor} from "./validatorMonitor.js";
132
133
 
@@ -311,7 +312,9 @@ export class BeaconChain implements IBeaconChain {
311
312
 
312
313
  const nodeId = computeNodeIdFromPrivateKey(privateKey);
313
314
  const initialCustodyGroupCount = opts.initialCustodyGroupCount ?? config.CUSTODY_REQUIREMENT;
314
- this.metrics?.peerDas.targetCustodyGroupCount.set(initialCustodyGroupCount);
315
+ this.metrics?.peerDas.custodyGroupCount.set(initialCustodyGroupCount);
316
+ // TODO: backfill not implemented yet
317
+ this.metrics?.peerDas.custodyGroupsBackfilled.set(0);
315
318
  this.custodyConfig = new CustodyConfig({
316
319
  nodeId,
317
320
  config,
@@ -373,7 +376,8 @@ export class BeaconChain implements IBeaconChain {
373
376
  const {checkpoint} = computeAnchorCheckpoint(config, anchorState);
374
377
  blockStateCache.add(anchorState);
375
378
  blockStateCache.setHeadState(anchorState);
376
- checkpointStateCache.add(checkpoint, anchorState);
379
+ const payloadPresent = getCheckpointPayloadStatus(anchorState, checkpoint.epoch) === PayloadStatus.FULL;
380
+ checkpointStateCache.add(checkpoint, anchorState, payloadPresent);
377
381
 
378
382
  const forkChoice = initializeForkChoice(
379
383
  config,
@@ -646,15 +650,18 @@ export class BeaconChain implements IBeaconChain {
646
650
  return this.cpStateDatastore.readLatestSafe();
647
651
  }
648
652
 
649
- const persistedKey = checkpointToDatastoreKey(checkpoint);
653
+ // TODO GLOAS: Need to revisit the design of this api. Currently we just retrieve FULL state of the checkpoint for backwards compatibility.
654
+ // because pre-gloas we always store FULL checkpoint state.
655
+ const persistedKey = checkpointToDatastoreKey(checkpoint, true);
650
656
  return this.cpStateDatastore.read(persistedKey);
651
657
  }
652
658
 
653
659
  getStateByCheckpoint(
654
- checkpoint: CheckpointWithHex
660
+ checkpoint: CheckpointWithPayloadStatus
655
661
  ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null {
656
662
  // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array
657
- const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint);
663
+ const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
664
+ const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHexPayload);
658
665
  if (cachedStateCtx) {
659
666
  const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot());
660
667
  const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
@@ -669,9 +676,10 @@ export class BeaconChain implements IBeaconChain {
669
676
  }
670
677
 
671
678
  async getStateOrBytesByCheckpoint(
672
- checkpoint: CheckpointWithHex
679
+ checkpoint: CheckpointWithPayloadStatus
673
680
  ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> {
674
- const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpoint);
681
+ const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
682
+ const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload);
675
683
  if (cachedStateCtx) {
676
684
  const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root);
677
685
  const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
@@ -1234,7 +1242,8 @@ export class BeaconChain implements IBeaconChain {
1234
1242
  checkpoint: CheckpointWithPayloadStatus,
1235
1243
  blockState: CachedBeaconStateAllForks
1236
1244
  ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} {
1237
- const state = this.regen.getCheckpointStateSync(checkpoint);
1245
+ const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
1246
+ const state = this.regen.getCheckpointStateSync(checkpointHexPayload);
1238
1247
  if (state) {
1239
1248
  return {state, stateId: "checkpoint_state", shouldWarn: false};
1240
1249
  }
@@ -1361,6 +1370,10 @@ export class BeaconChain implements IBeaconChain {
1361
1370
  private onClockEpoch(epoch: Epoch): void {
1362
1371
  this.metrics?.clockEpoch.set(epoch);
1363
1372
 
1373
+ if (epoch === this.config.GLOAS_FORK_EPOCH) {
1374
+ this.regen.upgradeForGloas(epoch);
1375
+ }
1376
+
1364
1377
  this.seenAttesters.prune(epoch);
1365
1378
  this.seenAggregators.prune(epoch);
1366
1379
  this.seenPayloadAttesters.prune(epoch);
@@ -1374,7 +1387,7 @@ export class BeaconChain implements IBeaconChain {
1374
1387
  this.seenContributionAndProof.prune(head.slot);
1375
1388
  }
1376
1389
 
1377
- private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithHex): void {
1390
+ private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayloadStatus): void {
1378
1391
  this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex});
1379
1392
  }
1380
1393
 
@@ -1385,7 +1398,7 @@ export class BeaconChain implements IBeaconChain {
1385
1398
  });
1386
1399
  }
1387
1400
 
1388
- private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise<void> {
1401
+ private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayloadStatus): Promise<void> {
1389
1402
  this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
1390
1403
  const finalizedSlot = computeStartSlotAtEpoch(cp.epoch);
1391
1404
  this.seenBlockProposers.prune(finalizedSlot);
@@ -1427,7 +1440,7 @@ export class BeaconChain implements IBeaconChain {
1427
1440
  }
1428
1441
  }
1429
1442
 
1430
- private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise<void> {
1443
+ private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayloadStatus): Promise<void> {
1431
1444
  if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) {
1432
1445
  // Custody requirements can only be increased, we can disable dynamic custody updates
1433
1446
  // if the node already maintains custody of all custody groups in case it is configured
@@ -1473,7 +1486,7 @@ export class BeaconChain implements IBeaconChain {
1473
1486
  // Only update if target is increased
1474
1487
  if (targetCustodyGroupCount > this.custodyConfig.targetCustodyGroupCount) {
1475
1488
  this.custodyConfig.updateTargetCustodyGroupCount(targetCustodyGroupCount);
1476
- this.metrics?.peerDas.targetCustodyGroupCount.set(targetCustodyGroupCount);
1489
+ this.metrics?.peerDas.custodyGroupCount.set(targetCustodyGroupCount);
1477
1490
  this.logger.verbose("Updated target custody group count", {
1478
1491
  finalizedEpoch: finalizedCheckpoint.epoch,
1479
1492
  validatorCount: validatorIndices.length,
@@ -1,6 +1,6 @@
1
1
  import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
2
2
  import {BeaconConfig} from "@lodestar/config";
3
- import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
3
+ import {CheckpointWithHex, CheckpointWithPayloadStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
4
4
  import {BeaconStateAllForks, CachedBeaconStateAllForks, EpochShuffling, PubkeyCache} from "@lodestar/state-transition";
5
5
  import {
6
6
  BeaconBlock,
@@ -192,7 +192,7 @@ export interface IBeaconChain {
192
192
  ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null;
193
193
  /** Return state bytes by checkpoint */
194
194
  getStateOrBytesByCheckpoint(
195
- checkpoint: CheckpointWithHex
195
+ checkpoint: CheckpointWithPayloadStatus
196
196
  ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>;
197
197
 
198
198
  /**
@@ -1,6 +1,6 @@
1
1
  import {routes} from "@lodestar/api";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
- import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
3
+ import {PayloadStatus, getSafeExecutionBlockHash} from "@lodestar/fork-choice";
4
4
  import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params";
5
5
  import {
6
6
  CachedBeaconStateAllForks,
@@ -211,7 +211,11 @@ export class PrepareNextSlotScheduler {
211
211
  // + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations
212
212
  if (isEpochTransition) {
213
213
  this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1);
214
- const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch);
214
+ // Determine payloadPresent from head block's payload status
215
+ // Pre-Gloas: payloadStatus is always FULL → payloadPresent = true
216
+ // Post-Gloas: FULL → true, EMPTY → false, PENDING → false (conservative, treat as block state)
217
+ const payloadPresent = headBlock.payloadStatus === PayloadStatus.FULL;
218
+ const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch, payloadPresent);
215
219
  if (previousHits === 0) {
216
220
  this.metrics?.precomputeNextEpochTransition.waste.inc();
217
221
  }
@@ -511,9 +511,16 @@ export async function produceBlockBody<T extends BlockType>(
511
511
  // NOTE: Even though the fulu.BlobsBundle type is superficially the same as deneb.BlobsBundle, it is NOT.
512
512
  // In fulu, proofs are _cell_ proofs, vs in deneb they are _blob_ proofs.
513
513
 
514
+ const timer = this?.metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
514
515
  const cells = blobsBundle.blobs.map((blob) => kzg.computeCells(blob));
516
+ timer?.();
515
517
  if (this.opts.sanityCheckExecutionEngineBlobs) {
516
- await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells);
518
+ const validationTimer = this.metrics?.peerDas.kzgVerificationDataColumnBatchTime.startTimer();
519
+ try {
520
+ await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells);
521
+ } finally {
522
+ validationTimer?.();
523
+ }
517
524
  }
518
525
 
519
526
  (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.commitments;
@@ -1,3 +1,4 @@
1
+ import {PayloadStatus} from "@lodestar/fork-choice";
1
2
  import {Root, RootHex, Slot} from "@lodestar/types";
2
3
 
3
4
  export enum RegenErrorCode {
@@ -9,6 +10,8 @@ export enum RegenErrorCode {
9
10
  BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB",
10
11
  STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR",
11
12
  INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT",
13
+ UNEXPECTED_PAYLOAD_STATUS = "REGEN_ERROR_UNEXPECTED_PAYLOAD_STATUS",
14
+ INTERNAL_ERROR = "REGEN_ERROR_INTERNAL_ERROR",
12
15
  }
13
16
 
14
17
  export type RegenErrorType =
@@ -19,7 +22,9 @@ export type RegenErrorType =
19
22
  | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root}
20
23
  | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root}
21
24
  | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error}
22
- | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex};
25
+ | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}
26
+ | {code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS; blockRoot: RootHex | Root; payloadStatus: PayloadStatus}
27
+ | {code: RegenErrorCode.INTERNAL_ERROR; message: string};
23
28
 
24
29
  export class RegenError extends Error {
25
30
  type: RegenErrorType;
@@ -2,7 +2,7 @@ import {routes} from "@lodestar/api";
2
2
  import {ProtoBlock} from "@lodestar/fork-choice";
3
3
  import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
4
4
  import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
5
- import {CheckpointHex} from "../stateCache/types.js";
5
+ import {CheckpointHexPayload} from "../stateCache/types.js";
6
6
 
7
7
  export enum RegenCaller {
8
8
  getDuties = "getDuties",
@@ -38,15 +38,21 @@ export interface IStateRegenerator extends IStateRegeneratorInternal {
38
38
  dumpCacheSummary(): routes.lodestar.StateCacheItem[];
39
39
  getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null;
40
40
  getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null;
41
- getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
42
- getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null;
41
+ getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
42
+ getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null;
43
43
  getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null;
44
44
  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
45
45
  pruneOnFinalized(finalizedEpoch: Epoch): void;
46
- processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void;
47
- addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void;
46
+ processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void;
47
+ processPayloadState(payloadState: CachedBeaconStateAllForks): void;
48
+ /**
49
+ * payloadPresent is true if this is payload state, false if block state.
50
+ * payloadPresent is always true for pre-gloas.
51
+ */
52
+ addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void;
48
53
  updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void;
49
- updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
54
+ updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null;
55
+ upgradeForGloas(epoch: Epoch): void;
50
56
  }
51
57
 
52
58
  /**
@@ -1,11 +1,11 @@
1
1
  import {routes} from "@lodestar/api";
2
- import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
2
+ import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
3
3
  import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
4
4
  import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types";
5
- import {Logger, toRootHex} from "@lodestar/utils";
5
+ import {Logger, fromHex, toRootHex} from "@lodestar/utils";
6
6
  import {Metrics} from "../../metrics/index.js";
7
7
  import {JobItemQueue} from "../../util/queue/index.js";
8
- import {BlockStateCache, CheckpointHex, CheckpointStateCache} from "../stateCache/types.js";
8
+ import {BlockStateCache, CheckpointHexPayload, CheckpointStateCache} from "../stateCache/types.js";
9
9
  import {RegenError, RegenErrorCode} from "./errors.js";
10
10
  import {
11
11
  IStateRegenerator,
@@ -104,9 +104,19 @@ export class QueuedStateRegenerator implements IStateRegenerator {
104
104
  const parentEpoch = computeEpochAtSlot(parentBlock.slot);
105
105
  const blockEpoch = computeEpochAtSlot(block.slot);
106
106
 
107
+ // Convert PayloadStatus to payloadPresent boolean
108
+ if (parentBlock.payloadStatus === PayloadStatus.PENDING) {
109
+ throw new RegenError({
110
+ code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
111
+ blockRoot: block.parentRoot,
112
+ payloadStatus: parentBlock.payloadStatus,
113
+ });
114
+ }
115
+ const payloadPresent = parentBlock.payloadStatus === PayloadStatus.FULL;
116
+
107
117
  // Check the checkpoint cache (if the pre-state is a checkpoint state)
108
118
  if (parentEpoch < blockEpoch) {
109
- const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch);
119
+ const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, payloadPresent);
110
120
  if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) {
111
121
  return checkpointState;
112
122
  }
@@ -125,14 +135,14 @@ export class QueuedStateRegenerator implements IStateRegenerator {
125
135
  return null;
126
136
  }
127
137
 
128
- async getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
138
+ async getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
129
139
  return this.checkpointStateCache.getStateOrBytes(cp);
130
140
  }
131
141
 
132
142
  /**
133
143
  * Get checkpoint state from cache
134
144
  */
135
- getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null {
145
+ getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null {
136
146
  return this.checkpointStateCache.get(cp);
137
147
  }
138
148
 
@@ -140,7 +150,19 @@ export class QueuedStateRegenerator implements IStateRegenerator {
140
150
  * Get state closest to head
141
151
  */
142
152
  getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null {
143
- return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot);
153
+ // Convert PayloadStatus to payloadPresent boolean
154
+ if (head.payloadStatus === PayloadStatus.PENDING) {
155
+ throw new RegenError({
156
+ code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
157
+ blockRoot: fromHex(head.blockRoot),
158
+ payloadStatus: head.payloadStatus,
159
+ });
160
+ }
161
+ const payloadPresent = head.payloadStatus === PayloadStatus.FULL;
162
+ return (
163
+ this.checkpointStateCache.getLatest(head.blockRoot, Infinity, payloadPresent) ||
164
+ this.blockStateCache.get(head.stateRoot)
165
+ );
144
166
  }
145
167
 
146
168
  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
@@ -153,15 +175,24 @@ export class QueuedStateRegenerator implements IStateRegenerator {
153
175
  this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch);
154
176
  }
155
177
 
156
- processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void {
178
+ processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void {
157
179
  this.blockStateCache.add(postState);
158
180
  this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => {
159
181
  this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e);
160
182
  });
161
183
  }
162
184
 
163
- addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void {
164
- this.checkpointStateCache.add(cp, item);
185
+ /**
186
+ * Process payload state for caching after importing execution payload.
187
+ */
188
+ processPayloadState(payloadState: CachedBeaconStateAllForks): void {
189
+ // Add payload state to block state cache (keyed by payload state root)
190
+ this.blockStateCache.add(payloadState);
191
+ }
192
+
193
+ // TODO GLOAS: This should also be called when importing execution payload after we implement it
194
+ addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void {
195
+ this.checkpointStateCache.add(cp, item, payloadPresent);
165
196
  }
166
197
 
167
198
  updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void {
@@ -197,8 +228,13 @@ export class QueuedStateRegenerator implements IStateRegenerator {
197
228
  }
198
229
  }
199
230
 
200
- updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null {
201
- return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch);
231
+ updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null {
232
+ return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch, payloadPresent);
233
+ }
234
+
235
+ upgradeForGloas(epoch: Epoch): void {
236
+ this.logger.verbose("Upgrading block state cache for Gloas fork", {epoch});
237
+ this.blockStateCache.upgradeToGloas();
202
238
  }
203
239
 
204
240
  /**
@@ -1,6 +1,6 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
- import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
3
- import {SLOTS_PER_EPOCH} from "@lodestar/params";
2
+ import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
3
+ import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
4
4
  import {
5
5
  CachedBeaconStateAllForks,
6
6
  DataAvailabilityStatus,
@@ -111,9 +111,20 @@ export class StateRegenerator implements IStateRegeneratorInternal {
111
111
  const {blockRoot} = block;
112
112
  const {checkpointStateCache} = this.modules;
113
113
  const epoch = computeEpochAtSlot(slot);
114
+
115
+ // Convert PayloadStatus to payloadPresent boolean
116
+ if (block.payloadStatus === PayloadStatus.PENDING) {
117
+ throw new RegenError({
118
+ code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
119
+ blockRoot: fromHex(blockRoot),
120
+ payloadStatus: block.payloadStatus,
121
+ });
122
+ }
123
+ const payloadPresent = block.payloadStatus === PayloadStatus.FULL;
124
+
114
125
  const latestCheckpointStateCtx = allowDiskReload
115
- ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch)
116
- : checkpointStateCache.getLatest(blockRoot, epoch);
126
+ ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, payloadPresent)
127
+ : checkpointStateCache.getLatest(blockRoot, epoch, payloadPresent);
117
128
 
118
129
  // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch
119
130
  // or needs to have empty slots processed until the requested epoch
@@ -166,9 +177,19 @@ export class StateRegenerator implements IStateRegeneratorInternal {
166
177
  const lastBlockToReplay = blocksToReplay.at(-1);
167
178
  if (!lastBlockToReplay) continue;
168
179
  const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1);
180
+
181
+ // Convert PayloadStatus to payloadPresent boolean
182
+ if (b.payloadStatus === PayloadStatus.PENDING) {
183
+ throw new RegenError({
184
+ code: RegenErrorCode.INTERNAL_ERROR,
185
+ message: `Unexpected PENDING payloadStatus for ancestor block ${b.blockRoot} at slot ${b.slot}`,
186
+ });
187
+ }
188
+ const payloadPresent = b.payloadStatus === PayloadStatus.FULL;
189
+
169
190
  state = allowDiskReload
170
- ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch)
171
- : checkpointStateCache.getLatest(b.blockRoot, epoch);
191
+ ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, payloadPresent)
192
+ : checkpointStateCache.getLatest(b.blockRoot, epoch, payloadPresent);
172
193
  if (state) {
173
194
  break;
174
195
  }
@@ -332,6 +353,11 @@ async function processSlotsByCheckpoint(
332
353
  * emitting "checkpoint" events after every epoch processed.
333
354
  *
334
355
  * Stops processing after no more full epochs can be processed.
356
+ *
357
+ * Output state variant:
358
+ * - Post-Gloas: If slots are processed, returns block state (payloadPresent=false).
359
+ * If no slots processed, returns preState as-is (preserves variant).
360
+ * - Pre-Gloas: Always payloadPresent=true (no block/payload distinction).
335
361
  */
336
362
  export async function processSlotsToNearestCheckpoint(
337
363
  modules: {
@@ -374,7 +400,11 @@ export async function processSlotsToNearestCheckpoint(
374
400
  // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped
375
401
  const checkpointState = postState;
376
402
  const cp = getCheckpointFromState(checkpointState);
377
- checkpointStateCache.add(cp, checkpointState);
403
+ // processSlots() only does epoch transitions, never processes payloads
404
+ // Pre-Gloas: payloadPresent is always true (execution payload embedded in block)
405
+ // Post-Gloas: result is a block state (payloadPresent=false)
406
+ const isPayloadPresent = checkpointState.config.getForkSeq(checkpointState.slot) < ForkSeq.gloas;
407
+ checkpointStateCache.add(cp, checkpointState, isPayloadPresent);
378
408
  // consumers should not mutate state ever
379
409
  emitter?.emit(ChainEvent.checkpoint, cp, checkpointState);
380
410
 
@@ -1,6 +1,6 @@
1
1
  import {SLOTS_PER_EPOCH} from "@lodestar/params";
2
2
  import {Epoch, phase0, ssz} from "@lodestar/types";
3
- import {MapDef} from "@lodestar/utils";
3
+ import {MapDef, byteArrayEquals} from "@lodestar/utils";
4
4
  import {IBeaconDb} from "../../../db/interface.js";
5
5
  import {
6
6
  getLastProcessedSlotFromBeaconStateSerialized,
@@ -14,8 +14,8 @@ import {CPStateDatastore, DatastoreKey} from "./types.js";
14
14
  export class DbCPStateDatastore implements CPStateDatastore {
15
15
  constructor(private readonly db: IBeaconDb) {}
16
16
 
17
- async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
18
- const serializedCheckpoint = checkpointToDatastoreKey(cpKey);
17
+ async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise<DatastoreKey> {
18
+ const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent);
19
19
  await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes);
20
20
  return serializedCheckpoint;
21
21
  }
@@ -40,18 +40,30 @@ export class DbCPStateDatastore implements CPStateDatastore {
40
40
  }
41
41
  }
42
42
 
43
+ function extractCheckpointBytes(key: DatastoreKey): Uint8Array {
44
+ const fixedSize = ssz.phase0.Checkpoint.minSize;
45
+ return key.subarray(0, fixedSize);
46
+ }
47
+
43
48
  export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
44
- return ssz.phase0.Checkpoint.deserialize(key);
49
+ return ssz.phase0.Checkpoint.deserialize(extractCheckpointBytes(key));
50
+ }
51
+
52
+ export function checkpointToDatastoreKey(cp: phase0.Checkpoint, payloadPresent: boolean): DatastoreKey {
53
+ const cpBytes = ssz.phase0.Checkpoint.serialize(cp);
54
+ const key = new Uint8Array(cpBytes.length + 1);
55
+ key.set(cpBytes);
56
+ key[cpBytes.length] = payloadPresent ? 1 : 0;
57
+ return key;
45
58
  }
46
59
 
47
- export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
48
- return ssz.phase0.Checkpoint.serialize(cp);
60
+ function isPayloadCheckpointState(key: DatastoreKey): boolean {
61
+ return key.at(-1) === 1;
49
62
  }
50
63
 
51
64
  /**
52
- * Get the latest safe checkpoint state the node can use to boot from
53
- * - it should be the checkpoint state that's unique in its epoch
54
- * - its last processed block slot should be at epoch boundary or last slot of previous epoch
65
+ * Get the latest "safe" checkpoint state the node can use to boot from
66
+ * - its last processed block slot should be at epoch boundary (CRCS) or last slot of previous epoch (PRCS)
55
67
  * - state slot should be at epoch boundary
56
68
  * - state slot should be equal to epoch * SLOTS_PER_EPOCH
57
69
  *
@@ -70,9 +82,20 @@ export async function getLatestSafeDatastoreKey(
70
82
 
71
83
  const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
72
84
  for (const [epoch, keys] of checkpointsByEpoch.entries()) {
73
- // only consider epochs with a single checkpoint to avoid ambiguity from forks
74
85
  if (keys.length === 1) {
86
+ // PRCS (skipped slot) or CRCS and no payloadPresent
87
+ // Pre-gloas always fall into this case
75
88
  dataStoreKeyByEpoch.set(epoch, keys[0]);
89
+ } else if (keys.length === 2) {
90
+ // CRCS without payload and CRCS with payload
91
+ // ie Two keys for the same checkpoint with different payloadPresent suffix (FULL/EMPTY)
92
+ // TODO GLOAS: Here we pick FULL key, there is a chance that payload is orphaned hence we not be able to sync
93
+ const cp0 = extractCheckpointBytes(keys[0]);
94
+ const cp1 = extractCheckpointBytes(keys[1]);
95
+ if (byteArrayEquals(cp0, cp1)) {
96
+ const fullKey = isPayloadCheckpointState(keys[0]) ? keys[0] : keys[1];
97
+ dataStoreKeyByEpoch.set(epoch, fullKey);
98
+ }
76
99
  }
77
100
  }
78
101
 
@@ -1,12 +1,13 @@
1
1
  import path from "node:path";
2
- import {phase0, ssz} from "@lodestar/types";
2
+ import {phase0} from "@lodestar/types";
3
3
  import {fromHex, toHex} from "@lodestar/utils";
4
4
  import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
5
- import {getLatestSafeDatastoreKey} from "./db.js";
5
+ import {checkpointToDatastoreKey, getLatestSafeDatastoreKey} from "./db.js";
6
6
  import {CPStateDatastore, DatastoreKey} from "./types.js";
7
7
 
8
8
  const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
9
- const CHECKPOINT_FILE_NAME_LENGTH = 82;
9
+ /** 41 bytes (40 checkpoint + 1 payloadPresent) = 82 hex chars + "0x" prefix = 84 */
10
+ const CHECKPOINT_FILE_NAME_LENGTH = 84;
10
11
 
11
12
  /**
12
13
  * Implementation of CPStateDatastore using file system, this is beneficial for debugging.
@@ -28,8 +29,8 @@ export class FileCPStateDatastore implements CPStateDatastore {
28
29
  }
29
30
  }
30
31
 
31
- async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
32
- const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey);
32
+ async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise<DatastoreKey> {
33
+ const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent);
33
34
  const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
34
35
  await writeIfNotExist(filePath, stateBytes);
35
36
  return serializedCheckpoint;
@@ -1,11 +1,12 @@
1
1
  import {phase0} from "@lodestar/types";
2
2
 
3
- // With db implementation, persistedKey is serialized data of a checkpoint
3
+ // With db implementation, persistedKey is serialized data of a checkpoint + 1
4
+ // ie a fixed size of `ssz.phase0.Checkpoint.minSize + 1`
4
5
  export type DatastoreKey = Uint8Array;
5
6
 
6
7
  // Make this generic to support testing
7
8
  export interface CPStateDatastore {
8
- write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
9
+ write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean) => Promise<DatastoreKey>;
9
10
  remove: (key: DatastoreKey) => Promise<void>;
10
11
  read: (key: DatastoreKey) => Promise<Uint8Array | null>;
11
12
  readLatestSafe: () => Promise<Uint8Array | null>;