@lodestar/beacon-node 1.39.0-dev.268dcb02bf → 1.39.0-dev.375b1b3da7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/lib/api/impl/beacon/state/utils.d.ts +2 -7
  2. package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
  3. package/lib/api/impl/beacon/state/utils.js +0 -12
  4. package/lib/api/impl/beacon/state/utils.js.map +1 -1
  5. package/lib/api/impl/lodestar/index.js +1 -1
  6. package/lib/api/impl/lodestar/index.js.map +1 -1
  7. package/lib/api/impl/proof/index.d.ts.map +1 -1
  8. package/lib/api/impl/proof/index.js +1 -2
  9. package/lib/api/impl/proof/index.js.map +1 -1
  10. package/lib/api/impl/validator/index.d.ts.map +1 -1
  11. package/lib/api/impl/validator/index.js +1 -3
  12. package/lib/api/impl/validator/index.js.map +1 -1
  13. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  14. package/lib/chain/blocks/importBlock.js +12 -3
  15. package/lib/chain/blocks/importBlock.js.map +1 -1
  16. package/lib/chain/blocks/verifyBlock.d.ts.map +1 -1
  17. package/lib/chain/blocks/verifyBlock.js +8 -1
  18. package/lib/chain/blocks/verifyBlock.js.map +1 -1
  19. package/lib/chain/blocks/verifyBlocksStateTransitionOnly.d.ts.map +1 -1
  20. package/lib/chain/blocks/verifyBlocksStateTransitionOnly.js +1 -0
  21. package/lib/chain/blocks/verifyBlocksStateTransitionOnly.js.map +1 -1
  22. package/lib/chain/chain.d.ts +3 -2
  23. package/lib/chain/chain.d.ts.map +1 -1
  24. package/lib/chain/chain.js +16 -6
  25. package/lib/chain/chain.js.map +1 -1
  26. package/lib/chain/interface.d.ts +2 -2
  27. package/lib/chain/interface.d.ts.map +1 -1
  28. package/lib/chain/lightClient/proofs.d.ts.map +1 -1
  29. package/lib/chain/lightClient/proofs.js +0 -2
  30. package/lib/chain/lightClient/proofs.js.map +1 -1
  31. package/lib/chain/opPools/aggregatedAttestationPool.d.ts +4 -7
  32. package/lib/chain/opPools/aggregatedAttestationPool.d.ts.map +1 -1
  33. package/lib/chain/opPools/aggregatedAttestationPool.js +10 -14
  34. package/lib/chain/opPools/aggregatedAttestationPool.js.map +1 -1
  35. package/lib/chain/opPools/opPool.js +5 -8
  36. package/lib/chain/opPools/opPool.js.map +1 -1
  37. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  38. package/lib/chain/prepareNextSlot.js +4 -7
  39. package/lib/chain/prepareNextSlot.js.map +1 -1
  40. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  41. package/lib/chain/produceBlock/produceBlockBody.js +1 -1
  42. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  43. package/lib/chain/regen/interface.d.ts +1 -5
  44. package/lib/chain/regen/interface.d.ts.map +1 -1
  45. package/lib/chain/regen/queued.d.ts +4 -7
  46. package/lib/chain/regen/queued.d.ts.map +1 -1
  47. package/lib/chain/regen/queued.js +15 -25
  48. package/lib/chain/regen/queued.js.map +1 -1
  49. package/lib/chain/regen/regen.d.ts +1 -1
  50. package/lib/chain/regen/regen.d.ts.map +1 -1
  51. package/lib/chain/regen/regen.js +13 -17
  52. package/lib/chain/regen/regen.js.map +1 -1
  53. package/lib/chain/shufflingCache.d.ts +16 -11
  54. package/lib/chain/shufflingCache.d.ts.map +1 -1
  55. package/lib/chain/shufflingCache.js +47 -41
  56. package/lib/chain/shufflingCache.js.map +1 -1
  57. package/lib/chain/stateCache/blockStateCacheImpl.d.ts +1 -2
  58. package/lib/chain/stateCache/blockStateCacheImpl.d.ts.map +1 -1
  59. package/lib/chain/stateCache/blockStateCacheImpl.js +2 -2
  60. package/lib/chain/stateCache/blockStateCacheImpl.js.map +1 -1
  61. package/lib/chain/stateCache/fifoBlockStateCache.d.ts +1 -2
  62. package/lib/chain/stateCache/fifoBlockStateCache.d.ts.map +1 -1
  63. package/lib/chain/stateCache/fifoBlockStateCache.js +4 -4
  64. package/lib/chain/stateCache/fifoBlockStateCache.js.map +1 -1
  65. package/lib/chain/stateCache/inMemoryCheckpointsCache.d.ts +4 -5
  66. package/lib/chain/stateCache/inMemoryCheckpointsCache.d.ts.map +1 -1
  67. package/lib/chain/stateCache/inMemoryCheckpointsCache.js +9 -10
  68. package/lib/chain/stateCache/inMemoryCheckpointsCache.js.map +1 -1
  69. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +5 -6
  70. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  71. package/lib/chain/stateCache/persistentCheckpointsCache.js +17 -17
  72. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  73. package/lib/chain/stateCache/types.d.ts +5 -6
  74. package/lib/chain/stateCache/types.d.ts.map +1 -1
  75. package/lib/chain/stateCache/types.js.map +1 -1
  76. package/lib/chain/validation/attestation.d.ts.map +1 -1
  77. package/lib/chain/validation/attestation.js +2 -2
  78. package/lib/chain/validation/attestation.js.map +1 -1
  79. package/lib/chain/validation/block.d.ts.map +1 -1
  80. package/lib/chain/validation/block.js +2 -0
  81. package/lib/chain/validation/block.js.map +1 -1
  82. package/lib/db/repositories/stateArchive.d.ts +1 -1
  83. package/lib/db/repositories/stateArchive.d.ts.map +1 -1
  84. package/lib/db/repositories/stateArchive.js +2 -2
  85. package/lib/db/repositories/stateArchive.js.map +1 -1
  86. package/lib/metrics/metrics/lodestar.d.ts +1 -6
  87. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  88. package/lib/metrics/metrics/lodestar.js +3 -17
  89. package/lib/metrics/metrics/lodestar.js.map +1 -1
  90. package/lib/network/reqresp/utils/dataColumnResponseValidation.js +1 -1
  91. package/lib/network/reqresp/utils/dataColumnResponseValidation.js.map +1 -1
  92. package/lib/util/sszBytes.js +1 -1
  93. package/lib/util/sszBytes.js.map +1 -1
  94. package/package.json +15 -15
  95. package/src/api/impl/beacon/state/utils.ts +2 -22
  96. package/src/api/impl/lodestar/index.ts +1 -1
  97. package/src/api/impl/proof/index.ts +1 -2
  98. package/src/api/impl/validator/index.ts +1 -3
  99. package/src/chain/blocks/importBlock.ts +13 -3
  100. package/src/chain/blocks/verifyBlock.ts +9 -3
  101. package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts +1 -0
  102. package/src/chain/chain.ts +20 -9
  103. package/src/chain/interface.ts +2 -2
  104. package/src/chain/lightClient/proofs.ts +0 -2
  105. package/src/chain/opPools/aggregatedAttestationPool.ts +19 -16
  106. package/src/chain/opPools/opPool.ts +5 -7
  107. package/src/chain/prepareNextSlot.ts +2 -6
  108. package/src/chain/produceBlock/produceBlockBody.ts +6 -1
  109. package/src/chain/regen/interface.ts +1 -5
  110. package/src/chain/regen/queued.ts +15 -34
  111. package/src/chain/regen/regen.ts +12 -18
  112. package/src/chain/shufflingCache.ts +67 -50
  113. package/src/chain/stateCache/blockStateCacheImpl.ts +2 -3
  114. package/src/chain/stateCache/fifoBlockStateCache.ts +4 -5
  115. package/src/chain/stateCache/inMemoryCheckpointsCache.ts +9 -15
  116. package/src/chain/stateCache/persistentCheckpointsCache.ts +17 -25
  117. package/src/chain/stateCache/types.ts +5 -10
  118. package/src/chain/validation/attestation.ts +3 -3
  119. package/src/chain/validation/block.ts +3 -0
  120. package/src/db/repositories/stateArchive.ts +2 -2
  121. package/src/metrics/metrics/lodestar.ts +3 -17
  122. package/src/network/reqresp/utils/dataColumnResponseValidation.ts +1 -1
  123. package/src/util/sszBytes.ts +1 -1
@@ -75,6 +75,10 @@ export async function verifyBlocksInEpoch(
75
75
  throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error});
76
76
  });
77
77
 
78
+ // in forky condition, make sure to populate ShufflingCache with regened state
79
+ // otherwise it may fail to get indexed attestations from shuffling cache later
80
+ this.shufflingCache.processState(preState0);
81
+
78
82
  if (!isStateValidatorsNodesPopulated(preState0)) {
79
83
  this.logger.verbose("verifyBlocksInEpoch preState0 SSZ cache stats", {
80
84
  slot: preState0.slot,
@@ -105,9 +109,11 @@ export async function verifyBlocksInEpoch(
105
109
  // Store indexed attestations for each block to avoid recomputing them during import
106
110
  const indexedAttestationsByBlock: IndexedAttestation[][] = [];
107
111
  for (const [i, block] of blocks.entries()) {
108
- indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) =>
109
- preState0.epochCtx.getIndexedAttestation(fork, attestation)
110
- );
112
+ indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) => {
113
+ const attEpoch = computeEpochAtSlot(attestation.data.slot);
114
+ const decisionRoot = preState0.epochCtx.getShufflingDecisionRoot(attEpoch);
115
+ return this.shufflingCache.getIndexedAttestation(attEpoch, decisionRoot, fork, attestation);
116
+ });
111
117
  }
112
118
 
113
119
  // batch all I/O operations to reduce overhead
@@ -59,6 +59,7 @@ export async function verifyBlocksStateTransitionOnly(
59
59
  // if block is trusted don't verify proposer or op signature
60
60
  verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
61
61
  verifySignatures: !useBlsBatchVerify && !validSignatures,
62
+ dontTransferCache: false,
62
63
  },
63
64
  {metrics, validatorMonitor}
64
65
  );
@@ -52,6 +52,7 @@ import {computeNodeIdFromPrivateKey} from "../network/subnets/interface.js";
52
52
  import {BufferPool} from "../util/bufferPool.js";
53
53
  import {Clock, ClockEvent, IClock} from "../util/clock.js";
54
54
  import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumns.js";
55
+ import {callInNextEventLoop} from "../util/eventLoop.js";
55
56
  import {ensureDir, writeIfNotExist} from "../util/file.js";
56
57
  import {isOptimisticBlock} from "../util/forkChoice.js";
57
58
  import {SerializedCache} from "../util/serializedCache.js";
@@ -291,7 +292,8 @@ export class BeaconChain implements IBeaconChain {
291
292
  });
292
293
 
293
294
  this._earliestAvailableSlot = anchorState.slot;
294
- this.shufflingCache = anchorState.epochCtx.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [
295
+
296
+ this.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [
295
297
  {
296
298
  shuffling: anchorState.epochCtx.previousShuffling,
297
299
  decisionRoot: anchorState.epochCtx.previousDecisionRoot,
@@ -417,6 +419,7 @@ export class BeaconChain implements IBeaconChain {
417
419
  clock.addListener(ClockEvent.epoch, this.onClockEpoch.bind(this));
418
420
  emitter.addListener(ChainEvent.forkChoiceFinalized, this.onForkChoiceFinalized.bind(this));
419
421
  emitter.addListener(ChainEvent.forkChoiceJustified, this.onForkChoiceJustified.bind(this));
422
+ emitter.addListener(ChainEvent.checkpoint, this.onCheckpoint.bind(this));
420
423
  }
421
424
 
422
425
  async init(): Promise<void> {
@@ -504,7 +507,7 @@ export class BeaconChain implements IBeaconChain {
504
507
  async getStateBySlot(
505
508
  slot: Slot,
506
509
  opts?: StateGetOpts
507
- ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null> {
510
+ ): Promise<{state: CachedBeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null> {
508
511
  const finalizedBlock = this.forkChoice.getFinalizedBlock();
509
512
 
510
513
  if (slot < finalizedBlock.slot) {
@@ -559,7 +562,7 @@ export class BeaconChain implements IBeaconChain {
559
562
  async getStateByStateRoot(
560
563
  stateRoot: RootHex,
561
564
  opts?: StateGetOpts
562
- ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null> {
565
+ ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> {
563
566
  if (opts?.allowRegen) {
564
567
  const state = await this.regen.getState(stateRoot, RegenCaller.restApi);
565
568
  const block = this.forkChoice.getBlock(state.latestBlockHeader.hashTreeRoot());
@@ -587,7 +590,8 @@ export class BeaconChain implements IBeaconChain {
587
590
  };
588
591
  }
589
592
 
590
- const data = await this.db.stateArchive.getByRoot(fromHex(stateRoot));
593
+ // this is mostly useful for a node with `--chain.archiveStateEpochFrequency 1`
594
+ const data = await this.db.stateArchive.getBinaryByRoot(fromHex(stateRoot));
591
595
  return data && {state: data, executionOptimistic: false, finalized: true};
592
596
  }
593
597
 
@@ -980,8 +984,8 @@ export class BeaconChain implements IBeaconChain {
980
984
  this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
981
985
  state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
982
986
  }
983
-
984
- // should always be the current epoch of the active context so no need to await a result from the ShufflingCache
987
+ // resolve the promise to unblock other calls of the same epoch and dependent root
988
+ this.shufflingCache.processState(state);
985
989
  return state.epochCtx.getShufflingAtEpoch(attEpoch);
986
990
  }
987
991
 
@@ -1165,6 +1169,13 @@ export class BeaconChain implements IBeaconChain {
1165
1169
  this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex});
1166
1170
  }
1167
1171
 
1172
+ private onCheckpoint(this: BeaconChain, _checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks): void {
1173
+ // Defer to not block other checkpoint event handlers, which can cause lightclient update delays
1174
+ callInNextEventLoop(() => {
1175
+ this.shufflingCache.processState(state);
1176
+ });
1177
+ }
1178
+
1168
1179
  private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise<void> {
1169
1180
  this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
1170
1181
  this.seenBlockProposers.prune(computeStartSlotAtEpoch(cp.epoch));
@@ -1295,9 +1306,9 @@ export class BeaconChain implements IBeaconChain {
1295
1306
 
1296
1307
  preState = processSlots(preState, block.slot); // Dial preState's slot to block.slot
1297
1308
 
1298
- const postState = this.regen.getStateSync(toRootHex(block.stateRoot)) ?? undefined;
1309
+ const proposerRewards = this.regen.getStateSync(toRootHex(block.stateRoot))?.proposerRewards ?? undefined;
1299
1310
 
1300
- return computeBlockRewards(this.config, block, preState.clone(), postState?.clone());
1311
+ return computeBlockRewards(this.config, block, preState, proposerRewards);
1301
1312
  }
1302
1313
 
1303
1314
  async getAttestationsRewards(
@@ -1338,6 +1349,6 @@ export class BeaconChain implements IBeaconChain {
1338
1349
 
1339
1350
  preState = processSlots(preState, block.slot); // Dial preState's slot to block.slot
1340
1351
 
1341
- return computeSyncCommitteeRewards(this.config, this.index2pubkey, block, preState.clone(), validatorIds);
1352
+ return computeSyncCommitteeRewards(this.config, this.index2pubkey, block, preState, validatorIds);
1342
1353
  }
1343
1354
  }
@@ -168,12 +168,12 @@ export interface IBeaconChain {
168
168
  getStateBySlot(
169
169
  slot: Slot,
170
170
  opts?: StateGetOpts
171
- ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
171
+ ): Promise<{state: CachedBeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
172
172
  /** Returns a local state by state root */
173
173
  getStateByStateRoot(
174
174
  stateRoot: RootHex,
175
175
  opts?: StateGetOpts
176
- ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
176
+ ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>;
177
177
  /** Return serialized bytes of a persisted checkpoint state */
178
178
  getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null>;
179
179
  /** Returns a cached state by checkpoint */
@@ -12,7 +12,6 @@ import {BeaconBlockBody, SSZTypesFor, ssz} from "@lodestar/types";
12
12
  import {SyncCommitteeWitness} from "./types.js";
13
13
 
14
14
  export function getSyncCommitteesWitness(fork: ForkName, state: BeaconStateAllForks): SyncCommitteeWitness {
15
- state.commit();
16
15
  const n1 = state.node;
17
16
  let witness: Uint8Array[];
18
17
  let currentSyncCommitteeRoot: Uint8Array;
@@ -71,7 +70,6 @@ export function getCurrentSyncCommitteeBranch(syncCommitteesWitness: SyncCommitt
71
70
  }
72
71
 
73
72
  export function getFinalizedRootProof(state: CachedBeaconStateAllForks): Uint8Array[] {
74
- state.commit();
75
73
  const finalizedRootGindex = state.epochCtx.isPostElectra() ? FINALIZED_ROOT_GINDEX_ELECTRA : FINALIZED_ROOT_GINDEX;
76
74
  return new Tree(state.node).getSingleProof(BigInt(finalizedRootGindex));
77
75
  }
@@ -35,6 +35,7 @@ import {MapDef, assert, toRootHex} from "@lodestar/utils";
35
35
  import {Metrics} from "../../metrics/metrics.js";
36
36
  import {IntersectResult, intersectUint8Arrays} from "../../util/bitArray.js";
37
37
  import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
38
+ import {ShufflingCache} from "../shufflingCache.js";
38
39
  import {InsertOutcome} from "./types.js";
39
40
  import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js";
40
41
 
@@ -207,22 +208,18 @@ export class AggregatedAttestationPool {
207
208
  this.lowestPermissibleSlot = Math.max(clockSlot - slotsToRetain, 0);
208
209
  }
209
210
 
210
- getAttestationsForBlock(fork: ForkName, forkChoice: IForkChoice, state: CachedBeaconStateAllForks): Attestation[] {
211
- const forkSeq = ForkSeq[fork];
212
- return forkSeq >= ForkSeq.electra
213
- ? this.getAttestationsForBlockElectra(fork, forkChoice, state)
214
- : this.getAttestationsForBlockPreElectra(fork, forkChoice, state);
215
- }
216
-
217
- /**
218
- * Get attestations to be included in a block pre-electra. Returns up to $MAX_ATTESTATIONS items
219
- */
220
- getAttestationsForBlockPreElectra(
211
+ getAttestationsForBlock(
221
212
  fork: ForkName,
222
213
  forkChoice: IForkChoice,
214
+ shufflingCache: ShufflingCache,
223
215
  state: CachedBeaconStateAllForks
224
- ): phase0.Attestation[] {
225
- throw new Error("Does not support producing blocks for pre-electra forks anymore");
216
+ ): Attestation[] {
217
+ const forkSeq = ForkSeq[fork];
218
+ if (forkSeq < ForkSeq.electra) {
219
+ throw new Error("Does not support producing blocks for pre-electra forks anymore");
220
+ }
221
+
222
+ return this.getAttestationsForBlockElectra(fork, forkChoice, shufflingCache, state);
226
223
  }
227
224
 
228
225
  /**
@@ -231,6 +228,7 @@ export class AggregatedAttestationPool {
231
228
  getAttestationsForBlockElectra(
232
229
  fork: ForkName,
233
230
  forkChoice: IForkChoice,
231
+ shufflingCache: ShufflingCache,
234
232
  state: CachedBeaconStateAllForks
235
233
  ): electra.Attestation[] {
236
234
  const stateSlot = state.slot;
@@ -238,7 +236,7 @@ export class AggregatedAttestationPool {
238
236
  const statePrevEpoch = stateEpoch - 1;
239
237
  const rootCache = new RootCache(state);
240
238
 
241
- const notSeenValidatorsFn = getNotSeenValidatorsFn(this.config, state);
239
+ const notSeenValidatorsFn = getNotSeenValidatorsFn(this.config, shufflingCache, state);
242
240
  const validateAttestationDataFn = getValidateAttestationDataFn(forkChoice, state);
243
241
 
244
242
  const slots = Array.from(this.attestationGroupByIndexByDataHexBySlot.keys()).sort((a, b) => b - a);
@@ -740,7 +738,11 @@ export function aggregateConsolidation({byCommittee, attData}: AttestationsConso
740
738
  * Pre-compute participation from a CachedBeaconStateAllForks, for use to check if an attestation's committee
741
739
  * has already attested or not.
742
740
  */
743
- export function getNotSeenValidatorsFn(config: BeaconConfig, state: CachedBeaconStateAllForks): GetNotSeenValidatorsFn {
741
+ export function getNotSeenValidatorsFn(
742
+ config: BeaconConfig,
743
+ shufflingCache: ShufflingCache,
744
+ state: CachedBeaconStateAllForks
745
+ ): GetNotSeenValidatorsFn {
744
746
  const stateSlot = state.slot;
745
747
  if (config.getForkName(stateSlot) === ForkName.phase0) {
746
748
  throw new Error("getNotSeenValidatorsFn is not supported phase0 state");
@@ -772,7 +774,8 @@ export function getNotSeenValidatorsFn(config: BeaconConfig, state: CachedBeacon
772
774
  return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
773
775
  }
774
776
 
775
- const committee = state.epochCtx.getBeaconCommittee(slot, committeeIndex);
777
+ const decisionRoot = state.epochCtx.getShufflingDecisionRoot(computeEpochAtSlot(slot));
778
+ const committee = shufflingCache.getBeaconCommittee(epoch, decisionRoot, slot, committeeIndex);
776
779
  notSeenCommitteeMembers = new Set<number>();
777
780
  for (const [i, validatorIndex] of committee.entries()) {
778
781
  // no need to check flagIsTimelySource as if validator is not seen, it's participation status is 0
@@ -1,5 +1,5 @@
1
1
  import {BeaconConfig} from "@lodestar/config";
2
- import {Id, Repository} from "@lodestar/db";
2
+ import {DbBatch, Id, Repository} from "@lodestar/db";
3
3
  import {
4
4
  BLS_WITHDRAWAL_PREFIX,
5
5
  ForkName,
@@ -440,23 +440,21 @@ async function persistDiff<K extends Id, V>(
440
440
  serializeKey: (key: K) => number | string
441
441
  ): Promise<void> {
442
442
  const persistedKeys = await dbRepo.keys();
443
- const itemsToPut: {key: K; value: V}[] = [];
444
- const keysToDelete: K[] = [];
443
+ const batch: DbBatch<K, V> = [];
445
444
 
446
445
  const persistedKeysSerialized = new Set(persistedKeys.map(serializeKey));
447
446
  for (const item of items) {
448
447
  if (!persistedKeysSerialized.has(serializeKey(item.key))) {
449
- itemsToPut.push(item);
448
+ batch.push({type: "put", key: item.key, value: item.value});
450
449
  }
451
450
  }
452
451
 
453
452
  const targetKeysSerialized = new Set(items.map((item) => serializeKey(item.key)));
454
453
  for (const persistedKey of persistedKeys) {
455
454
  if (!targetKeysSerialized.has(serializeKey(persistedKey))) {
456
- keysToDelete.push(persistedKey);
455
+ batch.push({type: "del", key: persistedKey});
457
456
  }
458
457
  }
459
458
 
460
- if (itemsToPut.length > 0) await dbRepo.batchPut(itemsToPut);
461
- if (keysToDelete.length > 0) await dbRepo.batchDelete(keysToDelete);
459
+ if (batch.length > 0) await dbRepo.batch(batch);
462
460
  }
@@ -117,12 +117,7 @@ export class PrepareNextSlotScheduler {
117
117
  // the slot 0 of next epoch will likely use this Previous Root Checkpoint state for state transition so we transfer cache here
118
118
  // the resulting state with cache will be cached in Checkpoint State Cache which is used for the upcoming block processing
119
119
  // for other slots dontTransferCached=true because we don't run state transition on this state
120
- //
121
- // Shuffling calculation will be done asynchronously when passing asyncShufflingCalculation=true. Shuffling will be queued in
122
- // beforeProcessEpoch and should theoretically be ready immediately after the synchronous epoch transition finished and the
123
- // event loop is free. In long periods of non-finality too many forks will cause the shufflingCache to throw an error for
124
- // too many queued shufflings so only run async during normal epoch transition. See issue ChainSafe/lodestar#7244
125
- {dontTransferCache: !isEpochTransition, asyncShufflingCalculation: true},
120
+ {dontTransferCache: !isEpochTransition},
126
121
  RegenCaller.precomputeEpoch
127
122
  );
128
123
 
@@ -148,6 +143,7 @@ export class PrepareNextSlotScheduler {
148
143
  updatedPrepareState = (await this.chain.regen.getBlockSlotState(
149
144
  proposerHeadRoot,
150
145
  prepareSlot,
146
+ // only transfer cache if epoch transition because that's the state we will use to stateTransition() the 1st block of epoch
151
147
  {dontTransferCache: !isEpochTransition},
152
148
  RegenCaller.predictProposerHead
153
149
  )) as CachedBeaconStateExecutions;
@@ -661,7 +661,12 @@ export async function produceCommonBlockBody<T extends BlockType>(
661
661
  this.opPool.getSlashingsAndExits(currentState, blockType, this.metrics);
662
662
 
663
663
  const endAttestations = stepsMetrics?.startTimer();
664
- const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(fork, this.forkChoice, currentState);
664
+ const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(
665
+ fork,
666
+ this.forkChoice,
667
+ this.shufflingCache,
668
+ currentState
669
+ );
665
670
  endAttestations?.({
666
671
  step: BlockProductionStep.attestations,
667
672
  });
@@ -31,10 +31,6 @@ export enum RegenFnName {
31
31
 
32
32
  export type StateRegenerationOpts = {
33
33
  dontTransferCache: boolean;
34
- /**
35
- * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch if not passed as `true`
36
- */
37
- asyncShufflingCalculation?: boolean;
38
34
  };
39
35
 
40
36
  export interface IStateRegenerator extends IStateRegeneratorInternal {
@@ -90,5 +86,5 @@ export interface IStateRegeneratorInternal {
90
86
  /**
91
87
  * Return the exact state with `stateRoot`
92
88
  */
93
- getState(stateRoot: RootHex, rCaller: RegenCaller, opts?: StateRegenerationOpts): Promise<CachedBeaconStateAllForks>;
89
+ getState(stateRoot: RootHex, rCaller: RegenCaller): Promise<CachedBeaconStateAllForks>;
94
90
  }
@@ -79,21 +79,15 @@ export class QueuedStateRegenerator implements IStateRegenerator {
79
79
 
80
80
  /**
81
81
  * Get a state from block state cache.
82
- * This is not for block processing so don't transfer cache
83
82
  */
84
83
  getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null {
85
- return this.blockStateCache.get(stateRoot, {dontTransferCache: true});
84
+ return this.blockStateCache.get(stateRoot);
86
85
  }
87
86
 
88
87
  /**
89
88
  * Get state for block processing.
90
- * By default, do not transfer cache except for the block at clock slot
91
- * which is usually the gossip block.
92
89
  */
93
- getPreStateSync(
94
- block: BeaconBlock,
95
- opts: StateRegenerationOpts = {dontTransferCache: true}
96
- ): CachedBeaconStateAllForks | null {
90
+ getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null {
97
91
  const parentRoot = toRootHex(block.parentRoot);
98
92
  const parentBlock = this.forkChoice.getBlockHex(parentRoot);
99
93
  if (!parentBlock) {
@@ -108,7 +102,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
108
102
 
109
103
  // Check the checkpoint cache (if the pre-state is a checkpoint state)
110
104
  if (parentEpoch < blockEpoch) {
111
- const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, opts);
105
+ const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch);
112
106
  if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) {
113
107
  return checkpointState;
114
108
  }
@@ -118,7 +112,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
118
112
  // Otherwise the state transition may not be cached and wasted. Queue for regen since the
119
113
  // work required will still be significant.
120
114
  if (parentEpoch === blockEpoch) {
121
- const state = this.blockStateCache.get(parentBlock.stateRoot, opts);
115
+ const state = this.blockStateCache.get(parentBlock.stateRoot);
122
116
  if (state) {
123
117
  return state;
124
118
  }
@@ -132,21 +126,17 @@ export class QueuedStateRegenerator implements IStateRegenerator {
132
126
  }
133
127
 
134
128
  /**
135
- * Get checkpoint state from cache, this function is not for block processing so don't transfer cache
129
+ * Get checkpoint state from cache
136
130
  */
137
131
  getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null {
138
- return this.checkpointStateCache.get(cp, {dontTransferCache: true});
132
+ return this.checkpointStateCache.get(cp);
139
133
  }
140
134
 
141
135
  /**
142
- * Get state closest to head, this function is not for block processing so don't transfer cache
136
+ * Get state closest to head
143
137
  */
144
138
  getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null {
145
- const opts = {dontTransferCache: true};
146
- return (
147
- this.checkpointStateCache.getLatest(head.blockRoot, Infinity, opts) ||
148
- this.blockStateCache.get(head.stateRoot, opts)
149
- );
139
+ return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot);
150
140
  }
151
141
 
152
142
  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
@@ -181,10 +171,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
181
171
  maybeHeadStateRoot,
182
172
  };
183
173
  const headState =
184
- newHeadStateRoot === maybeHeadStateRoot
185
- ? maybeHeadState
186
- : // maybeHeadState was already in block state cache so we don't transfer the cache
187
- this.blockStateCache.get(newHeadStateRoot, {dontTransferCache: true});
174
+ newHeadStateRoot === maybeHeadStateRoot ? maybeHeadState : this.blockStateCache.get(newHeadStateRoot);
188
175
 
189
176
  if (headState) {
190
177
  this.blockStateCache.setHeadState(headState);
@@ -199,9 +186,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
199
186
 
200
187
  // for the new FIFOBlockStateCache, it's important to reload state to regen head state here if needed
201
188
  const allowDiskReload = true;
202
- // transfer cache here because we want to regen state asap
203
- const cloneOpts = {dontTransferCache: false};
204
- this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, cloneOpts, allowDiskReload).then(
189
+ this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, allowDiskReload).then(
205
190
  (headStateRegen) => this.blockStateCache.setHeadState(headStateRegen),
206
191
  (e) => this.logger.error("Error on head state regen", logCtx, e)
207
192
  );
@@ -224,7 +209,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
224
209
  this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState});
225
210
 
226
211
  // First attempt to fetch the state from caches before queueing
227
- const cachedState = this.getPreStateSync(block, opts);
212
+ const cachedState = this.getPreStateSync(block);
228
213
 
229
214
  if (cachedState !== null) {
230
215
  return cachedState;
@@ -243,7 +228,7 @@ export class QueuedStateRegenerator implements IStateRegenerator {
243
228
  this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState});
244
229
 
245
230
  // First attempt to fetch the state from cache before queueing
246
- const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp), opts);
231
+ const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp));
247
232
  if (checkpointState) {
248
233
  return checkpointState;
249
234
  }
@@ -271,22 +256,18 @@ export class QueuedStateRegenerator implements IStateRegenerator {
271
256
  return this.jobQueue.push({key: "getBlockSlotState", args: [blockRoot, slot, opts, rCaller]});
272
257
  }
273
258
 
274
- async getState(
275
- stateRoot: RootHex,
276
- rCaller: RegenCaller,
277
- opts: StateRegenerationOpts = {dontTransferCache: true}
278
- ): Promise<CachedBeaconStateAllForks> {
259
+ async getState(stateRoot: RootHex, rCaller: RegenCaller): Promise<CachedBeaconStateAllForks> {
279
260
  this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState});
280
261
 
281
262
  // First attempt to fetch the state from cache before queueing
282
- const state = this.blockStateCache.get(stateRoot, opts);
263
+ const state = this.blockStateCache.get(stateRoot);
283
264
  if (state) {
284
265
  return state;
285
266
  }
286
267
 
287
268
  // The state is not immediately available in the cache, enqueue the job
288
269
  this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState});
289
- return this.jobQueue.push({key: "getState", args: [stateRoot, rCaller, opts]});
270
+ return this.jobQueue.push({key: "getState", args: [stateRoot, rCaller]});
290
271
  }
291
272
 
292
273
  private jobQueueProcessor = async (regenRequest: RegenRequest): Promise<CachedBeaconStateAllForks> => {
@@ -78,7 +78,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
78
78
  }
79
79
 
80
80
  // Otherwise, get the state normally.
81
- return this.getState(parentBlock.stateRoot, regenCaller, opts, allowDiskReload);
81
+ return this.getState(parentBlock.stateRoot, regenCaller, allowDiskReload);
82
82
  }
83
83
 
84
84
  /**
@@ -124,8 +124,8 @@ export class StateRegenerator implements IStateRegeneratorInternal {
124
124
  const {checkpointStateCache} = this.modules;
125
125
  const epoch = computeEpochAtSlot(slot);
126
126
  const latestCheckpointStateCtx = allowDiskReload
127
- ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, opts)
128
- : checkpointStateCache.getLatest(blockRoot, epoch, opts);
127
+ ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch)
128
+ : checkpointStateCache.getLatest(blockRoot, epoch);
129
129
 
130
130
  // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch
131
131
  // or needs to have empty slots processed until the requested epoch
@@ -136,7 +136,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
136
136
  // Otherwise, use the fork choice to get the stateRoot from block at the checkpoint root
137
137
  // regenerate that state,
138
138
  // then process empty slots until the requested epoch
139
- const blockStateCtx = await this.getState(block.stateRoot, regenCaller, opts, allowDiskReload);
139
+ const blockStateCtx = await this.getState(block.stateRoot, regenCaller, allowDiskReload);
140
140
  return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, regenCaller, opts);
141
141
  }
142
142
 
@@ -148,23 +148,15 @@ export class StateRegenerator implements IStateRegeneratorInternal {
148
148
  async getState(
149
149
  stateRoot: RootHex,
150
150
  caller: RegenCaller,
151
- opts?: StateRegenerationOpts,
152
151
  // internal option, don't want to expose to external caller
153
152
  allowDiskReload = false
154
153
  ): Promise<CachedBeaconStateAllForks> {
155
154
  // Trivial case, state at stateRoot is already cached
156
- const cachedStateCtx = this.modules.blockStateCache.get(stateRoot, opts);
155
+ const cachedStateCtx = this.modules.blockStateCache.get(stateRoot);
157
156
  if (cachedStateCtx) {
158
157
  return cachedStateCtx;
159
158
  }
160
159
 
161
- // in block gossip validation (getPreState() call), dontTransferCache is specified as true because we only want to transfer cache in verifyBlocksStateTransitionOnly()
162
- // but here we want to process blocks as fast as possible so force to transfer cache in this case
163
- if (opts && allowDiskReload) {
164
- // if there is no `opts` specified, it already means "false"
165
- opts.dontTransferCache = false;
166
- }
167
-
168
160
  // Otherwise we have to use the fork choice to traverse backwards, block by block,
169
161
  // searching the state caches
170
162
  // then replay blocks forward to the desired stateRoot
@@ -179,7 +171,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
179
171
  const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller});
180
172
  // iterateAncestorBlocks only returns ancestor blocks, not the block itself
181
173
  for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) {
182
- state = this.modules.blockStateCache.get(b.stateRoot, opts);
174
+ state = this.modules.blockStateCache.get(b.stateRoot);
183
175
  if (state) {
184
176
  break;
185
177
  }
@@ -187,8 +179,8 @@ export class StateRegenerator implements IStateRegeneratorInternal {
187
179
  if (!lastBlockToReplay) continue;
188
180
  const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1);
189
181
  state = allowDiskReload
190
- ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, opts)
191
- : checkpointStateCache.getLatest(b.blockRoot, epoch, opts);
182
+ ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch)
183
+ : checkpointStateCache.getLatest(b.blockRoot, epoch);
192
184
  if (state) {
193
185
  break;
194
186
  }
@@ -255,6 +247,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
255
247
  try {
256
248
  // Only advances state trusting block's signture and hashes.
257
249
  // We are only running the state transition to get a specific state's data.
250
+ // stateTransition() does the clone() inside, transfer cache to make the regen faster
258
251
  state = stateTransition(
259
252
  state,
260
253
  block,
@@ -265,6 +258,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
265
258
  verifyStateRoot: false,
266
259
  verifyProposer: false,
267
260
  verifySignatures: false,
261
+ dontTransferCache: false,
268
262
  },
269
263
  this.modules
270
264
  );
@@ -390,8 +384,8 @@ export async function processSlotsToNearestCheckpoint(
390
384
  const checkpointState = postState;
391
385
  const cp = getCheckpointFromState(checkpointState);
392
386
  checkpointStateCache.add(cp, checkpointState);
393
- // consumers should not mutate or get the transfered cache
394
- emitter?.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true));
387
+ // consumers should not mutate state ever
388
+ emitter?.emit(ChainEvent.checkpoint, cp, checkpointState);
395
389
 
396
390
  if (count >= 1) {
397
391
  // in normal condition, we only process 1 epoch so never reach this