@lodestar/beacon-node 1.50.0-dev.45e4260336 → 1.50.0-dev.4b00f7bf39

package/src/api/impl/validator/index.ts

@@ -5,6 +5,8 @@ import {
   BUILDER_INDEX_SELF_BUILD,
   ForkName,
   ForkPostBellatrix,
+  ForkPostDeneb,
+  ForkPreDeneb,
   ForkPreGloas,
   ForkSeq,
   GENESIS_SLOT,
@@ -111,14 +113,13 @@ import {
  * Post this time, race execution and builder to pick whatever resolves first.
  *
  * Empirically the builder block resolves in ~1 second, and execution block resolves in <500 ms.
- * A cutoff of 2 seconds gives enough time and if there are unexpected delays it ensures we publish
- * in time as proposals post 4 seconds into the slot will likely be orphaned due to proposer boost reorg.
+ * A cutoff of 1667 BPS (2 seconds for 12s slots) gives enough time and if there are unexpected delays
+ * it ensures we publish in time as proposals after the attestation deadline will likely be orphaned
+ * due to proposer boost reorg.
  *
  * TODO GLOAS: re-evaluate cutoff timing due to attestation deadline changes in gloas
  */
-const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000;
-/** Overall timeout for execution and block production apis */
-const BLOCK_PRODUCTION_RACE_TIMEOUT_MS = 12_000;
+const BLOCK_PRODUCTION_RACE_CUTOFF_BPS = 1667;
 /** Rejection message of the bid block branch when there is no viable bid to commit to */
 const NO_BID_AVAILABLE = "No builder bid available";
 
@@ -167,8 +168,8 @@ function compareBidCandidates(a: BidCandidate, b: BidCandidate): number {
 }
 
 type ProduceBlockContentsRes = {executionPayloadValue: Wei; consensusBlockValue: Wei} & {
-  data: BlockContents;
-  version: ForkName;
+  data: BlockContents<ForkPreGloas>;
+  version: ForkPreGloas;
 };
 type ProduceBlindedBlockRes = {executionPayloadValue: Wei; consensusBlockValue: Wei} & {
   data: BlindedBeaconBlock;
@@ -301,8 +302,8 @@ export function getValidatorApi(
    */
   function msToNextEpoch(): number {
     const nextEpoch = chain.clock.currentEpoch + 1;
-    const secPerEpoch = (SLOTS_PER_EPOCH * config.SLOT_DURATION_MS) / 1000;
-    const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch;
+    const nextEpochSlot = computeStartSlotAtEpoch(nextEpoch);
+    const nextEpochStartSec = computeTimeAtSlot(config, nextEpochSlot, chain.genesisTime);
     return nextEpochStartSec * 1000 - Date.now();
   }
 
@@ -511,6 +512,10 @@ export function getValidatorApi(
       parentBlock: ProtoBlock;
     }
   ): Promise<ProduceBlockContentsRes & {shouldOverrideBuilder?: boolean}> {
+    const version = config.getForkName(slot);
+    if (isForkPostGloas(version)) {
+      throw Error(`produceEngineBlockContents called for post-gloas fork=${version}`);
+    }
     const source = ProducedBlockSource.engine;
     metrics?.blockProductionRequests.inc({source});
 
@@ -525,7 +530,6 @@ export function getValidatorApi(
         feeRecipient,
         commonBlockBodyPromise,
       });
-      const version = config.getForkName(block.slot);
       if (strictFeeRecipientCheck && feeRecipient && isForkPostBellatrix(version)) {
         const blockFeeRecipient = toHex((block as bellatrix.BeaconBlock).body.executionPayload.feeRecipient);
         if (blockFeeRecipient !== feeRecipient) {
@@ -563,7 +567,7 @@ export function getValidatorApi(
 
         return {
           data: {
-            block,
+            block: block as BeaconBlock<ForkPostDeneb & ForkPreGloas>,
             blobs: blobsBundle.blobs,
             kzgProofs: blobsBundle.proofs,
           },
@@ -574,7 +578,13 @@ export function getValidatorApi(
         };
       }
 
-      return {data: {block}, version, executionPayloadValue, consensusBlockValue, shouldOverrideBuilder};
+      return {
+        data: {block: block as BeaconBlock<ForkPreDeneb>},
+        version,
+        executionPayloadValue,
+        consensusBlockValue,
+        shouldOverrideBuilder,
+      };
     } finally {
       if (timer) timer({source});
     }
@@ -680,17 +690,21 @@ export function getValidatorApi(
     });
 
     // Calculate cutoff time based on start of the slot
-    const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - chain.clock.msFromSlot(slot));
+    const cutoffMs = Math.max(
+      0,
+      config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_BPS) - chain.clock.msFromSlot(slot)
+    );
+    const blockProductionTimeoutMs = config.getSlotDurationMs(fork);
 
     logger.verbose("Block production race (builder vs execution) starting", {
       ...loggerContext,
       cutoffMs,
-      timeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+      timeoutMs: blockProductionTimeoutMs,
     });
 
     const blockProductionRacePromise = resolveOrRacePromises([builderPromise, enginePromise], {
       resolveTimeoutMs: cutoffMs,
-      raceTimeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+      raceTimeoutMs: blockProductionTimeoutMs,
       signal: controller.signal,
     });
 
@@ -878,6 +892,10 @@ export function getValidatorApi(
 
   return {
     async produceBlockV3({slot, randaoReveal, graffiti, skipRandaoVerification, builderBoostFactor, ...opts}) {
+      const forkName = config.getForkName(slot);
+      if (isForkPostGloas(forkName)) {
+        throw new ApiError(400, `produceBlockV3 not supported for post-gloas fork=${forkName}, use produceBlockV4`);
+      }
       const {data, ...meta} = await produceEngineOrBuilderBlock(
         slot,
         randaoReveal,
@@ -1095,7 +1113,10 @@ export function getValidatorApi(
 
       // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
       // not delay the proposal when a builder bid block is available (and vice versa)
-      const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - chain.clock.msFromSlot(slot));
+      const cutoffMs = Math.max(
+        0,
+        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_BPS) - chain.clock.msFromSlot(slot)
+      );
 
       // use abort controller to stop waiting for the bid block if the engine block will be selected
       const controller = new AbortController();
@@ -1123,7 +1144,7 @@ export function getValidatorApi(
 
       const [engineResult, bidBlockResult] = await resolveOrRacePromises([enginePromise, bidBlockPromise], {
         resolveTimeoutMs: cutoffMs,
-        raceTimeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+        raceTimeoutMs: config.getSlotDurationMs(fork),
         signal: controller.signal,
       });
 
@@ -1417,7 +1438,7 @@ export function getValidatorApi(
       // Spec: set payload_present only if the envelope was seen before get_payload_due_ms()
       // into the slot. Use the envelope's own arrival time (getPayloadEnvelopeSource), not
       // the input's creation time.
-      const payloadDueSec = config.getPayloadDueMs() / 1000;
+      const payloadDueSec = config.getPayloadDueMs(fork) / 1000;
       const payloadSeenSec =
         payloadInput?.hasPayloadEnvelope() === true
           ? chain.clock.secFromSlot(slot, payloadInput.getPayloadEnvelopeSource().seenTimestampSec)
@@ -1494,8 +1515,9 @@ export function getValidatorApi(
       const currentEpoch = currentEpochWithDisparity();
       const nextEpoch = currentEpoch + 1;
       const startSlot = computeStartSlotAtEpoch(epoch);
+      const prepareFork = config.getForkName(computeStartSlotAtEpoch(nextEpoch) - 1);
       const prepareNextSlotLookAheadMs =
-        config.SLOT_DURATION_MS - config.getSlotComponentDurationMs(PREPARE_NEXT_SLOT_BPS);
+        config.getSlotDurationMs(prepareFork) - config.getSlotComponentDurationMs(prepareFork, PREPARE_NEXT_SLOT_BPS);
       const toNextEpochMs = msToNextEpoch();
       const nearNextEpoch = toNextEpochMs < prepareNextSlotLookAheadMs;
       // Post-Fulu the proposer lookahead is deterministic and known a full epoch ahead, so
@@ -1932,6 +1954,17 @@ export function getValidatorApi(
     },
 
     async prepareBeaconCommitteeSubnet({subscriptions}) {
+      // The subscribing validators use this node for duties regardless of its sync state and
+      // the subscription must not fail on the tracking, e.g. if the finalized state is unavailable
+      try {
+        await chain.updateAttachedValidators(
+          chain.clock.currentEpoch,
+          Array.from(new Set(subscriptions.map(({validatorIndex}) => validatorIndex)))
+        );
+      } catch (e) {
+        logger.warn("Error tracking attached validators", {count: subscriptions.length}, e as Error);
+      }
+
       notWhileSyncing(chain, sync.state);
 
       await network.prepareBeaconCommitteeSubnets(
@@ -1995,6 +2028,13 @@ export function getValidatorApi(
     },
 
     async prepareBeaconProposer({proposers}) {
+      if (isForkPostGloas(config.getForkName(chain.clock.currentSlot))) {
+        throw new ApiError(
+          410,
+          "prepareBeaconProposer is no longer supported from gloas, submit signed proposer preferences via POST /eth/v1/validator/proposer_preferences instead"
+        );
+      }
+
       await chain.updateBeaconProposerData(chain.clock.currentEpoch, proposers);
     },
 
@@ -2029,6 +2069,13 @@ export function getValidatorApi(
     },
 
     async registerValidator({registrations}) {
+      if (isForkPostGloas(config.getForkName(chain.clock.currentSlot))) {
+        throw new ApiError(
+          410,
+          "registerValidator is no longer supported from gloas, submit builder preferences via POST /eth/v1/validator/builder_preferences instead"
+        );
+      }
+
       if (!chain.executionBuilder) {
         throw Error("External builder not configured");
       }
@@ -2064,11 +2111,14 @@ export function getValidatorApi(
 
     async submitProposerPreferences({signedProposerPreferences}) {
       const failures: FailureList = [];
+      // Preferences submitted here are signed by validators using this node, gossiped ones are not
+      const attachedValidatorIndices: ValidatorIndex[] = [];
 
       await Promise.all(
         signedProposerPreferences.map(async (signed, i) => {
           try {
             await validateGossipProposerPreferences(chain, signed);
+            attachedValidatorIndices.push(signed.message.validatorIndex);
 
             chain.proposerPreferencesPool.add(signed);
             await network.publishProposerPreferences(signed);
@@ -2084,6 +2134,7 @@ export function getValidatorApi(
             };
 
             if (e instanceof ProposerPreferencesError && e.type.code === ProposerPreferencesErrorCode.ALREADY_KNOWN) {
+              attachedValidatorIndices.push(signed.message.validatorIndex);
               logger.debug("Ignoring known signed proposer preferences", logCtx);
               return;
             }
@@ -2097,6 +2148,14 @@ export function getValidatorApi(
         })
       );
 
+      if (attachedValidatorIndices.length > 0) {
+        try {
+          await chain.updateAttachedValidators(chain.clock.currentEpoch, attachedValidatorIndices);
+        } catch (e) {
+          logger.warn("Error tracking attached validators", {count: attachedValidatorIndices.length}, e as Error);
+        }
+      }
+
       if (failures.length > 0) {
         throw new IndexedError("Error processing signed proposer preferences", failures);
       }

package/src/chain/ColumnReconstructionTracker.ts

@@ -42,16 +42,11 @@ export class ColumnReconstructionTracker {
   /** Track if a reconstruction attempt is in-flight */
   running = false;
 
-  private readonly minDelayMs: number;
-  private readonly maxDelayMs: number;
-
   constructor(init: ColumnReconstructionTrackerInit) {
     this.logger = init.logger;
     this.emitter = init.emitter;
     this.metrics = init.metrics;
     this.config = init.config;
-    this.minDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MIN_BPS);
-    this.maxDelayMs = this.config.getSlotComponentDurationMs(RECONSTRUCTION_DELAY_MAX_BPS);
   }
 
   triggerColumnReconstruction(input: BlockInputColumns | PayloadEnvelopeInput): void {
@@ -67,7 +62,10 @@ export class ColumnReconstructionTracker {
     // just that it has been triggered for this block root.
     this.running = true;
     this.lastBlockRootHex = input.blockRootHex;
-    const delay = this.minDelayMs + Math.random() * (this.maxDelayMs - this.minDelayMs);
+    const fork = this.config.getForkName(input.slot);
+    const minDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MIN_BPS);
+    const maxDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MAX_BPS);
+    const delay = minDelayMs + Math.random() * (maxDelayMs - minDelayMs);
     sleep(delay)
       .then(() => {
         const logCtx = {slot: input.slot, root: input.blockRootHex};

package/src/chain/archiveStore/archiveStore.ts

@@ -113,7 +113,9 @@ export class ArchiveStore {
           this.logger,
           this.metrics,
           this.opts.anchorState.finalizedCheckpoint.epoch,
-          this.chain.clock.currentEpoch
+          this.chain.clock.currentEpoch,
+          this.statePruneFromSlot,
+          this.advanceEarliestAvailableSlot
         ),
         () => this.logger.info("Still pruning historical data, please wait..."),
         30_000,
@@ -122,26 +124,6 @@ export class ArchiveStore {
       this.statePruneFromSlot = stateCutoffSlot;
     }
 
-    // Initialize earliestAvailableSlot from the earliest block actually retained (after any pruning
-    // above), rather than the anchor state slot set in the constructor. On an in-place restart the DB
-    // still holds finalized history below the anchor; leaving the value at the anchor makes the node
-    // reject by_range requests (beacon_blocks, execution_payload_envelopes and data_column_sidecars
-    // all gate on this slot) for data it still retains. An empty archive (e.g. a fresh checkpoint
-    // sync with nothing below the anchor) leaves the constructor's anchor slot in place.
-    const earliestArchivedBlockSlot = await this.db.blockArchive.firstKey();
-    if (earliestArchivedBlockSlot != null) {
-      const oldEarliestAvailableSlot = this.chain.earliestAvailableSlot;
-      this.chain.earliestAvailableSlot = earliestArchivedBlockSlot;
-      this.logger.verbose("Initialized earliestAvailableSlot from retained block archive", {
-        oldEarliestAvailableSlot,
-        newEarliestAvailableSlot: earliestArchivedBlockSlot,
-      });
-    } else {
-      this.logger.verbose("Empty block archive on init, keeping anchor earliestAvailableSlot", {
-        earliestAvailableSlot: this.chain.earliestAvailableSlot,
-      });
-    }
-
     if (this.opts.serveHistoricalState) {
       this.historicalStateRegen = await HistoricalStateRegen.init({
         opts: {
@@ -158,6 +140,18 @@ export class ArchiveStore {
     }
   }
 
+  /**
+   * The CLI owns the initial value; this only advances it during pruneHistory.
+   * Persisted before any deletion so a crash mid-prune never leaves the floor below deleted data.
+   */
+  private advanceEarliestAvailableSlot = async (blockCutoffSlot: Slot): Promise<void> => {
+    const earliestAvailableSlot = Math.max(this.chain.earliestAvailableSlot, blockCutoffSlot);
+    if (earliestAvailableSlot !== this.chain.earliestAvailableSlot) {
+      await this.db.earliestAvailableSlot.set(earliestAvailableSlot);
+      this.chain.earliestAvailableSlot = earliestAvailableSlot;
+    }
+  };
+
   async close(): Promise<void> {
     await this.historicalStateRegen?.close();
   }
@@ -259,17 +253,17 @@ export class ArchiveStore {
 
       if (this.opts.pruneHistory) {
         timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
-        const {blockCutoffSlot, stateCutoffSlot} = await pruneHistory(
+        const {stateCutoffSlot} = await pruneHistory(
           this.chain.config,
           this.db,
           this.logger,
           this.metrics,
           finalizedEpoch,
           this.chain.clock.currentEpoch,
-          this.statePruneFromSlot
+          this.statePruneFromSlot,
+          this.advanceEarliestAvailableSlot
         );
         this.statePruneFromSlot = stateCutoffSlot;
-        this.chain.earliestAvailableSlot = Math.max(this.chain.earliestAvailableSlot, blockCutoffSlot);
         timer?.({source: ArchiveStoreTask.PruneHistory});
       }
 

package/src/chain/archiveStore/utils/pruneHistory.ts

@@ -12,7 +12,8 @@ export async function pruneHistory(
   metrics: Metrics | null | undefined,
   finalizedEpoch: Epoch,
   currentEpoch: Epoch,
-  statePruneFromSlot: Slot = 0
+  statePruneFromSlot: Slot,
+  beforePrune: (blockCutoffSlot: Slot) => Promise<void>
 ): Promise<{blockCutoffSlot: Slot; stateCutoffSlot: Slot}> {
   const blockCutoffEpoch = Math.min(
     // set by config, with underflow protection
@@ -51,6 +52,8 @@ export async function pruneHistory(
     stateSlots: prettyPrintIndices(states),
   });
 
+  await beforePrune(blockCutoffSlot);
+
   const step1 = metrics?.pruneHistory.pruneKeys.startTimer();
   await Promise.all([
     // ->

package/src/chain/beaconProposerCache.ts

@@ -6,7 +6,7 @@ const PROPOSER_PRESERVE_EPOCHS = 2;
 export type ProposerPreparationData = routes.validator.ProposerPreparationData;
 
 export class BeaconProposerCache {
-  private readonly feeRecipientByValidatorIndex: Map<number, {epoch: Epoch; feeRecipient: string}>;
+  private readonly feeRecipientByValidatorIndex: Map<number, {epoch: Epoch; feeRecipient?: string}>;
   constructor(readonly opts: {suggestedFeeRecipient: string}) {
     this.feeRecipientByValidatorIndex = new Map();
   }
@@ -15,6 +15,24 @@ export class BeaconProposerCache {
     this.feeRecipientByValidatorIndex.set(validatorIndex, {epoch, feeRecipient});
   }
 
+  /**
+   * Track a validator that uses this node for duties, keeping the fee recipient it may have
+   * registered. Returns true if the validator was not tracked before.
+   */
+  track(epoch: Epoch, validatorIndex: number): boolean {
+    const entry = this.feeRecipientByValidatorIndex.get(validatorIndex);
+    if (entry !== undefined) {
+      entry.epoch = epoch;
+      return false;
+    }
+    this.feeRecipientByValidatorIndex.set(validatorIndex, {epoch});
+    return true;
+  }
+
+  has(validatorIndex: number): boolean {
+    return this.feeRecipientByValidatorIndex.has(validatorIndex);
+  }
+
   prune(epoch: Epoch): void {
     // This is not so optimized function, but could maintain a 2d array may be?
     for (const [validatorIndex, feeRecipientEntry] of this.feeRecipientByValidatorIndex.entries()) {

package/src/chain/blocks/blockInput/blockInput.ts

@@ -210,7 +210,7 @@ export class BlockInputPreData extends AbstractBlockInput<ForkPreDeneb, null> {
     this.blockPromise.resolve(state.block);
   }
 
-  static createFromBlock(props: AddBlock & CreateBlockInputMeta): BlockInputPreData {
+  static createFromBlock(props: AddBlock<ForkPreDeneb> & CreateBlockInputMeta): BlockInputPreData {
     const init: BlockInputInit = {
       daOutOfRange: props.daOutOfRange,
       timeCreated: props.seenTimestampSec,

package/src/chain/blocks/importBlock.ts

@@ -22,6 +22,7 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
+  getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -44,7 +45,7 @@ import {isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
-import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
+import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS} from "../reprocess.js";
 import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
@@ -129,11 +130,17 @@ export async function importBlock(
 
   // Should compute checkpoint balances before forkchoice.onBlock
   this.checkpointBalancesCache.processState(blockRootHex, postState);
-  if (fork >= ForkSeq.gloas) {
+  if (isGloasBeaconBlock(block.message)) {
+    // A gloas block has no payload of its own, it inherits the status of the payload it builds on,
+    // i.e. the parent variant matching the bid's parent block hash. The parent's default (PENDING)
+    // variant does not reflect a SYNCING parent payload, and onBlock would then mark that payload VALID.
     const parentRootHex = toRootHex(block.message.parentRoot);
-    const parentBlock = this.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+    const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
+    const parentBlock = this.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex);
     if (parentBlock === null) {
-      throw Error(`Parent block not found in forkChoice, parentRoot=${parentRootHex}`);
+      throw Error(
+        `Parent block not found in forkChoice, parentRoot=${parentRootHex} parentBlockHash=${parentBlockHashHex}`
+      );
     }
     if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
       throw Error(`Parent block has invalid execution status, parentRoot=${parentRootHex}`);
@@ -328,7 +335,8 @@ export async function importBlock(
       this.metrics.headSlot.set(newHead.slot);
       // Only track "recent" blocks. Otherwise sync can distort this metrics heavily.
       // We want to track recent blocks coming from gossip, unknown block sync, and API.
-      if (delaySec < (SLOTS_PER_EPOCH * this.config.SLOT_DURATION_MS) / 1000) {
+      const headFork = this.config.getForkName(newHead.slot);
+      if (delaySec < (SLOTS_PER_EPOCH * this.config.getSlotDurationMs(headFork)) / 1000) {
         this.metrics.importBlock.elapsedTimeTillBecomeHead.observe(delaySec);
         const cutOffSec = this.config.getAttestationDueMs(this.config.getForkName(blockSlot)) / 1000;
         if (delaySec > cutOffSec) {
@@ -400,9 +408,8 @@ export async function importBlock(
     const proposalSlot = blockSlot + 1;
     try {
       const proposerIndex = postState.getBeaconProposer(proposalSlot);
-      const feeRecipient = this.beaconProposerCache.get(proposerIndex);
 
-      if (feeRecipient) {
+      if (this.beaconProposerCache.has(proposerIndex)) {
         // We would set this to true if
         //  1) This is a gossip block
         //  2) We are proposer of next slot
@@ -580,7 +587,7 @@ export async function importBlock(
     this.validatorMonitor?.registerSyncAggregateInBlock(
       blockEpoch,
       (block as altair.SignedBeaconBlock).message.body.syncAggregate,
-      fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
+      fullyVerifiedBlock.postState.currentSyncCommitteeValidatorIndices
     );
   }
 
@@ -594,7 +601,10 @@ export async function importBlock(
     }
   }
 
-  const advancedSlot = this.clock.slotWithFutureTolerance(REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC);
+  const reprocessGuardSec =
+    this.config.getSlotComponentDurationMs(this.config.getForkName(blockSlot), REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS) /
+    1000;
+  const advancedSlot = this.clock.slotWithFutureTolerance(reprocessGuardSec);
 
   // Gossip blocks need to be imported as soon as possible, waiting attestations could be processed
   // in the next event loop. See https://github.com/ChainSafe/lodestar/issues/4789
@@ -657,8 +667,12 @@ export function addAttestationPostElectra(
   } else {
     const attSlot = attestation.data.slot;
     const attEpoch = computeEpochAtSlot(attSlot);
-    const decisionRoot = state.getShufflingDecisionRoot(attEpoch);
-    const committees = this.shufflingCache.getBeaconCommittees(attEpoch, decisionRoot, attSlot, committeeIndices);
+    // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+    // we should have the shuffling inside ShufflingCache most of the time
+    const shuffling =
+      this.shufflingCache.getSync(attEpoch, state.getShufflingDecisionRoot(attEpoch)) ??
+      state.getShufflingAtEpoch(attEpoch);
+    const committees = getBeaconCommittees(shuffling, attSlot, committeeIndices);
     const aggregationBools = attestation.aggregationBits.toBoolArray();
     let offset = 0;
     for (let i = 0; i < committees.length; i++) {

package/src/chain/blocks/verifyBlock.ts

@@ -4,6 +4,7 @@ import {
   DataAvailabilityStatus,
   IBeaconStateView,
   computeEpochAtSlot,
+  getIndexedAttestation,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
 import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
@@ -81,10 +82,6 @@ export async function verifyBlocksInEpoch(
       throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error});
     });
 
-  // in forky condition, make sure to populate ShufflingCache with regened state
-  // otherwise it may fail to get indexed attestations from shuffling cache later
-  this.shufflingCache.processState(preState0);
-
   if (!preState0.isStateValidatorsNodesPopulated()) {
     this.logger.verbose("verifyBlocksInEpoch preState0 SSZ cache stats", {
       slot: preState0.slot,
@@ -117,8 +114,12 @@ export async function verifyBlocksInEpoch(
     for (const [i, block] of blocks.entries()) {
       indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) => {
         const attEpoch = computeEpochAtSlot(attestation.data.slot);
-        const decisionRoot = preState0.getShufflingDecisionRoot(attEpoch);
-        return this.shufflingCache.getIndexedAttestation(attEpoch, decisionRoot, fork, attestation);
+        // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+        // we should have the shuffling inside ShufflingCache most of the time
+        const shuffling =
+          this.shufflingCache.getSync(attEpoch, preState0.getShufflingDecisionRoot(attEpoch)) ??
+          preState0.getShufflingAtEpoch(attEpoch);
+        return getIndexedAttestation(shuffling, fork, attestation);
       });
     }