@lodestar/beacon-node 1.50.1 → 1.51.0-dev.0586e6e617

package/src/network/reqresp/handlers/inclusionListsByIndices.ts

@@ -0,0 +1,44 @@
+import {PeerId} from "@libp2p/interface";
+import {ResponseOutgoing} from "@lodestar/reqresp";
+import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {heze, ssz} from "@lodestar/types";
+import {toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../../chain/index.js";
+import {prettyPrintPeerId} from "../../util.js";
+
+/**
+ * Only lists that passed gossip validation are stored and the store skips equivocators, so the spec's
+ * "SHOULD NOT respond with lists that fail gossip validation or are from equivocators" holds by construction.
+ */
+export async function* onInclusionListsByIndices(
+  requestBody: heze.InclusionListsByIndicesRequest,
+  chain: IBeaconChain,
+  peerId: PeerId,
+  peerClient: string
+): AsyncIterable<ResponseOutgoing> {
+  const {slot, dependentRoot, indices} = requestBody;
+
+  const minimumRequestSlot = Math.max(
+    chain.clock.currentSlot - chain.config.MIN_SLOTS_FOR_INCLUSION_LISTS_REQUESTS,
+    computeStartSlotAtEpoch(chain.config.HEZE_FORK_EPOCH)
+  );
+  if (slot < minimumRequestSlot) {
+    chain.logger.debug("Cannot serve InclusionListsByIndices: slot below minimum request slot", {
+      slot,
+      minimumRequestSlot,
+      peer: prettyPrintPeerId(peerId),
+      client: peerClient,
+    });
+    return;
+  }
+
+  const signedInclusionLists = chain.inclusionListStore.getByIndices(slot, toRootHex(dependentRoot), indices);
+
+  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot));
+  for (const signedInclusionList of signedInclusionLists.slice(0, chain.config.MAX_REQUEST_INCLUSION_LIST)) {
+    yield {
+      data: ssz.heze.SignedInclusionList.serialize(signedInclusionList),
+      boundary,
+    };
+  }
+}

package/src/network/reqresp/handlers/index.ts

@@ -19,6 +19,7 @@ import {onDataColumnSidecarsByRange} from "./dataColumnSidecarsByRange.js";
 import {onDataColumnSidecarsByRoot} from "./dataColumnSidecarsByRoot.js";
 import {onExecutionPayloadEnvelopesByRange} from "./executionPayloadEnvelopesByRange.js";
 import {onExecutionPayloadEnvelopesByRoot} from "./executionPayloadEnvelopesByRoot.js";
+import {onInclusionListsByIndices} from "./inclusionListsByIndices.js";
 import {onLightClientBootstrap} from "./lightClientBootstrap.js";
 import {onLightClientFinalityUpdate} from "./lightClientFinalityUpdate.js";
 import {onLightClientOptimisticUpdate} from "./lightClientOptimisticUpdate.js";
@@ -142,6 +143,10 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.LightClientFinalityUpdate]: () => onLightClientFinalityUpdate(chain),
     [ReqRespMethod.LightClientOptimisticUpdate]: () => onLightClientOptimisticUpdate(chain),
+    [ReqRespMethod.InclusionListsByIndices]: (req, peerId, peerClient) => {
+      const body = deserializeRequestBody(ssz.heze.InclusionListsByIndicesRequest, req.data);
+      return onInclusionListsByIndices(body, chain, peerId, peerClient);
+    },
   };
 
   return (method) => handlers[method];

package/src/network/reqresp/protocols.ts

@@ -113,6 +113,12 @@ export const ExecutionPayloadEnvelopesByRange = toProtocol({
   contextBytesType: ContextBytesType.ForkDigest,
 });
 
+export const InclusionListsByIndices = toProtocol({
+  method: ReqRespMethod.InclusionListsByIndices,
+  version: Version.V1,
+  contextBytesType: ContextBytesType.ForkDigest,
+});
+
 export const LightClientBootstrap = toProtocol({
   method: ReqRespMethod.LightClientBootstrap,
   version: Version.V1,

package/src/network/reqresp/rateLimit.ts

@@ -107,12 +107,16 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
   [ReqRespMethod.LightClientFinalityUpdate]: {
     // Finality updates should not be requested more than once per epoch.
     // Allow 2 per slot and a very safe bound until there's more testing of real usage.
-    byPeer: {quota: 2, quotaTimeMs: 12_000},
+    byPeer: {quota: 2, quotaTimeMs: config.getSlotDurationMs(fork)},
+  },
+  [ReqRespMethod.InclusionListsByIndices]: {
+    // Peers only backfill what gossip missed, so a small per-slot allowance is enough
+    byPeer: {quota: 5, quotaTimeMs: config.getSlotDurationMs(fork)},
   },
   [ReqRespMethod.LightClientOptimisticUpdate]: {
     // Optimistic updates should not be requested more than once per slot.
     // Allow 2 per slot and a very safe bound until there's more testing of real usage.
-    byPeer: {quota: 2, quotaTimeMs: 12_000},
+    byPeer: {quota: 2, quotaTimeMs: config.getSlotDurationMs(fork)},
   },
 });
 

package/src/network/reqresp/types.ts

@@ -16,6 +16,7 @@ import {
   deneb,
   fulu,
   gloas,
+  heze,
   phase0,
   ssz,
   sszTypesFor,
@@ -53,6 +54,7 @@ export enum ReqRespMethod {
   LightClientUpdatesByRange = "light_client_updates_by_range",
   LightClientFinalityUpdate = "light_client_finality_update",
   LightClientOptimisticUpdate = "light_client_optimistic_update",
+  InclusionListsByIndices = "inclusion_lists_by_indices",
 }
 
 // To typesafe events to network
@@ -74,6 +76,7 @@ export type RequestBodyByMethod = {
   [ReqRespMethod.LightClientUpdatesByRange]: altair.LightClientUpdatesByRange;
   [ReqRespMethod.LightClientFinalityUpdate]: null;
   [ReqRespMethod.LightClientOptimisticUpdate]: null;
+  [ReqRespMethod.InclusionListsByIndices]: heze.InclusionListsByIndicesRequest;
 };
 
 type ResponseBodyByMethod = {
@@ -96,6 +99,7 @@ type ResponseBodyByMethod = {
   [ReqRespMethod.LightClientUpdatesByRange]: LightClientUpdate;
   [ReqRespMethod.LightClientFinalityUpdate]: LightClientFinalityUpdate;
   [ReqRespMethod.LightClientOptimisticUpdate]: LightClientOptimisticUpdate;
+  [ReqRespMethod.InclusionListsByIndices]: heze.SignedInclusionList;
 };
 
 /** Request SSZ type for each method and ForkName */
@@ -126,6 +130,7 @@ export const requestSszTypeByMethod: (
   [ReqRespMethod.LightClientUpdatesByRange]: ssz.altair.LightClientUpdatesByRange,
   [ReqRespMethod.LightClientFinalityUpdate]: null,
   [ReqRespMethod.LightClientOptimisticUpdate]: null,
+  [ReqRespMethod.InclusionListsByIndices]: ssz.heze.InclusionListsByIndicesRequest,
 });
 
 export type ResponseTypeGetter<T> = (fork: ForkName, version: number) => Type<T>;
@@ -158,6 +163,7 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter<
   [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: () => ssz.gloas.SignedExecutionPayloadEnvelope,
   [ReqRespMethod.LightClientOptimisticUpdate]: (fork) =>
     sszTypesFor(onlyPostAltairFork(fork)).LightClientOptimisticUpdate,
+  [ReqRespMethod.InclusionListsByIndices]: () => ssz.heze.SignedInclusionList,
 };
 
 function onlyPostAltairFork(fork: ForkName): ForkPostAltair {

package/src/network/subnets/attnetsService.ts

@@ -3,6 +3,7 @@ import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {Epoch, Slot, SubnetID, ssz} from "@lodestar/types";
 import {Logger, MapDef} from "@lodestar/utils";
+import {HALF_SLOT_BPS} from "../../constants/index.js";
 import {ClockEvent, IClock} from "../../util/clock.js";
 import {NetworkCoreMetrics} from "../core/metrics.js";
 import {getActiveForkBoundaries} from "../forks.js";
@@ -163,9 +164,13 @@ export class AttnetsService implements IAttnetsService {
    */
   private onSlot = (clockSlot: Slot): void => {
     try {
-      setTimeout(() => {
-        this.onHalfSlot(clockSlot);
-      }, this.config.SLOT_DURATION_MS * 0.5);
+      const fork = this.config.getForkName(clockSlot);
+      setTimeout(
+        () => {
+          this.onHalfSlot(clockSlot);
+        },
+        this.config.getSlotComponentDurationMs(fork, HALF_SLOT_BPS)
+      );
 
       for (const [dutiedSlot, dutiedInfo] of this.aggregatorSlotSubnet.entries()) {
         if (dutiedSlot === clockSlot + this.opts.slotsToSubscribeBeforeAggregatorDuty) {

package/src/node/nodejs.ts

@@ -1,23 +1,29 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {hasher} from "@chainsafe/persistent-merkle-tree";
+import {BeaconApiMethods} from "@lodestar/api/beacon/server";
+import {BeaconConfig} from "@lodestar/config";
+import type {LoggerNode} from "@lodestar/logger/node";
 import {
   init as initNativeMetrics,
   registerLocalValidator as registerNativeLocalValidator,
   scrapeMetrics as scrapeNativeMetrics,
   unregisterLocalValidator as unregisterNativeLocalValidator,
-} from "@chainsafe/lodestar-z/metrics";
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
-import {hasher} from "@chainsafe/persistent-merkle-tree";
-import {BeaconApiMethods} from "@lodestar/api/beacon/server";
-import {BeaconConfig} from "@lodestar/config";
-import type {LoggerNode} from "@lodestar/logger/node";
+} from "@lodestar/native/metrics";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -212,6 +218,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos

package/src/node/notifier.ts

@@ -5,6 +5,7 @@ import {
   IBeaconStateView,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
+  computeTimeAtSlot,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
 import {Epoch} from "@lodestar/types";
@@ -140,21 +141,21 @@ export async function runNodeNotifier(modules: NodeNotifierModules): Promise<voi
 }
 
 function timeToNextHalfSlot(config: BeaconConfig, chain: IBeaconChain, isFirstTime: boolean): number {
-  const msPerSlot = config.SLOT_DURATION_MS;
-  const msPerHalfSlot = msPerSlot / 2;
-  const msFromGenesis = Date.now() - chain.genesisTime * 1000;
-  const msToNextSlot =
-    msFromGenesis < 0
-      ? // For future genesis time, calculate time left in the slot
-        -msFromGenesis % msPerSlot
-      : // For past genesis time, calculate time until the next slot
-        msPerSlot - (msFromGenesis % msPerSlot);
+  const currentSlot = chain.clock.currentSlot;
+  const nextSlotTimeSec = computeTimeAtSlot(config, currentSlot + 1, chain.genesisTime);
+  const nowMs = Date.now();
+
   if (isFirstTime) {
-    // at the 1st time we may miss middle of the current clock slot
-    return msToNextSlot > msPerHalfSlot ? msToNextSlot - msPerHalfSlot : msToNextSlot + msPerHalfSlot;
+    const currentSlotTimeSec = computeTimeAtSlot(config, currentSlot, chain.genesisTime);
+    const currentMidpointMs = ((currentSlotTimeSec + nextSlotTimeSec) / 2) * 1000;
+    if (currentMidpointMs > nowMs) {
+      return currentMidpointMs - nowMs;
+    }
   }
-  // after the 1st time always wait until middle of next clock slot
-  return msToNextSlot + msPerHalfSlot;
+
+  const nextSlotEndTimeSec = computeTimeAtSlot(config, currentSlot + 2, chain.genesisTime);
+  const nextMidpointMs = ((nextSlotTimeSec + nextSlotEndTimeSec) / 2) * 1000;
+  return Math.max(0, nextMidpointMs - nowMs);
 }
 
 function getHeadExecutionInfo(

package/src/node/options.ts

@@ -19,16 +19,15 @@ import {SyncOptions, defaultSyncOptions} from "../sync/options.js";
 
 // Re-export so the CLI doesn't need to depend on lodestar-api
 export {allNamespaces} from "../api/rest/index.js";
-
+export type {EngineApiMode};
 // Re-export to use as default values in CLI args
 export {
-  defaultExecutionEngineHttpOpts,
-  defaultExecutionBuilderHttpOpts,
-  engineApiModes,
   ArchiveMode,
   DEFAULT_ARCHIVE_MODE,
+  defaultExecutionBuilderHttpOpts,
+  defaultExecutionEngineHttpOpts,
+  engineApiModes,
 };
-export type {EngineApiMode};
 
 export interface IBeaconNodeOptions {
   api: ApiOptions;

package/src/sync/interface.ts

@@ -7,6 +7,7 @@ import {IBeaconDb} from "../db/index.js";
 import {Metrics} from "../metrics/index.js";
 import {INetwork} from "../network/index.js";
 import {SyncChainDebugState} from "./range/chain.js";
+
 export type {SyncChainDebugState};
 
 export type SyncingStatus = routes.node.SyncingStatus;

package/src/sync/sync.ts

@@ -1,13 +1,17 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Slot, Status} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
-import {GENESIS_SLOT} from "../constants/constants.js";
+import {GENESIS_EPOCH, GENESIS_SLOT} from "../constants/constants.js";
+import {GoodByeReasonCode} from "../constants/index.js";
+import {IBeaconDb} from "../db/index.js";
 import {ExecutionEngineState} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
+import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js";
 import {ClockEvent} from "../util/clock.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
+import {PeerIdStr} from "../util/peerId.js";
 import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
 import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
 import {SyncOptions} from "./options.js";
@@ -19,6 +23,7 @@ export class BeaconSync implements IBeaconSync {
   private readonly logger: Logger;
   private readonly network: INetwork;
   private readonly chain: IBeaconChain;
+  private readonly db: IBeaconDb;
   private readonly metrics: Metrics | null;
   private readonly opts: SyncOptions;
 
@@ -30,10 +35,11 @@ export class BeaconSync implements IBeaconSync {
   private readonly slotImportTolerance: Slot;
 
   constructor(opts: SyncOptions, modules: SyncModules) {
-    const {config, chain, metrics, network, logger} = modules;
+    const {config, chain, db, metrics, network, logger} = modules;
     this.opts = opts;
     this.network = network;
     this.chain = chain;
+    this.db = db;
     this.metrics = metrics;
     this.logger = logger;
     this.rangeSync = new RangeSync(modules, opts);
@@ -192,6 +198,11 @@ export class BeaconSync implements IBeaconSync {
    */
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
+    // Genesis and equal finalized epochs are already checked by assertPeerRelevance in the network worker
+    if (data.status.finalizedEpoch > GENESIS_EPOCH && data.status.finalizedEpoch < localStatus.finalizedEpoch) {
+      void this.verifyPastFinalizedCheckpoint(data.peer, data.status);
+    }
+
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
     this.logger.verbose("Peer sync type classified", {
       peer: data.peer,
@@ -216,6 +227,56 @@ export class BeaconSync implements IBeaconSync {
     this.updateSyncState();
   };
 
+  /**
+   * A behind peer's finalized checkpoint must be the canonical block in effect at the start slot of its epoch
+   */
+  async verifyPastFinalizedCheckpoint(peer: PeerIdStr, remote: Status): Promise<void> {
+    const startSlot = computeStartSlotAtEpoch(remote.finalizedEpoch);
+    const root = toRootHex(remote.finalizedRoot);
+    try {
+      let reason: string | null = null;
+      // After a finality jump, finalized blocks stay in fork choice until the archiver migrates them.
+      // Fork choice is pruned only after archiving, so a miss here means the db already has this range.
+      const canonicalBlock = this.chain.forkChoice.getCanonicalBlockClosestLteSlot(startSlot);
+      if (canonicalBlock !== null) {
+        if (canonicalBlock.blockRoot === root) return;
+        reason = "finalized_root_not_canonical";
+      } else {
+        const slot = await this.db.blockArchive.getSlotByRoot(remote.finalizedRoot);
+        if (slot !== null && slot > startSlot) {
+          reason = "finalized_root_after_epoch_start";
+        } else if (slot !== startSlot) {
+          const childSlot = await this.db.blockArchive.getSlotByParentRoot(remote.finalizedRoot);
+          if (childSlot !== null) {
+            if (childSlot <= startSlot) reason = "finalized_root_not_in_effect_at_epoch_start";
+          } else if (slot === null && startSlot > this.chain.earliestAvailableSlot) {
+            // childSlot == null too, neither the root nor a child is canonical
+            // or they are canonical and both pruned but peer's finalized epoch starts after earliestAvailableSlot
+            reason = "unknown_finalized_root";
+          }
+        }
+      }
+
+      if (reason !== null) {
+        this.logger.debug("Peer finalized checkpoint not in our chain", {
+          peer,
+          finalizedEpoch: remote.finalizedEpoch,
+          root,
+          reason,
+        });
+        this.metrics?.syncPeerFinalizedCheckpointMismatch.inc({reason});
+        this.network.reportPeer(peer, PeerAction.Fatal, "PeerFinalizedCheckpointMismatch");
+        await this.network.goodbyeAndDisconnect(peer, GoodByeReasonCode.IRRELEVANT_NETWORK);
+      }
+    } catch (e) {
+      this.logger.debug(
+        "Error verifying peer finalized checkpoint",
+        {peer, finalizedEpoch: remote.finalizedEpoch, root},
+        e as Error
+      );
+    }
+  }
+
   /**
    * Must be called by libp2p when a peer is removed from the peer manager
    */

package/src/sync/unknownBlock.ts

@@ -300,10 +300,19 @@ export class BlockInputSync {
 
   private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
     try {
-      this.addByPayloadInput(data.payloadInput, data.peer);
-      this.triggerUnknownBlockSearch();
-      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
-      this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      const isNewRoot = this.addByPayloadInput(data.payloadInput, data.peer);
+      const pendingPayload = this.pendingPayloads.get(data.payloadInput.blockRootHex);
+      if (
+        pendingPayload &&
+        pendingPayload.status !== PendingPayloadInputStatus.fetching &&
+        pendingPayload.status !== PendingPayloadInputStatus.processing
+      ) {
+        this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
+        this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
+        this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      }
     } catch (e) {
       this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e as Error);
     }
@@ -599,43 +608,44 @@ export class BlockInputSync {
     return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
   }
 
-  private addByPayloadInput = (
-    payloadInput: PayloadEnvelopeInput,
-    peerIdStr?: PeerIdStr,
-    envelope?: gloas.SignedExecutionPayloadEnvelope
-  ): void => {
+  private addByPayloadInput = (payloadInput: PayloadEnvelopeInput, peerIdStr?: PeerIdStr): boolean => {
     if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
       this.logger.debug("Ignoring payload input for known bad block root", {
         slot: payloadInput.slot,
         root: payloadInput.blockRootHex,
         peerIdStr,
       });
-      return;
+      return false;
     }
 
-    const pendingPayload = this.toPendingPayloadInput(
-      payloadInput,
-      this.pendingPayloads.get(payloadInput.blockRootHex),
-      envelope
-    );
+    let pendingPayload = this.pendingPayloads.get(payloadInput.blockRootHex);
+    const isNewRoot = pendingPayload === undefined;
+    // Envelope-only entries must go through reconciliation to validate their speculative envelope.
+    if (
+      !pendingPayload ||
+      pendingPayload.status === PendingPayloadInputStatus.pending ||
+      pendingPayload.status === PendingPayloadInputStatus.downloaded
+    ) {
+      pendingPayload = this.toPendingPayloadInput(payloadInput, pendingPayload);
+      this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
+
+      this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
+        slot: payloadInput.slot,
+        root: payloadInput.blockRootHex,
+        delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
+      });
+    }
 
     if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
       pendingPayload.peerIdStrings.add(peerIdStr);
     }
 
-    this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
-
-    this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
-      slot: payloadInput.slot,
-      root: payloadInput.blockRootHex,
-      delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
-    });
-
     const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
     if (prunedItemCount > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.capacity}, prunedItemCount);
       this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
     }
+    return isNewRoot;
   };
 
   private onPeerConnected = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
@@ -918,16 +928,17 @@ export class BlockInputSync {
    * Poll unknown payloads of the current slot.
    */
   private async pollPayloadsAtSlot(slot: Slot, signal: AbortSignal): Promise<void> {
+    const fork = this.config.getForkName(slot);
     // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
     const tickCount = Math.floor(
-      (this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS
+      (this.config.getSlotDurationMs(fork) - this.config.getPayloadDueMs(fork)) / PAYLOAD_POLL_INTERVAL_MS
     );
     // a root still missing on several ticks is searched on each of them, recorded once
     const searchedRoots = new Set<RootHex>();
     let polls = 0;
 
     try {
-      await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+      await sleep(Math.max(0, this.config.getPayloadDueMs(fork) - this.chain.clock.msFromSlot(slot)), signal);
 
       for (polls = 0; polls < tickCount; polls++) {
         let importedCount = 0;
@@ -1266,6 +1277,9 @@ export class BlockInputSync {
     }
 
     const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
+    if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+      return;
+    }
     if (!payloadInput) {
       this.logger.debug("PayloadEnvelopeInput not yet reloadable for imported block, will retry", {
         root: rootHex,
@@ -1277,6 +1291,9 @@ export class BlockInputSync {
       const validationResult = await wrapError(
         validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
       );
+      if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+        return;
+      }
       if (validationResult.err) {
         this.logger.debug(
           "Pending payload envelope failed validation after block import, refetching by root",
@@ -1344,14 +1361,13 @@ export class BlockInputSync {
     payload.status = PendingPayloadInputStatus.fetching;
 
     const res = await wrapError(this.fetchPayloadInput(payload));
+    if (this.pendingPayloads.get(rootHex) !== payload) {
+      this.logger.verbose("Dropping downloaded payload, entry changed during fetch", logCtx);
+      return;
+    }
     if (!res.err) {
       const pendingPayload = res.result;
       const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
-      // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
-      if (!this.pendingPayloads.has(resultRootHex)) {
-        this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
-        return;
-      }
       this.pendingPayloads.set(resultRootHex, pendingPayload);
       const result = isPendingPayloadEnvelope(pendingPayload)
         ? DownloadResult.WaitingForBlock
@@ -1422,12 +1438,16 @@ export class BlockInputSync {
     this.logger.debug("Processing downloaded payload", logCtx);
 
     const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
+    // The executionPayload event emitted during import usually removes the entry before this point
     if (!res.err) {
       this.logger.debug("Processed payload from unknown sync", logCtx);
       this.pendingPayloads.delete(rootHex);
       this.triggerUnknownBlockSearch();
       return;
     }
+    if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+      return;
+    }
 
     if (res.err instanceof PayloadError) {
       switch (res.err.type.code) {

package/src/util/clock.ts

@@ -78,9 +78,9 @@ export class Clock extends EventEmitter implements IClock {
 
     this.config = config;
     this.genesisTime = genesisTime;
-    this.timeoutId = setTimeout(this.onNextSlot, this.msUntilNextSlot());
     this.signal = signal;
     this._currentSlot = getCurrentSlot(this.config, this.genesisTime);
+    this.timeoutId = setTimeout(this.onNextSlot, this.msUntilNextSlot());
     this.signal.addEventListener("abort", () => clearTimeout(this.timeoutId), {once: true});
   }
 
@@ -204,8 +204,7 @@ export class Clock extends EventEmitter implements IClock {
   };
 
   private msUntilNextSlot(): number {
-    const milliSecondsPerSlot = this.config.SLOT_DURATION_MS;
-    const diffInMilliSeconds = Date.now() - this.genesisTime * 1000;
-    return milliSecondsPerSlot - (diffInMilliSeconds % milliSecondsPerSlot);
+    const nextSlotTimeSec = computeTimeAtSlot(this.config, this._currentSlot + 1, this.genesisTime);
+    return Math.max(0, nextSlotTimeSec * 1000 - Date.now());
   }
 }

package/src/util/file.ts

@@ -1,25 +1,24 @@
-import fs from "node:fs";
-import {promisify} from "node:util";
+import {readFile as fsReadFile, mkdir, readdir, stat, unlink, writeFile} from "node:fs/promises";
 
 /** Ensure a directory exists */
 export async function ensureDir(path: string): Promise<void> {
   try {
-    await promisify(fs.stat)(path);
+    await stat(path);
   } catch (_) {
     // not exists
-    await promisify(fs.mkdir)(path, {recursive: true});
+    await mkdir(path, {recursive: true});
   }
 }
 
 /** Write data to a file if it does not exist */
 export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise<boolean> {
   try {
-    await promisify(fs.stat)(filepath);
+    await stat(filepath);
     return false;
     // file exists, do nothing
   } catch (_) {
     // not exists
-    await promisify(fs.writeFile)(filepath, bytes);
+    await writeFile(filepath, bytes);
     return true;
   }
 }
@@ -27,7 +26,7 @@ export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Prom
 /** Remove a file if it exists */
 export async function removeFile(path: string): Promise<boolean> {
   try {
-    await promisify(fs.unlink)(path);
+    await unlink(path);
     return true;
   } catch (_) {
     // may not exists
@@ -37,7 +36,7 @@ export async function removeFile(path: string): Promise<boolean> {
 
 export async function readFile(path: string): Promise<Uint8Array | null> {
   try {
-    return await fs.promises.readFile(path);
+    return await fsReadFile(path);
   } catch (_) {
     return null;
   }
@@ -45,7 +44,7 @@ export async function readFile(path: string): Promise<Uint8Array | null> {
 
 export async function readFileNames(folderPath: string): Promise<string[]> {
   try {
-    return await fs.promises.readdir(folderPath);
+    return await readdir(folderPath);
   } catch (_) {
     return [];
   }

package/src/util/queue/itemQueue.ts

@@ -69,7 +69,7 @@ export class JobItemQueue<Args extends any[], R> {
       if (this.jobs.length === 1 && this.opts.noYieldIfOneItem) {
         void this.runJob();
       } else if (this.runningJobs < this.opts.maxConcurrency) {
-        callInNextEventLoop(this.runJob);
+        callInNextEventLoop(() => void this.runJob());
       }
     });
   }