@lodestar/beacon-node 1.44.0-dev.4d5e0a5266 → 1.44.0-dev.51165b8806

package/src/network/processor/gossipHandlers.ts

@@ -185,7 +185,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       peerIdStr,
     });
     try {
-      await validateGossipBlock(config, chain, signedBlock, fork);
+      const {skippedSlots} = await validateGossipBlock(config, chain, signedBlock, fork);
 
       if (isForkPostGloas(fork)) {
         chain.seenPayloadEnvelopeInputCache.add({
@@ -205,8 +205,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
       metrics?.gossipBlock.gossipValidation.recvToValidation.observe(recvToValidation);
       metrics?.gossipBlock.gossipValidation.validationTime.observe(validationTime);
+      metrics?.gossipBlock.skippedSlots.observe(skippedSlots);
 
-      logger.debug("Validated gossip block", {...blockInputMeta, ...logCtx, recvToValidation, validationTime});
+      logger.debug("Validated gossip block", {
+        ...blockInputMeta,
+        ...logCtx,
+        recvToValidation,
+        validationTime,
+        skippedSlots,
+      });
 
       chain.emitter.emit(routes.events.EventType.blockGossip, {slot, block: blockRootHex});
 

package/src/network/processor/index.ts

@@ -186,9 +186,11 @@ export class NetworkProcessor {
   // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
   private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingBlockMessageCount = 0;
   // we may not receive the payload for messages that require the FULL payload variant to be processed,
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingPayloadMessageCount = 0;
   private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
   private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
@@ -228,8 +230,8 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
-        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.awaitingBlockMessageCount);
+        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.awaitingPayloadMessageCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -297,7 +299,7 @@ export class NetworkProcessor {
       return;
     }
     this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -497,7 +499,7 @@ export class NetworkProcessor {
         this.pushPendingGossipsubMessageToQueue(message);
         break;
       case PreprocessAction.AwaitBlock: {
-        if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+        if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
           // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           this.metrics?.awaitingBlockGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
@@ -509,10 +511,11 @@ export class NetworkProcessor {
         this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
         const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
         awaitingGossipsubMessages.add(message);
+        this.awaitingBlockMessageCount++;
         break;
       }
       case PreprocessAction.AwaitEnvelope: {
-        if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+        if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
           this.metrics?.awaitingPayloadGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
             topic: topicType,
@@ -525,6 +528,7 @@ export class NetworkProcessor {
           preprocessResult.root
         );
         awaitingPayloadGossipsubMessages.add(message);
+        this.awaitingPayloadMessageCount++;
         break;
       }
     }
@@ -548,6 +552,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+      this.awaitingBlockMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -567,8 +577,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByBlockRoot.delete(rootHex);
   };
 
   private onPayloadEnvelopeProcessed = async ({blockRoot: rootHex}: {blockRoot: RootHex}): Promise<void> => {
@@ -577,6 +585,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+      this.awaitingPayloadMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const message of waitingGossipsubMessages) {
@@ -593,8 +607,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
   };
 
   private onClockSlot = (clockSlot: Slot): void => {
@@ -618,7 +630,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+            this.awaitingBlockMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownBlocksBySlot.delete(slot);
@@ -641,7 +655,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+            this.awaitingPayloadMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownEnvelopesBySlot.delete(slot);
@@ -784,20 +800,4 @@ export class NetworkProcessor {
 
     return null;
   }
-
-  private get unknownBlockGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
-
-  private get unknownPayloadGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
 }

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -1,6 +1,6 @@
 import {PeerId} from "@libp2p/interface";
 import {BeaconConfig} from "@lodestar/config";
-import {GENESIS_SLOT, isForkPostDeneb, isForkPostFulu} from "@lodestar/params";
+import {GENESIS_SLOT, isForkPostDeneb} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {deneb, phase0} from "@lodestar/types";
@@ -29,13 +29,20 @@ export async function* onBeaconBlocksByRange(
   // starts above it to avoid duplicate yields. See archiveBlocks.ts for the migration logic.
   const archiveMaxSlot = finalizedSlot;
 
-  const forkName = chain.config.getForkName(startSlot);
-  if (isForkPostFulu(forkName) && startSlot < chain.earliestAvailableSlot) {
-    chain.logger.verbose("Peer did not respect earliestAvailableSlot for BeaconBlocksByRange", {
+  // endSlot is exclusive, so highest served slot is endSlot - 1.
+  // Throw only when the entire requested range is below earliestAvailableSlot.
+  if (endSlot - 1 < chain.earliestAvailableSlot) {
+    chain.logger.verbose("Peer requested range before earliestAvailableSlot for BeaconBlocksByRange", {
       peer: prettyPrintPeerId(peerId),
       client: peerClient,
+      startSlot,
+      count,
+      earliestAvailableSlot: chain.earliestAvailableSlot,
     });
-    return;
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      `Requested range is before earliestAvailableSlot startSlot=${startSlot} count=${count} earliestAvailableSlot=${chain.earliestAvailableSlot}`
+    );
   }
 
   // Finalized range of blocks

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -34,12 +34,20 @@ export async function* onDataColumnSidecarsByRange(
     return;
   }
 
-  if (startSlot < chain.earliestAvailableSlot) {
-    chain.logger.verbose("Peer did not respect earliestAvailableSlot for DataColumnSidecarsByRange", {
+  // endSlot is exclusive, so highest served slot is endSlot - 1.
+  // Throw only when the entire requested range is below earliestAvailableSlot.
+  if (endSlot - 1 < chain.earliestAvailableSlot) {
+    chain.logger.verbose("Peer requested range before earliestAvailableSlot for DataColumnSidecarsByRange", {
       peer: prettyPrintPeerId(peerId),
       client: peerClient,
+      startSlot,
+      count,
+      earliestAvailableSlot: chain.earliestAvailableSlot,
     });
-    return;
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      `Requested range is before earliestAvailableSlot startSlot=${startSlot} count=${count} earliestAvailableSlot=${chain.earliestAvailableSlot}`
+    );
   }
 
   const finalized = db.dataColumnSidecarArchive;

package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -30,7 +30,7 @@ export async function* onDataColumnSidecarsByRoot(
     const {blockRoot, columns: requestedColumns} = dataColumnsByRootIdentifier;
     const availableColumns = validateRequestedDataColumns(chain, requestedColumns);
     if (availableColumns.length === 0) {
-      return;
+      continue;
     }
 
     const blockRootHex = toRootHex(blockRoot);

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -1,3 +1,4 @@
+import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
 import {PayloadStatus} from "@lodestar/fork-choice";
 import {GENESIS_SLOT} from "@lodestar/params";
@@ -6,23 +7,38 @@ import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {gloas} from "@lodestar/types";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {prettyPrintPeerId} from "../../util.js";
 
 export async function* onExecutionPayloadEnvelopesByRange(
   request: gloas.ExecutionPayloadEnvelopesByRangeRequest,
   chain: IBeaconChain,
-  db: IBeaconDb
+  db: IBeaconDb,
+  peerId: PeerId,
+  peerClient: string
 ): AsyncIterable<ResponseOutgoing> {
   const {startSlot, count} = validateExecutionPayloadEnvelopesByRangeRequest(chain.config, request);
   const endSlot = startSlot + count;
 
-  if (startSlot < chain.earliestAvailableSlot) {
-    return;
+  // endSlot is exclusive, so highest served slot is endSlot - 1.
+  // Throw only when the entire requested range is below earliestAvailableSlot.
+  if (endSlot - 1 < chain.earliestAvailableSlot) {
+    chain.logger.verbose("Peer requested range before earliestAvailableSlot for ExecutionPayloadEnvelopesByRange", {
+      peer: prettyPrintPeerId(peerId),
+      client: peerClient,
+      startSlot,
+      count,
+      earliestAvailableSlot: chain.earliestAvailableSlot,
+    });
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      `Requested range is before earliestAvailableSlot startSlot=${startSlot} count=${count} earliestAvailableSlot=${chain.earliestAvailableSlot}`
+    );
   }
 
   const finalized = db.executionPayloadEnvelopeArchive;
-  const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
-  // The current finalized block's envelope is still in the hot db; archive migration happens
-  // in the next finalization run (see migrateExecutionPayloadEnvelopesFromHotToColdDb).
+  // Use the finalized block's actual slot as the checkpoint epoch-boundary slot may be skipped
+  const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // The finalized block's envelope stays in the hot db until the next finalization run
   const archiveMaxSlot = finalizedSlot - 1;
 
   // Finalized range of envelopes

package/src/network/reqresp/handlers/index.ts

@@ -74,9 +74,9 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
       const body = ExecutionPayloadEnvelopesByRootRequestType(chain.config).deserialize(req.data);
       return onExecutionPayloadEnvelopesByRoot(body, chain, db, peerId, peerClient);
     },
-    [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: (req) => {
+    [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: (req, peerId, peerClient) => {
       const body = ssz.gloas.ExecutionPayloadEnvelopesByRangeRequest.deserialize(req.data);
-      return onExecutionPayloadEnvelopesByRange(body, chain, db);
+      return onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient);
     },
 
     [ReqRespMethod.LightClientBootstrap]: (req) => {

package/src/network/reqresp/utils/dataColumnResponseValidation.ts

@@ -1,6 +1,5 @@
 import {LogData} from "@lodestar/logger";
 import {ForkSeq} from "@lodestar/params";
-import {RespStatus, ResponseError} from "@lodestar/reqresp";
 import {ColumnIndex, Slot} from "@lodestar/types";
 import {prettyBytes, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/interface.js";
@@ -76,12 +75,22 @@ export async function handleColumnSidecarUnavailability({
 
 export function validateRequestedDataColumns(chain: IBeaconChain, requestedColumns: ColumnIndex[]): ColumnIndex[] {
   if (requestedColumns.length === 0) {
-    throw new ResponseError(RespStatus.INVALID_REQUEST, "dataColumnSidecar requested without column indices");
+    return [];
   }
 
-  const custodyColumns = chain.custodyConfig.custodyColumns;
-  const availableColumns = requestedColumns.filter((c) => custodyColumns.includes(c));
-  const missingColumns = requestedColumns.filter((c) => !custodyColumns.includes(c));
+  const {custodyColumns, custodyColumnsIndex} = chain.custodyConfig;
+  const availableColumns: ColumnIndex[] = [];
+  const missingColumns: ColumnIndex[] = [];
+  for (const c of requestedColumns) {
+    // `c` is peer-controlled and SSZ-deserialized as `uint64`, so it may exceed
+    // `NUMBER_OF_COLUMNS - 1`; `Uint8Array` returns `undefined` for OOB reads,
+    // and `undefined !== 0` would silently classify OOB indices as custodied.
+    if ((custodyColumnsIndex[c] ?? 0) !== 0) {
+      availableColumns.push(c);
+    } else {
+      missingColumns.push(c);
+    }
+  }
 
   if (missingColumns.length > 0) {
     chain.logger.verbose("Requested dataColumnSidecar for non-custody columns", {

package/src/sync/types.ts

@@ -19,7 +19,14 @@ export enum PendingBlockType {
    */
   INCOMPLETE_BLOCK_INPUT = "IncompleteBlockInput",
 
-  UNKNOWN_DATA = "unknown_data",
+  /**
+   * Payload analog of UNKNOWN_BLOCK_ROOT: we have a beacon block root but not its execution payload envelope.
+   */
+  UNKNOWN_PAYLOAD_BLOCK_ROOT = "unknown_payload_block_root",
+  /**
+   * Payload analog of INCOMPLETE_BLOCK_INPUT: we have a partial payload input that did not complete in time.
+   */
+  INCOMPLETE_PAYLOAD_ENVELOPE = "incomplete_payload_envelope",
 }
 
 export enum PendingBlockInputStatus {
@@ -70,6 +77,8 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
+  // message slot hint, may be missing when resolving a parent payload
+  slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;
   peerIdStrings: Set<string>;
@@ -125,5 +134,5 @@ export function getPayloadSyncCacheItemSlot(payload: PayloadSyncCacheItem): Slot
     return payload.envelope.message.payload.slotNumber;
   }
 
-  return "unknown";
+  return payload.slot ?? "unknown";
 }

package/src/sync/unknownBlock.ts

@@ -3,7 +3,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {ForkSeq} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
-import {RootHex, gloas} from "@lodestar/types";
+import {RootHex, Slot, gloas} from "@lodestar/types";
 import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
@@ -215,10 +215,10 @@ export class BlockInputSync {
 
   private onUnknownEnvelopeBlockRoot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]): void => {
     try {
-      this.addByPayloadRootHex(data.rootHex, data.peer);
+      this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
       this.triggerUnknownBlockSearch();
-      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_DATA});
-      this.metrics?.blockInputSync.source.inc({source: data.source});
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT});
+      this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
     } catch (e) {
       this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e as Error);
     }
@@ -228,8 +228,8 @@ export class BlockInputSync {
     try {
       this.addByPayloadInput(data.payloadInput, data.peer);
       this.triggerUnknownBlockSearch();
-      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_DATA});
-      this.metrics?.blockInputSync.source.inc({source: data.source});
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
+      this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
     } catch (e) {
       this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e as Error);
     }
@@ -331,7 +331,10 @@ export class BlockInputSync {
       };
       this.pendingBlocks.set(blockInput.blockRootHex, pendingBlock);
 
-      this.logger.verbose("Added blockInput to BlockInputSync.pendingBlocks", pendingBlock.blockInput.getLogMeta());
+      this.logger.verbose("Added blockInput to BlockInputSync.pendingBlocks", {
+        ...pendingBlock.blockInput.getLogMeta(),
+        delaySec: this.chain.clock.secFromSlot(blockInput.slot),
+      });
     }
 
     if (peerIdStr) {
@@ -346,13 +349,14 @@ export class BlockInputSync {
     }
   };
 
-  private addByPayloadRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
+  private addByPayloadRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr, slot?: Slot): boolean => {
     let pendingPayload = this.pendingPayloads.get(rootHex);
     let added = false;
     if (!pendingPayload) {
       pendingPayload = {
         status: PendingPayloadInputStatus.pending,
         rootHex,
+        slot,
         peerIdStrings: new Set(),
         timeAddedSec: Date.now() / 1000,
       };
@@ -360,6 +364,8 @@ export class BlockInputSync {
       added = true;
 
       this.logger.verbose("Added new payload rootHex to BlockInputSync.pendingPayloads", {
+        slot: slot ?? "unknown",
+        delaySec: slot != null ? this.chain.clock.secFromSlot(slot) : "unknown",
         root: rootHex,
         peerIdStr: peerIdStr ?? "unknown peer",
       });
@@ -392,6 +398,13 @@ export class BlockInputSync {
     }
 
     this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
+
+    this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
+      slot: payloadInput.slot,
+      root: payloadInput.blockRootHex,
+      delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
+    });
+
     const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
     if (prunedItemCount > 0) {
       this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
@@ -654,10 +667,12 @@ export class BlockInputSync {
     }
 
     const rootHex = getBlockInputSyncCacheItemRootHex(block);
+    const blockSlot = getBlockInputSyncCacheItemSlot(block);
     const logCtx = {
-      slot: getBlockInputSyncCacheItemSlot(block),
+      slot: blockSlot,
       root: rootHex,
       pendingBlocks: this.pendingBlocks.size,
+      ...(typeof blockSlot === "number" && {delaySec: this.chain.clock.secFromSlot(blockSlot)}),
     };
 
     this.logger.verbose("BlockInputSync.downloadBlock()", logCtx);
@@ -679,6 +694,7 @@ export class BlockInputSync {
       const logCtx2 = {
         ...logCtx,
         slot: blockSlot,
+        delaySec,
         parentInForkChoice,
       };
       this.logger.verbose("Downloaded unknown block", logCtx2);
@@ -748,6 +764,12 @@ export class BlockInputSync {
     // this prevents unbundling attack
     // see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
     const {slot: blockSlot, proposerIndex} = pendingBlock.blockInput.getBlock().message;
+    const logCtx = {
+      slot: blockSlot,
+      root: pendingBlock.blockInput.blockRootHex,
+      delaySec: this.chain.clock.secFromSlot(blockSlot),
+    };
+    this.logger.verbose("Processing downloaded block", logCtx);
     const fork = this.config.getForkName(blockSlot);
     const proposerBoostWindowMs = this.config.getAttestationDueMs(fork);
     if (
@@ -783,6 +805,7 @@ export class BlockInputSync {
     else this.metrics?.blockInputSync.processedBlocksSuccess.inc();
 
     if (!res.err) {
+      this.logger.verbose("Processed block from unknown sync", logCtx);
       // no need to update status to "processed", delete anyway
       this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
       // Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
@@ -850,25 +873,28 @@ export class BlockInputSync {
     }
 
     const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
-    if (!payloadInput) {
-      if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
-        // Column commitments live on the block body, so an envelope-only entry has to pull the block first.
-        if (!this.pendingBlocks.has(rootHex)) {
-          this.addByRootHex(rootHex);
-        }
+    if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+      // Block not in fork choice yet. payloadInput may be seeded from the block body during download, so a
+      // non-null payloadInput does not imply the block is imported; defer regardless and pull the block first.
+      // onBlockImported re-triggers the search to resume this envelope.
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.addByRootHex(rootHex);
+      }
 
-        const pendingBlock = this.pendingBlocks.get(rootHex);
-        if (pendingBlock && this.network.getConnectedPeers().length > 0) {
-          await this.downloadBlock(pendingBlock);
-        }
-      } else {
-        this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
-          root: rootHex,
-        });
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && this.network.getConnectedPeers().length > 0) {
+        await this.downloadBlock(pendingBlock);
       }
       return;
     }
 
+    if (!payloadInput) {
+      this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+        root: rootHex,
+      });
+      return;
+    }
+
     if (!payloadInput.hasPayloadEnvelope()) {
       const validationResult = await wrapError(
         validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
@@ -922,10 +948,12 @@ export class BlockInputSync {
       return;
     }
 
+    const payloadSlot = getPayloadSyncCacheItemSlot(payload);
     const logCtx = {
-      slot: getPayloadSyncCacheItemSlot(payload),
+      slot: payloadSlot,
       root: rootHex,
       pendingPayloads: this.pendingPayloads.size,
+      ...(typeof payloadSlot === "number" && {delaySec: this.chain.clock.secFromSlot(payloadSlot)}),
     };
 
     this.logger.verbose("BlockInputSync.downloadPayload()", logCtx);
@@ -953,7 +981,11 @@ export class BlockInputSync {
 
   private async processPayload(pendingPayload: PendingPayloadInput): Promise<void> {
     const rootHex = pendingPayload.payloadInput.blockRootHex;
-    const logCtx = {slot: pendingPayload.payloadInput.slot, root: rootHex};
+    const logCtx = {
+      slot: pendingPayload.payloadInput.slot,
+      root: rootHex,
+      delaySec: this.chain.clock.secFromSlot(pendingPayload.payloadInput.slot),
+    };
 
     if (pendingPayload.status !== PendingPayloadInputStatus.downloaded) {
       this.logger.debug("Skipping payload processing before payload input is downloaded", {
@@ -980,6 +1012,7 @@ export class BlockInputSync {
     }
 
     pendingPayload.status = PendingPayloadInputStatus.processing;
+    this.logger.debug("Processing downloaded payload", logCtx);
 
     const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
     if (!res.err) {
@@ -1073,11 +1106,11 @@ export class BlockInputSync {
         }
 
         payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
-        if (!payloadInput) {
-          if (this.chain.forkChoice.hasBlockHex(rootHex)) {
-            throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
-          }
-          // Keep the validated envelope around, but wait for the block body before turning it into a full payload input.
+        if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+          // Block not in fork choice yet. Validating now would throw BLOCK_ROOT_UNKNOWN, so keep the downloaded
+          // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
+          // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
+          // imply the block is imported.
           return {
             status: PendingPayloadInputStatus.waitingForBlock,
             envelope,
@@ -1086,6 +1119,11 @@ export class BlockInputSync {
           };
         }
 
+        if (!payloadInput) {
+          // Block is in fork choice but no PayloadEnvelopeInput exists, should have been created during block import.
+          throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
+        }
+
         if (!payloadInput.hasPayloadEnvelope()) {
           await validateGossipExecutionPayloadEnvelope(this.chain, envelope);
         }
@@ -1117,6 +1155,7 @@ export class BlockInputSync {
           rootHex,
           peerId,
           peerClient,
+          ...(typeof slot === "number" && {delaySec: this.chain.clock.secFromSlot(slot)}),
           hasPayload: pendingPayload.payloadInput.hasPayloadEnvelope(),
           hasAllData: pendingPayload.payloadInput.hasAllData(),
         });
@@ -1288,7 +1327,7 @@ export class BlockInputSync {
           this.metrics?.blockInputSync.fetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
         }
 
-        const logCtx = {slot, rootHex, peerId, peerClient};
+        const logCtx = {slot, rootHex, peerId, peerClient, delaySec: this.chain.clock.secFromSlot(slot)};
         this.logger.verbose("BlockInputSync.fetchBlockInput: successful download", logCtx);
         this.metrics?.blockInputSync.downloadByRoot.success.inc();
         const warnings = downloadResult.warnings;