@lodestar/beacon-node 1.47.0-dev.7a8fd842fc → 1.47.0-dev.7aa8c9c93a

package/src/node/nodejs.ts

@@ -1,12 +1,13 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";

package/src/sync/range/batch.ts

@@ -39,6 +39,7 @@ export enum BatchStatus {
   AwaitingValidation = "AwaitingValidation",
 }
 
+/** Identity of one download attempt*/
 export type Attempt = {
   /** The peer that made the attempt */
   peers: PeerIdStr[];
@@ -46,6 +47,16 @@ export type Attempt = {
   hash: RootHex;
 };
 
+/**
+ * A failed attempt that has been classified for blame.
+ */
+export type FailedAttempt = Attempt & {
+  /**
+   * True when the failure is evidence the peers served bad data, so they may be downscored..
+   */
+  peerAttributable: boolean;
+};
+
 type TrackedRequest = {
   /** only happen for the 1st batch in checkpoint sync */
   parentPayload: boolean;
@@ -108,10 +119,12 @@ export type BatchMetadata = {
   // Retry counters
   downloadAttempts: number;
   processingAttempts: number;
+  executionErrorAttempts: number;
 
   // Cumulative peer attribution for failed attempts (only present when non-empty)
   failedDownloadPeers?: string;
   failedProcessingPeers?: string;
+  executionErrorPeers?: string;
 };
 
 function formatRangeReq(req: {startSlot: Slot; count: number}): string {
@@ -152,10 +165,10 @@ export class Batch {
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   /** Peers that provided good data, with column coverage for by_range requests */
   private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
-  /** The `Attempts` that have been made and failed to send us this batch. */
-  readonly failedProcessingAttempts: Attempt[] = [];
-  /** The `Attempts` that have been made and failed because of execution malfunction. */
-  readonly executionErrorAttempts: Attempt[] = [];
+  /** The attempts that have been made and failed to send us this batch. */
+  readonly failedProcessingAttempts: FailedAttempt[] = [];
+  /** The attempts that have been made and failed because of execution malfunction. */
+  readonly executionErrorAttempts: FailedAttempt[] = [];
   /** The number of download retries this batch has undergone due to a failed request. */
   private readonly failedDownloadAttempts: PeerIdStr[] = [];
   private readonly config: ChainForkConfig;
@@ -419,10 +432,18 @@ export class Batch {
   }
 
   /**
-   * Gives a list of peers from which this batch has had a failed download or processing attempt.
+   * Gives a list of peers from which this batch has had a failed download attempt, or a
+   * peer-attributable failed processing/execution attempt.
    */
   getFailedPeers(): PeerIdStr[] {
-    return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
+    return [...this.failedDownloadAttempts, ...this.getPeerAttributableAttempts().flatMap((a) => a.peers)];
+  }
+
+  /**
+   * Attempts whose failure is peer-attributable, across both the processing and execution error
+   */
+  getPeerAttributableAttempts(): FailedAttempt[] {
+    return [...this.failedProcessingAttempts, ...this.executionErrorAttempts].filter((a) => a.peerAttributable);
   }
 
   /**
@@ -457,6 +478,7 @@ export class Batch {
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
+    const executionErrorPeerList = this.executionErrorAttempts.flatMap((a) => a.peers);
     return {
       startEpoch: this.startEpoch,
       startSlot: this.startSlot,
@@ -468,12 +490,16 @@ export class Batch {
       ...(envelopesRequest && {envelopesReq: formatRangeReq(envelopesRequest)}),
       downloadAttempts: this.failedDownloadAttempts.length,
       processingAttempts: this.failedProcessingAttempts.length,
+      executionErrorAttempts: this.executionErrorAttempts.length,
       ...(this.failedDownloadAttempts.length > 0 && {
         failedDownloadPeers: this.failedDownloadAttempts.join(","),
       }),
       ...(failedProcessingPeerList.length > 0 && {
         failedProcessingPeers: failedProcessingPeerList.join(","),
       }),
+      ...(executionErrorPeerList.length > 0 && {
+        executionErrorPeers: executionErrorPeerList.join(","),
+      }),
     };
   }
 
@@ -703,11 +729,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
-    } else {
-      this.onProcessingError(this.state.attempt);
-    }
+    this.routeProcessingFailure(err, this.state.attempt);
   }
 
   /**
@@ -718,10 +740,17 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingValidation));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
+    this.routeProcessingFailure(err, this.state.attempt);
+  }
+
+  private routeProcessingFailure(err: Error, attempt: Attempt): void {
+    const code = err instanceof BlockError || err instanceof PayloadError ? err.type.code : null;
+    const failedAttempt: FailedAttempt = {...attempt, peerAttributable: isPeerAttributableFailure(code)};
+
+    if (isExecutionEngineFailure(code)) {
+      this.onExecutionEngineError(failedAttempt);
     } else {
-      this.onProcessingError(this.state.attempt);
+      this.onProcessingError(failedAttempt);
     }
   }
 
@@ -735,7 +764,7 @@ export class Batch {
     return this.state.attempt;
   }
 
-  private onExecutionEngineError(attempt: Attempt): void {
+  private onExecutionEngineError(attempt: FailedAttempt): void {
     this.executionErrorAttempts.push(attempt);
     if (this.executionErrorAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS}));
@@ -746,7 +775,7 @@ export class Batch {
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   }
 
-  private onProcessingError(attempt: Attempt): void {
+  private onProcessingError(attempt: FailedAttempt): void {
     this.failedProcessingAttempts.push(attempt);
     if (this.failedProcessingAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_PROCESSING_ATTEMPTS}));
@@ -789,18 +818,44 @@ type BatchErrorMetadata = {
 
 export class BatchError extends LodestarError<BatchErrorType & BatchErrorMetadata> {}
 
-function isExecutionEngineError(err: Error): boolean {
-  if (!(err instanceof BlockError)) {
-    return false;
-  }
-
-  if (err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
-    return true;
-  }
-
+/**
+ * Whether a processing failure is an execution-engine failure. Pre-gloas, all EL errors stay in
+ * BlockErrorCode, post-gloas, they stay in PayloadErrorCode.
+ */
+function isExecutionEngineFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
   return (
-    err.type.code === BlockErrorCode.BEACON_CHAIN_ERROR &&
-    err.type.error instanceof PayloadError &&
-    err.type.error.type.code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
+    code === BlockErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === BlockErrorCode.EXECUTION_ENGINE_ERROR ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
   );
 }
+
+/**
+ * Based on an error code, return if peer is attributable for FailedAttempt.
+ */
+function isPeerAttributableFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
+  switch (code) {
+    // EL returned a definitive INVALID verdict on the payload
+    case BlockErrorCode.EXECUTION_ENGINE_INVALID:
+    case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+    // a block or envelope signature is invalid
+    case BlockErrorCode.INVALID_SIGNATURE:
+    case PayloadErrorCode.INVALID_SIGNATURE:
+    // the block's state transition produced an invalid state root, or failed per_slot/per_block processing
+    case BlockErrorCode.INVALID_STATE_ROOT:
+    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
+    // the segment this peer served is internally inconsistent (in-segment link breaks, see PR #9684)
+    case BlockErrorCode.NON_LINEAR_SLOTS:
+    case BlockErrorCode.NON_LINEAR_PARENT_ROOTS:
+    case BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS:
+    // the envelope references a mismatched block root, or fails field verification
+    case BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH:
+    case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+    // the peer served a block we have explicitly blacklisted
+    case BlockErrorCode.BLACKLISTED_BLOCK:
+      return true;
+    default:
+      return false;
+  }
+}

package/src/sync/range/chain.ts

@@ -358,9 +358,10 @@ export class SyncChain {
       // If a batch exceeds it's retry limit, maybe downscore peers.
       // shouldDownscoreOnBatchError() functions enforces that all BatchErrorCode values are covered
       if (e instanceof BatchError) {
-        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code);
+        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code, this.batches.get(e.type.startEpoch));
         if (shouldReportPeer) {
-          for (const peer of this.peerset.keys()) {
+          // Only the peers actually at fault are reported, never the whole peerset.
+          for (const peer of shouldReportPeer.peers) {
             this.reportPeer(peer, shouldReportPeer.action, shouldReportPeer.reason);
           }
         }
@@ -768,8 +769,11 @@ export class SyncChain {
 
         // The last batch attempt is right, all others are wrong. Penalize other peers
         const attemptOk = batch.validationSuccess();
-        for (const attempt of batch.failedProcessingAttempts) {
+        for (const attempt of batch.getPeerAttributableAttempts()) {
           if (attempt.hash !== attemptOk.hash) {
+            // attempt.peers contains block/column/envelope peers.
+            // And if there is one bad envelope we will downscore block and column peers to
+            // TODO: resolve this
             for (const badAttemptPeer of attempt.peers) {
               if (attemptOk.peers.find((goodPeer) => goodPeer === badAttemptPeer)) {
                 // The same peer corrected its previous attempt
@@ -830,21 +834,32 @@ export class SyncChain {
  * If peer should not be downscored, returns null.
  */
 export function shouldReportPeerOnBatchError(
-  code: BatchErrorCode
-): {action: PeerAction.LowToleranceError; reason: string} | null {
+  code: BatchErrorCode,
+  batch: Batch | undefined
+): {action: PeerAction.LowToleranceError; reason: string; peers: PeerIdStr[]} | null {
   switch (code) {
-    // A batch could not be processed after max retry limit. It's likely that all peers
-    // in this chain are sending invalid batches repeatedly so are either malicious or faulty.
-    // We drop the chain and report all peers.
-    // There are some edge cases with forks that could cause this situation, but it's unlikely.
+    //  A batch could not be processed after max retry limit. Report
+    // only the peers that served a peer-attributable failed attempt.
     case BatchErrorCode.MAX_PROCESSING_ATTEMPTS:
-      return {action: PeerAction.LowToleranceError, reason: "SyncChainMaxProcessingAttempts"};
+    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS: {
+      // penalize both the processing-error and execution-error peers
+      const attributablePeers = [
+        ...new Set(batch?.getPeerAttributableAttempts().flatMap((attempt) => attempt.peers) ?? []),
+      ];
+      if (attributablePeers.length === 0) {
+        return null;
+      }
+      const reason =
+        code === BatchErrorCode.MAX_PROCESSING_ATTEMPTS
+          ? "SyncChainMaxProcessingAttempts"
+          : "SyncChainMaxExecutionEngineErrorAttempts";
+      return {action: PeerAction.LowToleranceError, reason, peers: attributablePeers};
+    }
 
     // TODO: Should peers be reported for MAX_DOWNLOAD_ATTEMPTS?
     case BatchErrorCode.MAX_DOWNLOAD_ATTEMPTS:
     case BatchErrorCode.INVALID_COUNT:
     case BatchErrorCode.WRONG_STATUS:
-    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS:
       return null;
   }
 }

package/src/sync/types.ts

@@ -77,7 +77,9 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
-  // message slot hint, may be missing when resolving a parent payload
+  // Trusted slot only (fork choice / validated data), may be missing until resolved. NOT the gossip
+  // message slot from ChainEvent.unknownEnvelopeBlockRoot, which is untrusted and not necessarily the
+  // payload/block slot. See BlockInputSync.resolvePayloadSlot.
   slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;

package/src/sync/unknownBlock.ts

@@ -1,5 +1,6 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
+import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {ForkSeq, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
@@ -154,6 +155,7 @@ export class BlockInputSync {
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+      this.chain.emitter.on(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
       this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -170,6 +172,7 @@ export class BlockInputSync {
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+    this.chain.emitter.off(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
     this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -215,7 +218,7 @@ export class BlockInputSync {
 
   private onUnknownEnvelopeBlockRoot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]): void => {
     try {
-      this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+      this.addByPayloadRootHex(data.rootHex, data.peer, this.resolvePayloadSlot(data.rootHex));
       this.triggerUnknownBlockSearch();
       this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT});
       this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
@@ -258,7 +261,7 @@ export class BlockInputSync {
       }
 
       if (missingDependency.kind === "parentPayload") {
-        this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
+        this.addByPayloadRootHex(missingDependency.rootHex, data.peer, missingDependency.slot);
       } else if (missingDependency.kind === "parentBlock") {
         this.addByRootHex(missingDependency.rootHex, data.peer);
       }
@@ -284,6 +287,61 @@ export class BlockInputSync {
     this.triggerUnknownBlockSearch();
   };
 
+  /**
+   * Drop pending payloads and blocks that fell below the finalized slot.
+   * See https://github.com/ChainSafe/lodestar/issues/9800
+   */
+  private pruneFinalized = (checkpoint: CheckpointWithHex): void => {
+    // Use the actual finalized block slot, not computeStartSlotAtEpoch(checkpoint.epoch)
+    // because the payload at the finalized block is not finalized
+    const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
+
+    let deletedPayloads = 0;
+    let unresolvedSlotPayloads = 0;
+    for (const [rootHex, payload] of this.pendingPayloads) {
+      // Last-effort resolve for a slot-less PendingPayloadRootHex, and store it back for later passes.
+      if (!isPendingPayloadInput(payload) && !isPendingPayloadEnvelope(payload) && payload.slot === undefined) {
+        payload.slot = this.resolvePayloadSlot(rootHex);
+      }
+
+      const slot = getPayloadSyncCacheItemSlot(payload);
+      if (typeof slot !== "number") {
+        unresolvedSlotPayloads++;
+        this.logger.debug("BlockInputSync.pruneFinalized: pending payload with unresolved slot", {
+          root: rootHex,
+          finalizedSlot,
+        });
+        continue;
+      }
+
+      if (slot < finalizedSlot) {
+        this.pendingPayloads.delete(rootHex);
+        deletedPayloads++;
+      }
+    }
+
+    let deletedBlocks = 0;
+    for (const [rootHex, block] of this.pendingBlocks) {
+      const slot = getBlockInputSyncCacheItemSlot(block);
+      if (typeof slot === "number" && slot < finalizedSlot) {
+        this.pendingBlocks.delete(rootHex);
+        deletedBlocks++;
+      }
+    }
+
+    if (deletedPayloads > 0 || deletedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc(deletedPayloads + deletedBlocks);
+    }
+
+    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+      finalizedSlot,
+      finalizedRoot: checkpoint.rootHex,
+      deletedPayloads,
+      deletedBlocks,
+      unresolvedSlotPayloads,
+    });
+  };
+
   private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
     let pendingBlock = this.pendingBlocks.get(rootHex);
     let added = false;
@@ -382,6 +440,29 @@ export class BlockInputSync {
     return added;
   };
 
+  /**
+   * Resolve the block slot for a payload root.
+   */
+  private resolvePayloadSlot(rootHex: RootHex): Slot | undefined {
+    // a pending block of the same root
+    const pendingBlock = this.pendingBlocks.get(rootHex);
+    if (pendingBlock) {
+      const pendingBlockSlot = getBlockInputSyncCacheItemSlot(pendingBlock);
+      if (typeof pendingBlockSlot === "number") {
+        return pendingBlockSlot;
+      }
+    }
+
+    // the block is in fork choice
+    const forkChoiceSlot = this.chain.forkChoice.getBlockHexDefaultStatus(rootHex)?.slot;
+    if (forkChoiceSlot !== undefined) {
+      return forkChoiceSlot;
+    }
+
+    // a seen payload envelope input
+    return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
+  }
+
   private addByPayloadInput = (
     payloadInput: PayloadEnvelopeInput,
     peerIdStr?: PeerIdStr,
@@ -436,10 +517,12 @@ export class BlockInputSync {
     blockInput: IBlockInput
   ):
     | {kind: "ready"}
-    | {kind: "block" | "parentBlock" | "parentPayload"; rootHex: RootHex}
+    | {kind: "block" | "parentBlock"; rootHex: RootHex}
+    | {kind: "parentPayload"; rootHex: RootHex; slot: Slot}
     | {kind: "invalidParentPayload"; parentRootHex: RootHex; parentBlockHashHex: RootHex} {
     const parentRootHex = blockInput.parentRootHex;
-    if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
+    const parentBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+    if (!parentBlock) {
       return {kind: "parentBlock", rootHex: parentRootHex};
     }
 
@@ -461,16 +544,8 @@ export class BlockInputSync {
       return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
     }
 
-    const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
-    if (parentPayloadInput) {
-      if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
-        return {kind: "parentPayload", rootHex: parentRootHex};
-      }
-
-      return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
-    }
-
-    return {kind: "parentPayload", rootHex: parentRootHex};
+    // Parent is in fork choice but its payload `parentBlockHashHex` isn't revealed yet
+    return {kind: "parentPayload", rootHex: parentRootHex, slot: parentBlock.slot};
   }
 
   private advancePendingBlock(pendingBlock: PendingBlockInput): AdvancePendingBlockResult {
@@ -493,9 +568,9 @@ export class BlockInputSync {
       }
 
       case "parentPayload": {
-        let added = this.addByPayloadRootHex(missingDependency.rootHex);
+        let added = this.addByPayloadRootHex(missingDependency.rootHex, undefined, missingDependency.slot);
         for (const peerIdStr of pendingBlock.peerIdStrings) {
-          added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
+          added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr, missingDependency.slot) || added;
         }
         return added ? "queued_parent_payload" : "blocked";
       }
@@ -811,9 +886,9 @@ export class BlockInputSync {
       this.pendingBlocks.delete(blockRootHex);
 
       if (isForkPostGloas(fork)) {
-        const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+        const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(blockRootHex);
         if (!payloadInput) {
-          this.logger.warn("PayloadEnvelopeInput not seeded during unknown sync processReadyBlock()", logCtx);
+          this.logger.debug("PayloadEnvelopeInput unavailable during unknown sync processReadyBlock()", logCtx);
         } else {
           // Similar to the gossip path: immediately attempt fetch of data columns from the execution engine.
           // The bid already carries the kzg commitments, so there is no reason to wait for the payload to arrive.
@@ -886,12 +961,12 @@ export class BlockInputSync {
    */
   private async reconcilePayloadEnvelope(pendingPayload: PendingPayloadEnvelope): Promise<void> {
     const rootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+
     if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
       this.pendingPayloads.delete(rootHex);
       return;
     }
 
-    const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
     if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
       // Block not in fork choice yet. payloadInput may be seeded from the block body during download, so a
       // non-null payloadInput does not imply the block is imported; defer regardless and pull the block first.
@@ -907,8 +982,9 @@ export class BlockInputSync {
       return;
     }
 
+    const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
     if (!payloadInput) {
-      this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+      this.logger.debug("PayloadEnvelopeInput not yet reloadable for imported block, will retry", {
         root: rootHex,
       });
       return;
@@ -928,6 +1004,7 @@ export class BlockInputSync {
         const pendingPayloadByRoot: PendingPayloadRootHex = {
           status: PendingPayloadInputStatus.pending,
           rootHex,
+          slot: pendingPayload.envelope.message.payload.slotNumber,
           timeAddedSec: pendingPayload.timeAddedSec,
           peerIdStrings: new Set(pendingPayload.peerIdStrings),
         };
@@ -967,6 +1044,10 @@ export class BlockInputSync {
       return;
     }
 
+    if (!isPendingPayloadInput(payload) && payload.slot === undefined) {
+      payload.slot = this.resolvePayloadSlot(rootHex);
+    }
+
     const payloadSlot = getPayloadSyncCacheItemSlot(payload);
     const logCtx = {
       slot: payloadSlot,
@@ -982,7 +1063,13 @@ export class BlockInputSync {
     const res = await wrapError(this.fetchPayloadInput(payload));
     if (!res.err) {
       const pendingPayload = res.result;
-      this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
+      const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+      // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingPayloads.has(resultRootHex)) {
+        this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingPayloads.set(resultRootHex, pendingPayload);
 
       if (isPendingPayloadEnvelope(pendingPayload)) {
         await this.reconcilePayloadEnvelope(pendingPayload);
@@ -1093,7 +1180,7 @@ export class BlockInputSync {
     let slot = getPayloadSyncCacheItemSlot(cacheItem);
     let payloadInput = isPendingPayloadInput(cacheItem)
       ? cacheItem.payloadInput
-      : this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+      : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
     let i = 0;
@@ -1110,9 +1197,11 @@ export class BlockInputSync {
           );
         }
 
-        throw Error(
-          `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
-        );
+        const reason =
+          pendingColumns.size > 0
+            ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
+            : "no peer available to download pending payload";
+        throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
       }
 
       const {peerId, client: peerClient} = peerMeta;
@@ -1124,7 +1213,7 @@ export class BlockInputSync {
           slot = envelope.message.payload.slotNumber;
         }
 
-        payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+        payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
         if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
           // Block not in fork choice yet. Validating now would throw BLOCK_ROOT_UNKNOWN, so keep the downloaded
           // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.

package/src/sync/utils/downloadByRange.ts

@@ -136,6 +136,8 @@ export function cacheByRangeResponses({
   const source = BlockInputSource.byRange;
   const updatedBatchBlocks = new Map<Slot, IBlockInput>(batchBlocks.map((block) => [block.slot, block]));
 
+  const payloadEnvelopes = new Map<Slot, PayloadEnvelopeInput>(existingPayloadEnvelopes);
+
   const blocks = responses.validatedBlocks ?? [];
   for (let i = 0; i < blocks.length; i++) {
     const {block, blockRoot} = blocks[i];
@@ -172,15 +174,17 @@ export function cacheByRangeResponses({
     // later step) can throw and abort the batch — otherwise a gloas block would sit in
     // seenBlockInputCache but unseeded here, and payload-by-root sync would later throw "Missing
     // PayloadEnvelopeInput for known block" (see issue #9306). add() is idempotent.
-    if (isForkPostGloas(blockInput.forkName)) {
-      seenPayloadEnvelopeInputCache.add({
+    if (isForkPostGloas(blockInput.forkName) && !payloadEnvelopes.has(blockInput.slot)) {
+      const payloadInput = seenPayloadEnvelopeInputCache.add({
         blockRootHex: blockInput.blockRootHex,
         block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
         forkName: blockInput.forkName,
         sampledColumns: custodyConfig.sampledColumns,
         custodyColumns: custodyConfig.custodyColumns,
-        timeCreatedSec: seenTimestampSec,
+        seenTimestampSec,
+        source: PayloadEnvelopeInputSource.byRange,
       });
+      payloadEnvelopes.set(blockInput.slot, payloadInput);
     }
   }
 
@@ -222,17 +226,22 @@ export function cacheByRangeResponses({
     }
   }
 
-  let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null =
-    existingPayloadEnvelopes !== null ? new Map(existingPayloadEnvelopes) : null;
   if (downloadedPayloadEnvelopes !== null) {
-    payloadEnvelopes ??= new Map();
     for (const [slot, envelope] of downloadedPayloadEnvelopes) {
-      const envelopeBlockRootHex = toRootHex(envelope.message.beaconBlockRoot);
-      const payloadInput = seenPayloadEnvelopeInputCache.get(envelopeBlockRootHex);
+      // the only PayloadEnvelopeInput miss is the dangling parent, which we can get from the seen cache
+      let payloadInput = payloadEnvelopes.get(slot);
       if (payloadInput === undefined) {
-        // Unreachable given the validatedBlocks loop above seeded an entry for every gloas block in
-        // the batch. for the parent block, it's populated at BeaconChain init
-        throw new Error(`Missing PayloadEnvelopeInput for block ${envelopeBlockRootHex}`);
+        if (updatedBatchBlocks.has(slot)) {
+          throw new Error(
+            `Missing PayloadEnvelopeInput for in-batch slot ${slot} root ${toRootHex(envelope.message.beaconBlockRoot)}`
+          );
+        }
+        payloadInput = seenPayloadEnvelopeInputCache.get(toRootHex(envelope.message.beaconBlockRoot));
+      }
+      if (payloadInput === undefined) {
+        throw new Error(
+          `Missing PayloadEnvelopeInput for slot ${slot} root ${toRootHex(envelope.message.beaconBlockRoot)}`
+        );
       }
 
       if (!payloadInput.hasPayloadEnvelope()) {
@@ -262,7 +271,7 @@ export function cacheByRangeResponses({
       // Gloas columns are attached to the matching PayloadEnvelopeInput, NOT to IBlockInput.
       // Gloas DataColumnSidecar has `slot` directly (no signedBlockHeader).
       const dataSlot = firstColumn.slot;
-      const payloadInput = payloadEnvelopes?.get(dataSlot);
+      const payloadInput = payloadEnvelopes.get(dataSlot);
       if (!payloadInput) {
         // Should not happen: we built payloadInputs for all gloas blocks above
         continue;

package/src/sync/utils/downloadByRoot.ts

@@ -13,6 +13,7 @@ import {BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lod
 import {LodestarError, byteArrayEquals, fromHex, prettyPrintIndices, toHex, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "../../chain/emitter.js";
 import {IBeaconChain} from "../../chain/interface.js";
 import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js";
@@ -122,7 +123,8 @@ export async function downloadByRoot({
       forkName: blockInput.forkName,
       sampledColumns: chain.custodyConfig.sampledColumns,
       custodyColumns: chain.custodyConfig.custodyColumns,
-      timeCreatedSec: Date.now() / 1000,
+      seenTimestampSec: Date.now() / 1000,
+      source: PayloadEnvelopeInputSource.byRoot,
     });
   }