@lodestar/beacon-node 1.45.0-dev.d7859e978b → 1.45.0-dev.e10fb5e991

package/src/api/impl/validator/index.ts

@@ -82,7 +82,7 @@ import {CommitteeSubscription} from "../../../network/subnets/index.js";
 import {SyncState} from "../../../sync/index.js";
 import {callInNextEventLoop} from "../../../util/eventLoop.js";
 import {isOptimisticBlock} from "../../../util/forkChoice.js";
-import {getDefaultGraffiti, toGraffitiBytes} from "../../../util/graffiti.js";
+import {getBlockGraffiti, toGraffitiBytes} from "../../../util/graffiti.js";
 import {getLodestarClientVersion} from "../../../util/metadata.js";
 import {ApiOptions} from "../../options.js";
 import {getStateResponseWithRegen} from "../beacon/state/utils.js";
@@ -611,7 +611,10 @@ export function getValidatorApi(
     }
 
     const graffitiBytes = toGraffitiBytes(
-      graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts)
+      getBlockGraffiti(graffiti, getLodestarClientVersion(opts), chain.executionEngine.clientVersion, {
+        private: opts.private,
+        graffitiAppend: chain.opts.graffitiAppend,
+      })
     );
 
     const loggerContext = {
@@ -921,7 +924,10 @@ export function getValidatorApi(
       metrics?.blockProductionSlotDelta.set(slot - parentSlot);
 
       const graffitiBytes = toGraffitiBytes(
-        graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts)
+        getBlockGraffiti(graffiti, getLodestarClientVersion(opts), chain.executionEngine.clientVersion, {
+          private: opts.private,
+          graffitiAppend: chain.opts.graffitiAppend,
+        })
       );
 
       // TODO GLOAS: respect builderSelection (MaxProfit, BuilderAlways, ExecutionAlways, etc.) to let

package/src/chain/blocks/importBlock.ts

@@ -9,7 +9,14 @@ import {
   NotReorgedReason,
   getSafeExecutionBlockHash,
 } from "@lodestar/fork-choice";
-import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {
+  ForkPostAltair,
+  ForkPostElectra,
+  ForkSeq,
+  GENESIS_EPOCH,
+  MAX_SEED_LOOKAHEAD,
+  SLOTS_PER_EPOCH,
+} from "@lodestar/params";
 import {
   IBeaconStateView,
   RootCache,
@@ -419,7 +426,7 @@ export async function importBlock(
       }
     } catch (e) {
       if (isStartSlotOfEpoch(proposalSlot)) {
-        notOverrideFcuReason = NotReorgedReason.NotShufflingStable;
+        notOverrideFcuReason = NotReorgedReason.AtEpochBoundary;
       } else {
         this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
       }
@@ -483,8 +490,15 @@ export async function importBlock(
   // Cache shufflings when crossing an epoch boundary
   const parentEpoch = computeEpochAtSlot(parentBlockSlot);
   if (parentEpoch < blockEpoch) {
-    this.shufflingCache.processState(postState);
-    this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
+    const previousEpoch = blockEpoch === GENESIS_EPOCH ? GENESIS_EPOCH : blockEpoch - 1;
+    if (
+      !this.shufflingCache.has(previousEpoch, postState.previousDecisionRoot) ||
+      !this.shufflingCache.has(blockEpoch, postState.currentDecisionRoot) ||
+      !this.shufflingCache.has(blockEpoch + 1, postState.nextDecisionRoot)
+    ) {
+      this.shufflingCache.processState(postState);
+      this.logger.verbose("Processed shuffling for next epoch", {parentEpoch, blockEpoch, slot: blockSlot});
+    }
   }
 
   if (blockSlot % SLOTS_PER_EPOCH === 0) {

package/src/chain/blocks/payloadEnvelopeProcessor.ts

@@ -8,28 +8,24 @@ import {ImportPayloadOpts} from "./types.js";
 // TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now
 const QUEUE_MAX_LENGTH = 16;
 
-enum PayloadEnvelopeImportStatus {
-  queued = "queued",
-  importing = "importing",
-  imported = "imported",
-}
-
 /**
  * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other.
  *
  * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
  * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
  * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped via `importStatus`.
+ * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
+ * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
+ * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
+ * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
  */
 export class PayloadEnvelopeProcessor {
   readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
-  private readonly importStatus = new WeakMap<PayloadEnvelopeInput, PayloadEnvelopeImportStatus>();
+  private readonly imports = new WeakMap<PayloadEnvelopeInput, Promise<void>>();
 
   constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) {
     this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>(
       (payloadInput, opts) => {
-        this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing);
         return processExecutionPayload.call(chain, payloadInput, signal, opts);
       },
       {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal},
@@ -38,24 +34,28 @@ export class PayloadEnvelopeProcessor {
   }
 
   async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise<void> {
-    if (this.importStatus.get(payloadInput) !== undefined) {
-      return;
+    const existing = this.imports.get(payloadInput);
+    if (existing) {
+      return existing;
     }
 
     await this.jobQueue.waitForSpace();
 
     // Re-check after await, as another call may have queued this payload.
-    if (this.importStatus.get(payloadInput) !== undefined) {
-      return;
+    const queued = this.imports.get(payloadInput);
+    if (queued) {
+      return queued;
     }
 
-    this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.queued);
+    const importPromise = this.jobQueue.push(payloadInput, opts);
+    this.imports.set(payloadInput, importPromise);
 
     try {
-      await this.jobQueue.push(payloadInput, opts);
-      this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.imported);
+      await importPromise;
     } catch (e) {
-      this.importStatus.delete(payloadInput);
+      // Drop the failed import so a later attempt can retry once the failure cause is resolved
+      // (e.g. BLOCK_NOT_IN_FORK_CHOICE after the block lands in fork choice)
+      this.imports.delete(payloadInput);
       throw e;
     }
   }

package/src/chain/chain.ts

@@ -107,7 +107,6 @@ import {
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
   SeenPayloadEnvelopeInput,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -189,7 +188,6 @@ export class BeaconChain implements IBeaconChain {
   readonly seenPayloadAttesters = new SeenPayloadAttesters();
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids();
-  readonly seenProposerPreferences = new SeenProposerPreferences();
   readonly seenBlockProposers = new SeenBlockProposers();
   readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages();
   readonly seenContributionAndProof: SeenContributionAndProof;
@@ -1473,7 +1471,6 @@ export class BeaconChain implements IBeaconChain {
     this.payloadAttestationPool.prune(slot);
     this.executionPayloadBidPool.prune(slot);
     this.seenExecutionPayloadBids.prune(slot);
-    this.seenProposerPreferences.prune(slot);
     this.proposerPreferencesPool.prune(slot);
     this.seenAttestationDatas.onSlot(slot);
     this.reprocessController.onSlot(slot);

package/src/chain/interface.ts

@@ -61,7 +61,6 @@ import {
   SeenContributionAndProof,
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -133,7 +132,6 @@ export interface IBeaconChain {
   readonly seenPayloadAttesters: SeenPayloadAttesters;
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids: SeenExecutionPayloadBids;
-  readonly seenProposerPreferences: SeenProposerPreferences;
   readonly seenBlockProposers: SeenBlockProposers;
   readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages;
   readonly seenContributionAndProof: SeenContributionAndProof;

package/src/chain/opPools/proposerPreferencesPool.ts

@@ -1,4 +1,4 @@
-import {RootHex, Slot, gloas} from "@lodestar/types";
+import {RootHex, Slot, ValidatorIndex, gloas} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 
 /**
@@ -22,6 +22,10 @@ export class ProposerPreferencesPool {
     return this.bySlot.get(slot)?.get(dependentRootHex) ?? null;
   }
 
+  isKnown(proposalSlot: Slot, dependentRoot: RootHex, validatorIndex: ValidatorIndex): boolean {
+    return this.get(proposalSlot, dependentRoot)?.message.validatorIndex === validatorIndex;
+  }
+
   add(signed: gloas.SignedProposerPreferences): void {
     const {proposalSlot, dependentRoot} = signed.message;
     const rootHex = toRootHex(dependentRoot);

package/src/chain/options.ts

@@ -35,6 +35,7 @@ export type IChainOptions = BlockProcessOpts &
     persistOrphanedBlocksDir?: string;
     skipCreateStateCacheIfAvailable?: boolean;
     suggestedFeeRecipient: string;
+    graffitiAppend?: boolean;
     maxSkipSlots?: number;
     /** Ensure blobs returned by the execution engine are valid */
     sanityCheckExecutionEngineBlobs?: boolean;
@@ -107,6 +108,7 @@ export const defaultChainOptions: IChainOptions = {
   computeUnrealized: true,
   fastConfirmation: false,
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
+  graffitiAppend: true,
   serveHistoricalState: false,
   assertCorrectProgressiveBalances: false,
   archiveStateEpochFrequency: 1024,

package/src/chain/seenCache/index.ts

@@ -5,4 +5,3 @@ export {SeenContributionAndProof} from "./seenCommitteeContribution.js";
 export {SeenExecutionPayloadBids} from "./seenExecutionPayloadBids.js";
 export {SeenBlockInput} from "./seenGossipBlockInput.js";
 export {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenPayloadEnvelopeInput.js";
-export {SeenProposerPreferences} from "./seenProposerPreferences.js";

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -152,6 +152,14 @@ export class SeenPayloadEnvelopeInput {
     return this.payloadInputs.size;
   }
 
+  prune(blockRootHex: RootHex): void {
+    const input = this.payloadInputs.get(blockRootHex);
+    if (input) {
+      this.evictPayloadInput(input);
+      this.logger?.verbose("SeenPayloadEnvelopeInput.prune deleted", {slot: input.slot, root: blockRootHex});
+    }
+  }
+
   pruneBelowParent(parentBlock: ProtoBlock): void {
     for (const block of this.forkChoice.getAllAncestorBlocks(parentBlock.blockRoot, parentBlock.payloadStatus)) {
       // Only evict once the payload is FULL (revealed/imported) — on an EMPTY/PENDING branch we may

package/src/chain/shufflingCache.ts

@@ -1,4 +1,4 @@
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, GENESIS_EPOCH} from "@lodestar/params";
 import {
   EpochShuffling,
   IBeaconStateView,
@@ -155,19 +155,35 @@ export class ShufflingCache {
     return null;
   }
 
+  /**
+   * Check if a shuffling is cached by (epoch, decisionRoot).
+   * For native bindings, this avoids having to memoize heavy epoch shufflings within
+   * `NativeBeaconStateView`.
+   */
+  has(epoch: Epoch, decisionRoot: RootHex): boolean {
+    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    return cacheItem !== undefined && isShufflingCacheItem(cacheItem);
+  }
+
   /**
    * Process a state to extract and cache all shufflings (previous, current, next).
    * Uses the stored decision roots from epochCtx.
    */
   processState(state: IBeaconStateView): void {
-    // Cache previous shuffling
-    this.set(state.getPreviousShuffling(), state.previousDecisionRoot);
+    const currentEpoch = state.epoch;
+    const previousEpoch = currentEpoch === GENESIS_EPOCH ? GENESIS_EPOCH : currentEpoch - 1;
 
-    // Cache current shuffling
-    this.set(state.getCurrentShuffling(), state.currentDecisionRoot);
+    if (!this.has(previousEpoch, state.previousDecisionRoot)) {
+      this.set(state.getPreviousShuffling(), state.previousDecisionRoot);
+    }
+
+    if (!this.has(currentEpoch, state.currentDecisionRoot)) {
+      this.set(state.getCurrentShuffling(), state.currentDecisionRoot);
+    }
 
-    // Cache next shuffling
-    this.set(state.getNextShuffling(), state.nextDecisionRoot);
+    if (!this.has(currentEpoch + 1, state.nextDecisionRoot)) {
+      this.set(state.getNextShuffling(), state.nextDecisionRoot);
+    }
   }
 
   getIndexedAttestation(

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -331,8 +331,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     const key = toCacheKey(cpHex);
     const cacheItem = this.cache.get(key);
     this.metrics?.cpStateCache.adds.inc();
-    if (cacheItem !== undefined && isPersistedCacheItem(cacheItem)) {
-      const persistedKey = cacheItem.value;
+    // keep an existing persistedKey (persisted or reloaded-in-memory); dropping it orphans the on-disk file
+    const persistedKey = cacheItem && (isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey);
+    if (persistedKey !== undefined) {
       // was persisted to disk, set back to memory
       this.cache.set(key, {type: CacheItemType.inMemory, state, persistedKey});
       this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {

package/src/chain/validation/proposerPreferences.ts

@@ -81,7 +81,7 @@ export async function validateGossipProposerPreferences(
   }
 
   // [IGNORE] First valid message for (dependent_root, proposal_slot, validator_index).
-  if (chain.seenProposerPreferences.isKnown(dependentRootHex, proposalSlot, validatorIndex)) {
+  if (chain.proposerPreferencesPool.isKnown(proposalSlot, dependentRootHex, validatorIndex)) {
     throw new ProposerPreferencesError(GossipAction.IGNORE, {
       code: ProposerPreferencesErrorCode.ALREADY_KNOWN,
       proposalSlot,
@@ -105,6 +105,15 @@ export async function validateGossipProposerPreferences(
     });
   }
 
-  // Valid
-  chain.seenProposerPreferences.add(dependentRootHex, proposalSlot, validatorIndex);
+  // Repeated check - deals with race-condition between preferences submissions
+  if (chain.proposerPreferencesPool.isKnown(proposalSlot, dependentRootHex, validatorIndex)) {
+    throw new ProposerPreferencesError(GossipAction.IGNORE, {
+      code: ProposerPreferencesErrorCode.ALREADY_KNOWN,
+      proposalSlot,
+      validatorIndex,
+      dependentRoot: dependentRootHex,
+    });
+  }
+
+  chain.proposerPreferencesPool.add(signedProposerPreferences);
 }

package/src/network/peers/discover.ts

@@ -141,6 +141,22 @@ export class PeerDiscovery {
     this.discv5FirstQueryDelayMs = opts.discv5FirstQueryDelayMs;
     this.connectToDiscv5BootnodesOnStart = opts.connectToDiscv5Bootnodes;
 
+    // Transport tags vary by library: @libp2p/tcp uses '@libp2p/tcp', @chainsafe/libp2p-quic uses 'quic'
+    // Normalize to simple 'tcp' / 'quic' strings for matching.
+    // Must be initialized before the discovery listeners are registered and the bootENRs are processed
+    // below, since handleDiscoveredPeer() reads this.transports. Previously this was assigned at the end
+    // of the constructor, so when network.connectToDiscv5Bootnodes is enabled every bootENR was processed
+    // while this.transports was still undefined, throwing "Cannot read properties of undefined (reading
+    // 'includes')" and preventing the node from dialing its bootnodes on startup.
+    this.transports = libp2p.services.components.transportManager
+      .getTransports()
+      .map((t) => t[Symbol.toStringTag])
+      .map((tag) => {
+        if (tag?.includes("tcp")) return "tcp";
+        if (tag?.includes("quic")) return "quic";
+        return tag;
+      });
+
     this.libp2p.addEventListener("peer:discovery", this.onDiscoveredPeer);
     this.discv5.on("discovered", this.onDiscoveredENR);
 
@@ -183,17 +199,6 @@ export class PeerDiscovery {
         }
       });
     }
-
-    // Transport tags vary by library: @libp2p/tcp uses '@libp2p/tcp', @chainsafe/libp2p-quic uses 'quic'
-    // Normalize to simple 'tcp' / 'quic' strings for matching
-    this.transports = libp2p.services.components.transportManager
-      .getTransports()
-      .map((t) => t[Symbol.toStringTag])
-      .map((tag) => {
-        if (tag?.includes("tcp")) return "tcp";
-        if (tag?.includes("quic")) return "quic";
-        return tag;
-      });
   }
 
   static async init(modules: PeerDiscoveryModules, opts: PeerDiscoveryOpts): Promise<PeerDiscovery> {

package/src/network/processor/gossipHandlers.ts

@@ -165,6 +165,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     // tracked in https://github.com/ChainSafe/lodestar/issues/7957
 
     const logCtx = {
+      slot,
       currentSlot: chain.clock.currentSlot,
       peerId: peerIdStr,
       delaySec,
@@ -184,20 +185,24 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       seenTimestampSec,
       peerIdStr,
     });
+
+    // Optimistically seed the payload-envelope cache too, mirroring seenBlockInputCache above.
+    // This ensures we have PayloadEnvelopeInput, even through "PARENT_UNKNOWN" error
+    // see https://github.com/ChainSafe/lodestar/issues/9475
+    if (isForkPostGloas(fork)) {
+      chain.seenPayloadEnvelopeInputCache.add({
+        blockRootHex,
+        block: signedBlock as SignedBeaconBlock<ForkPostGloas>,
+        forkName: fork,
+        sampledColumns: chain.custodyConfig.sampledColumns,
+        custodyColumns: chain.custodyConfig.custodyColumns,
+        timeCreatedSec: seenTimestampSec,
+      });
+    }
+
     try {
       const {skippedSlots} = await validateGossipBlock(config, chain, signedBlock, fork);
 
-      if (isForkPostGloas(fork)) {
-        chain.seenPayloadEnvelopeInputCache.add({
-          blockRootHex,
-          block: signedBlock as SignedBeaconBlock<ForkPostGloas>,
-          forkName: fork,
-          sampledColumns: chain.custodyConfig.sampledColumns,
-          custodyColumns: chain.custodyConfig.custodyColumns,
-          timeCreatedSec: seenTimestampSec,
-        });
-      }
-
       const blockInputMeta = blockInput.getLogMeta();
 
       const recvToValidation = Date.now() / 1000 - seenTimestampSec;
@@ -234,12 +239,22 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           throw e;
         }
 
-        if (e.action === GossipAction.REJECT) {
-          chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
+        // IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
+        // Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
+        // (provably invalid) and unexpected errors prune below.
+        if (e.action === GossipAction.IGNORE) {
+          throw e;
         }
+
+        chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
       }
 
+      // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
+      // both caches, keeping them consistent.
       chain.seenBlockInputCache.prune(blockRootHex);
+      if (isForkPostGloas(fork)) {
+        chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+      }
       throw e;
     }
   }
@@ -1127,9 +1142,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
 
       logger.debug("Received gossip payload envelope", {
+        slot,
         currentSlot: chain.clock.currentSlot,
         peerId: peerIdStr,
-        slot,
         blockRoot: toRootHex(envelope.beaconBlockRoot),
         delaySec,
       });
@@ -1258,7 +1273,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const signedProposerPreferences = sszDeserialize(topic, serializedData);
       await validateGossipProposerPreferences(chain, signedProposerPreferences);
 
-      chain.proposerPreferencesPool.add(signedProposerPreferences);
       chain.emitter.emit(routes.events.EventType.proposerPreferences, {
         version: ForkName.gloas,
         data: signedProposerPreferences,

package/src/sync/range/chain.ts

@@ -186,10 +186,6 @@ export class SyncChain {
     this.logger = logger;
     this.logId = `${syncType}-${nextChainId++}`;
 
-    if (metrics) {
-      metrics.syncRange.headSyncPeers.addCollect(() => this.scrapeMetrics(metrics));
-    }
-
     // Trigger event on parent class
     this.sync().then(
       () => fns.onEnd(null, this.target),
@@ -775,7 +771,10 @@ export class SyncChain {
     });
   }
 
-  private scrapeMetrics(metrics: Metrics): void {
+  /**
+   * Called by `RangeSync`'s to avoid collecting metrics of removed chains.
+   */
+  scrapeMetrics(metrics: Metrics): void {
     const syncPeersMetric =
       this.syncType === RangeSyncType.Finalized
         ? metrics.syncRange.finalizedSyncPeers

package/src/sync/range/range.ts

@@ -105,6 +105,16 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
 
     if (metrics) {
       metrics.syncStatus.addCollect(() => this.scrapeMetrics(metrics));
+      metrics.syncRange.headSyncPeers.addCollect(() => {
+        // Gauges retain their last set value, so a removed chain would keep reporting stale
+        // per-column peer counts (finalizedSyncPeers especially, as it is rarely recreated once
+        // synced)
+        metrics.syncRange.headSyncPeers.reset();
+        metrics.syncRange.finalizedSyncPeers.reset();
+        for (const syncChain of this.chains.values()) {
+          syncChain.scrapeMetrics(metrics);
+        }
+      });
     }
   }
 

package/src/sync/utils/downloadByRange.ts

@@ -142,6 +142,7 @@ export function cacheByRangeResponses({
     const blockRootHex = toRootHex(blockRoot);
 
     const existing = updatedBatchBlocks.get(block.message.slot);
+    let blockInput: IBlockInput;
     if (existing) {
       // In practice this code block shouldn't be reached because we shouldn't be refetching a block we already have, see Batch#getRequests.
       // Will throw if root hex does not match (meaning we are following the wrong chain)
@@ -155,8 +156,9 @@ export function cacheByRangeResponses({
         },
         {throwOnDuplicateAdd: false}
       );
+      blockInput = existing;
     } else {
-      const blockInput = cache.getByBlock({
+      blockInput = cache.getByBlock({
         block,
         blockRootHex,
         source,
@@ -165,6 +167,21 @@ export function cacheByRangeResponses({
       });
       updatedBatchBlocks.set(blockInput.slot, blockInput);
     }
+
+    // Seed seenPayloadEnvelopeInputCache the instant we have the block, before the blob loop (or any
+    // later step) can throw and abort the batch — otherwise a gloas block would sit in
+    // seenBlockInputCache but unseeded here, and payload-by-root sync would later throw "Missing
+    // PayloadEnvelopeInput for known block" (see issue #9306). add() is idempotent.
+    if (isForkPostGloas(blockInput.forkName)) {
+      seenPayloadEnvelopeInputCache.add({
+        blockRootHex: blockInput.blockRootHex,
+        block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
+        forkName: blockInput.forkName,
+        sampledColumns: custodyConfig.sampledColumns,
+        custodyColumns: custodyConfig.custodyColumns,
+        timeCreatedSec: seenTimestampSec,
+      });
+    }
   }
 
   for (const {blockRoot, blobSidecars} of responses.validatedBlobSidecars ?? []) {
@@ -205,22 +222,6 @@ export function cacheByRangeResponses({
     }
   }
 
-  // Seed seenPayloadEnvelopeInputCache for every gloas block in the batch, regardless of whether
-  // the peer returned its envelope. Without this, a block returned without its envelope would be
-  // imported with no cache entry, and later payload-by-root sync would throw
-  // "Missing PayloadEnvelopeInput for known block" (see issue #9306).
-  for (const blockInput of updatedBatchBlocks.values()) {
-    if (!blockInput.hasBlock() || !isForkPostGloas(blockInput.forkName)) continue;
-    seenPayloadEnvelopeInputCache.add({
-      blockRootHex: blockInput.blockRootHex,
-      block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
-      forkName: blockInput.forkName,
-      sampledColumns: custodyConfig.sampledColumns,
-      custodyColumns: custodyConfig.custodyColumns,
-      timeCreatedSec: seenTimestampSec,
-    });
-  }
-
   let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null =
     existingPayloadEnvelopes !== null ? new Map(existingPayloadEnvelopes) : null;
   if (downloadedPayloadEnvelopes !== null) {
@@ -229,8 +230,8 @@ export function cacheByRangeResponses({
       const envelopeBlockRootHex = toRootHex(envelope.message.beaconBlockRoot);
       const payloadInput = seenPayloadEnvelopeInputCache.get(envelopeBlockRootHex);
       if (payloadInput === undefined) {
-        // Unreachable given the loop above seeded an entry for every gloas block in the batch.
-        // for the parent block, it's populated at BeaconChain init
+        // Unreachable given the validatedBlocks loop above seeded an entry for every gloas block in
+        // the batch. for the parent block, it's populated at BeaconChain init
         throw new Error(`Missing PayloadEnvelopeInput for block ${envelopeBlockRootHex}`);
       }
 
@@ -980,6 +981,12 @@ export async function validateColumnsByRangeResponse(
           missingIndices: prettyPrintIndices(request.columns),
         })
       );
+      // Gloas: the bid commits to blobs even for payloads that get orphaned, whose columns are
+      // pruned and unservable. Skip instead of truncating + re-requesting forever; DA is enforced
+      // on envelope processing. Pre-Gloas commitments are canonical, so keep strict.
+      if (isForkPostGloas(forkName)) {
+        continue;
+      }
       break;
     }