@lodestar/beacon-node 1.47.0-dev.cec7fd63e3 → 1.47.0-dev.d79d81f994

package/src/network/network.ts

@@ -49,7 +49,12 @@ import {INetworkCore, NetworkCore, WorkerNetworkCore} from "./core/index.js";
 import {INetworkEventBus, NetworkEvent, NetworkEventBus, NetworkEventData} from "./events.js";
 import {getActiveForkBoundaries} from "./forks.js";
 import {GossipHandlers, GossipTopicMap, GossipType, GossipTypeMap} from "./gossip/index.js";
-import {getGossipSSZType, gossipTopicIgnoreDuplicatePublishError, stringifyGossipTopic} from "./gossip/topic.js";
+import {
+  getGossipSSZType,
+  gossipTopicAllowPublishToZeroPeers,
+  gossipTopicIgnoreDuplicatePublishError,
+  stringifyGossipTopic,
+} from "./gossip/topic.js";
 import {INetwork} from "./interface.js";
 import {NetworkOptions} from "./options.js";
 import {PeerAction, PeerScoreStats} from "./peers/index.js";
@@ -65,7 +70,7 @@ import {
 } from "./reqresp/utils/collect.js";
 import {collectSequentialBlocksInRange} from "./reqresp/utils/collectSequentialBlocksInRange.js";
 import {CommitteeSubscription} from "./subnets/index.js";
-import {isPublishDuplicateError, isPublishToZeroPeersError, prettyPrintPeerIdStr} from "./util.js";
+import {isPublishDuplicateError, prettyPrintPeerIdStr} from "./util.js";
 
 type NetworkModules = {
   opts: NetworkOptions;
@@ -352,9 +357,7 @@ export class Network implements INetwork {
     const epoch = computeEpochAtSlot(signedBlock.message.slot);
     const boundary = this.config.getForkBoundaryAtEpoch(epoch);
 
-    return this.publishGossip<GossipType.beacon_block>({type: GossipType.beacon_block, boundary}, signedBlock, {
-      ignoreDuplicatePublishError: true,
-    });
+    return this.publishGossip<GossipType.beacon_block>({type: GossipType.beacon_block, boundary}, signedBlock);
   }
 
   async publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise<number> {
@@ -363,9 +366,7 @@ export class Network implements INetwork {
 
     const subnet = blobSidecar.index;
 
-    return this.publishGossip<GossipType.blob_sidecar>({type: GossipType.blob_sidecar, boundary, subnet}, blobSidecar, {
-      ignoreDuplicatePublishError: true,
-    });
+    return this.publishGossip<GossipType.blob_sidecar>({type: GossipType.blob_sidecar, boundary, subnet}, blobSidecar);
   }
 
   async publishDataColumnSidecar(
@@ -381,14 +382,7 @@ export class Network implements INetwork {
     try {
       const sentPeers = await this.publishGossip<GossipType.data_column_sidecar>(
         {type: GossipType.data_column_sidecar, boundary, subnet},
-        dataColumnSidecar,
-        {
-          // we ensure having all topic peers via prioritizePeers() function
-          // in the worse case, if there is 0 peer on the topic, the overall publish operation could be still a success
-          // because supernode will rebuild and publish missing data column sidecars for us
-          // hence we want to track sent peers as 0 instead of an error
-          allowPublishToZeroTopicPeers: true,
-        }
+        dataColumnSidecar
       );
       return {sentPeers, alreadyPublished: false};
     } catch (e) {
@@ -405,8 +399,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.beacon_aggregate_and_proof>(
       {type: GossipType.beacon_aggregate_and_proof, boundary},
-      aggregateAndProof,
-      {ignoreDuplicatePublishError: true}
+      aggregateAndProof
     );
   }
 
@@ -416,8 +409,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.beacon_attestation>(
       {type: GossipType.beacon_attestation, boundary, subnet},
-      attestation,
-      {ignoreDuplicatePublishError: true}
+      attestation
     );
   }
 
@@ -425,9 +417,7 @@ export class Network implements INetwork {
     const epoch = voluntaryExit.message.epoch;
     const boundary = this.config.getForkBoundaryAtEpoch(epoch);
 
-    return this.publishGossip<GossipType.voluntary_exit>({type: GossipType.voluntary_exit, boundary}, voluntaryExit, {
-      ignoreDuplicatePublishError: true,
-    });
+    return this.publishGossip<GossipType.voluntary_exit>({type: GossipType.voluntary_exit, boundary}, voluntaryExit);
   }
 
   async publishBlsToExecutionChange(blsToExecutionChange: capella.SignedBLSToExecutionChange): Promise<number> {
@@ -438,8 +428,7 @@ export class Network implements INetwork {
       if (fork >= ForkSeq.capella) {
         const publishPromise = this.publishGossip<GossipType.bls_to_execution_change>(
           {type: GossipType.bls_to_execution_change, boundary},
-          blsToExecutionChange,
-          {ignoreDuplicatePublishError: true}
+          blsToExecutionChange
         );
         publishChanges.push(publishPromise);
       }
@@ -457,8 +446,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.proposer_slashing>(
       {type: GossipType.proposer_slashing, boundary},
-      proposerSlashing,
-      {ignoreDuplicatePublishError: true}
+      proposerSlashing
     );
   }
 
@@ -478,10 +466,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.sync_committee>(
       {type: GossipType.sync_committee, boundary, subnet},
-      signature,
-      {
-        ignoreDuplicatePublishError: true,
-      }
+      signature
     );
   }
 
@@ -491,8 +476,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.sync_committee_contribution_and_proof>(
       {type: GossipType.sync_committee_contribution_and_proof, boundary},
-      contributionAndProof,
-      {ignoreDuplicatePublishError: true}
+      contributionAndProof
     );
   }
 
@@ -522,8 +506,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.execution_payload>(
       {type: GossipType.execution_payload, boundary},
-      signedEnvelope,
-      {ignoreDuplicatePublishError: true}
+      signedEnvelope
     );
   }
 
@@ -534,7 +517,7 @@ export class Network implements INetwork {
     return this.publishGossip<GossipType.execution_payload_bid>(
       {type: GossipType.execution_payload_bid, boundary},
       signedBid,
-      {ignoreDuplicatePublishError: true}
+      {floodPublish: true}
     );
   }
 
@@ -544,8 +527,7 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.payload_attestation_message>(
       {type: GossipType.payload_attestation_message, boundary},
-      payloadAttestationMessage,
-      {ignoreDuplicatePublishError: true}
+      payloadAttestationMessage
     );
   }
 
@@ -555,24 +537,25 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.proposer_preferences>(
       {type: GossipType.proposer_preferences, boundary},
-      signedProposerPreferences,
-      {ignoreDuplicatePublishError: true}
+      signedProposerPreferences
     );
   }
 
   private async publishGossip<K extends GossipType>(
     topic: GossipTopicMap[K],
     object: GossipTypeMap[K],
-    opts?: PublishOpts | undefined
+    opts?: PublishOpts
   ): Promise<number> {
     const topicStr = stringifyGossipTopic(this.config, topic);
     const sszType = getGossipSSZType(topic);
     const messageData = (sszType.serialize as (object: GossipTypeMap[GossipType]) => Uint8Array)(object);
-    opts = {
-      ...opts,
+    const publishOpts: PublishOpts = {
       ignoreDuplicatePublishError: gossipTopicIgnoreDuplicatePublishError[topic.type],
+      // Leave undefined unless the topic opts out, so `--network.allowPublishToZeroPeers` still applies
+      allowPublishToZeroTopicPeers: gossipTopicAllowPublishToZeroPeers[topic.type] ? true : undefined,
+      ...opts,
     };
-    const sentPeers = await this.core.publishGossip(topicStr, messageData, opts);
+    const sentPeers = await this.core.publishGossip(topicStr, messageData, publishOpts);
 
     this.logger.verbose("Publish to topic", {topic: topicStr, sentPeers, currentSlot: this.clock.currentSlot});
     return sentPeers;
@@ -813,11 +796,7 @@ export class Network implements INetwork {
       await this.waitForSyncMessageCutoff(finalityUpdate.signatureSlot);
       await this.publishLightClientFinalityUpdate(finalityUpdate);
     } catch (e) {
-      // Non-mandatory route on most of network as of Oct 2022. May not have found any peers on topic yet
-      // Remove once https://github.com/ChainSafe/js-libp2p-gossipsub/issues/367
-      if (!isPublishToZeroPeersError(e as Error)) {
-        this.logger.debug("Error on BeaconGossipHandler.onLightclientFinalityUpdate", {}, e as Error);
-      }
+      this.logger.debug("Error on BeaconGossipHandler.onLightclientFinalityUpdate", {}, e as Error);
     }
   };
 
@@ -830,11 +809,7 @@ export class Network implements INetwork {
       await this.waitForSyncMessageCutoff(optimisticUpdate.signatureSlot);
       await this.publishLightClientOptimisticUpdate(optimisticUpdate);
     } catch (e) {
-      // Non-mandatory route on most of network as of Oct 2022. May not have found any peers on topic yet
-      // Remove once https://github.com/ChainSafe/js-libp2p-gossipsub/issues/367
-      if (!isPublishToZeroPeersError(e as Error)) {
-        this.logger.debug("Error on BeaconGossipHandler.onLightclientOptimisticUpdate", {}, e as Error);
-      }
+      this.logger.debug("Error on BeaconGossipHandler.onLightclientOptimisticUpdate", {}, e as Error);
     }
   };
 

package/src/network/processor/constants.ts

@@ -0,0 +1,6 @@
+/**
+ * Max distinct peers tracked per unknown block/payload root.
+ * Normally we have 8 mesh peers per topic, and multi topics may point to the same root.
+ * Having 32 peers per root seems enough to search for missing blocks/payloads.
+ **/
+export const MAX_PEERS_PER_ROOT = 32;

package/src/network/processor/gossipHandlers.ts

@@ -196,7 +196,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         forkName: fork,
         sampledColumns: chain.custodyConfig.sampledColumns,
         custodyColumns: chain.custodyConfig.custodyColumns,
-        timeCreatedSec: seenTimestampSec,
+        seenTimestampSec,
+        source: PayloadEnvelopeInputSource.gossip,
       });
     }
 
@@ -640,6 +641,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec);
 
         if (isForkPostGloas(blockInput.forkName)) {
+          // we should have the payloadInput in the seen cache so no need getOrReload() here
           const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockInput.blockRootHex);
           // This payloadInput should have been created just after gossip validation
           if (!payloadInput) {
@@ -1189,6 +1191,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.gossip, delaySec, signedEnvelope);
 
       const blockRootHex = toRootHex(envelope.beaconBlockRoot);
+      // a gossip payload cannot be `MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE` slots after the block
+      // otherwise it'll get to UnknownBlockInput flow
       const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
 
       if (!payloadInput) {
@@ -1255,11 +1259,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.payload_attestation_message]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.payload_attestation_message>) => {
       const {serializedData} = gossipData;
       const payloadAttestationMessage = sszDeserialize(topic, serializedData);
       const validationResult = await validateGossipPayloadAttestationMessage(chain, payloadAttestationMessage);
 
+      const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
+      metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec);
+
       try {
         const insertOutcome = chain.payloadAttestationPool.add(
           payloadAttestationMessage,
@@ -1286,14 +1294,19 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.execution_payload_bid]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload_bid>) => {
       const {serializedData} = gossipData;
       const executionPayloadBid = sszDeserialize(topic, serializedData);
       const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, executionPayloadBid);
 
+      // this could be negative, because it's most likely the bid of next slot comes at this clock slot
+      const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.gossip}, elapsedSec);
+
       // Handle valid payload bid by storing in a bid pool
       try {
-        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);
+        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid, Math.floor(elapsedSec * 1000));
         metrics?.opPool.executionPayloadBidPool.gossipInsertOutcome.inc({insertOutcome});
       } catch (e) {
         logger.error("Error adding to executionPayloadBid pool", {}, e as Error);

package/src/network/processor/gossipQueues/indexed.ts

@@ -86,7 +86,6 @@ export class IndexedGossipQueueMinSize<T extends {indexed?: string; queueAddedMs
     const now = Date.now();
     // here we mutate item, which is used for gossip validation later
     item.indexed = key;
-    item.queueAddedMs = now;
     let queueItem = this.indexedItems.get(key);
     if (queueItem == null) {
       queueItem = {firstSeenMs: now, listItems: new LinkedList<T>()};

package/src/network/processor/index.ts

@@ -30,6 +30,7 @@ import {
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
+import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
@@ -191,8 +192,12 @@ export class NetworkProcessor {
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
   private awaitingPayloadMessageCount = 0;
-  private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
-  private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private unknownBlocksBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
+  private unknownEnvelopesBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
 
   constructor(
     modules: NetworkProcessorModules,
@@ -271,35 +276,53 @@ export class NetworkProcessor {
    * Search block via `ChainEvent.unknownBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * We forward every distinct peer that gossiped this root (not just the first) so BlockInputSync can
+   * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenBlock(root) ||
-      this.awaitingMessagesByBlockRoot.has(root) ||
-      this.unknownBlocksBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenBlock(root)) {
       return;
     }
-    // Search for the unknown block
-    this.unknownBlocksBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    const peersForRoot = this.unknownBlocksBySlot.getOrDefault(slot);
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    // brand-new search always emits (peer may be undefined)
+    let shouldEmit = !alreadySearching;
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    }
   }
 
   /**
    * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
   searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenPayloadEnvelope(root) ||
-      this.awaitingMessagesByPayloadBlockRoot.has(root) ||
-      this.unknownEnvelopesBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
-    this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    const peersForRoot = this.unknownEnvelopesBySlot.getOrDefault(slot);
+    // capture "already searching" BEFORE getOrDefault(root) creates the entry
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByPayloadBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    let shouldEmit = !alreadySearching; // brand-new search always emits (peer may be undefined)
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    }
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -537,6 +560,7 @@ export class NetworkProcessor {
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
+    message.queueAddedMs = Date.now();
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
       // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
@@ -616,7 +640,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownBlocksBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -641,7 +665,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownEnvelopesBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -723,12 +747,20 @@ export class NetworkProcessor {
       for (const msg of messageOrArray) {
         msg.startProcessUnixSec = nowSec;
         if (msg.queueAddedMs !== undefined) {
-          this.metrics?.gossipValidationQueue.queueTime.observe(nowSec - msg.queueAddedMs / 1000);
+          this.metrics?.gossipValidationQueue.queueTime.observe(
+            {topic: msg.topic.type},
+            nowSec - msg.queueAddedMs / 1000
+          );
         }
       }
     } else {
-      // indexed queue is not used here
       messageOrArray.startProcessUnixSec = nowSec;
+      if (messageOrArray.queueAddedMs !== undefined) {
+        this.metrics?.gossipValidationQueue.queueTime.observe(
+          {topic: messageOrArray.topic.type},
+          nowSec - messageOrArray.queueAddedMs / 1000
+        );
+      }
     }
 
     const acceptanceArr = Array.isArray(messageOrArray)

package/src/network/reqresp/score.ts

@@ -44,17 +44,11 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
         // failure, so keep the stronger penalty (Fatal for Ping, as before).
         return method === ReqRespMethod.Ping ? PeerAction.Fatal : PeerAction.LowToleranceError;
       }
-      switch (method) {
-        // Ping and Status are liveness probes; their dial timeouts are dominated by transient
-        // network congestion rather than peer misbehavior, so penalize leniently to avoid
-        // self-inflicted peer starvation (https://github.com/ChainSafe/lodestar/issues/9562),
-        // while still applying some penalty so genuinely dead peers eventually free the slot.
-        case ReqRespMethod.Ping:
-        case ReqRespMethod.Status:
-          return PeerAction.HighToleranceError;
-        default:
-          return PeerAction.LowToleranceError;
-      }
+      // A dial failure means we could not open a stream. That is evidence about the transport, not
+      // about the peer's behavior, and it is dominated by transient congestion on our side, so
+      // penalize leniently for every method to avoid self-inflicted peer starvation (#9562). A
+      // genuinely dead peer still accumulates penalty and is eventually evicted, freeing the slot.
+      return PeerAction.HighToleranceError;
     // TODO: Detect SSZDecodeError and return PeerAction.Fatal
 
     case RequestErrorCode.RESP_TIMEOUT:

package/src/network/util.ts

@@ -23,11 +23,6 @@ export function getConnection(libp2p: Libp2p, peerIdStr: string): Connection | u
   return getConnectionsMap(libp2p).get(peerIdStr)?.value[0] ?? undefined;
 }
 
-// https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076
-export function isPublishToZeroPeersError(e: Error): boolean {
-  return e.message.includes("PublishError.NoPeersSubscribedToTopic");
-}
-
 // https://github.com/libp2p/js-libp2p/blob/f87cba928991736d9646b3e054c367f55cab315c/packages/gossipsub/src/gossipsub.ts#L2076
 export function isPublishDuplicateError(e: Error): boolean {
   return e.message.includes("PublishError.Duplicate");

package/src/node/deferredVoluntaryExitPublisher.ts

@@ -0,0 +1,54 @@
+import {routes} from "@lodestar/api";
+import {Logger} from "@lodestar/utils";
+import {IBeaconChain} from "../chain/index.js";
+import {INetwork} from "../network/index.js";
+import {ClockEvent} from "../util/clock.js";
+
+/**
+ * On every epoch, drain the deferred voluntary exit pool and publish the exits that have become
+ * processable. Lives at the node layer rather than inside `BeaconChain` because publishing needs
+ * `network`, which the chain does not (and should not) depend on.
+ */
+export function startDeferredVoluntaryExitPublisher({
+  chain,
+  network,
+  logger,
+  signal,
+}: {
+  chain: IBeaconChain;
+  network: INetwork;
+  logger: Logger;
+  signal: AbortSignal;
+}): void {
+  const onEpoch = async (): Promise<void> => {
+    try {
+      // Use the cached head state directly rather than getHeadStateAtCurrentEpoch, which would regen
+      // the head forward to the current epoch. At init from a stale db-finalized state that regen can
+      // be many epoch transitions. The head state is fine here: this is a best-effort drain that
+      // re-runs every epoch, so a just-processable exit is republished next tick.
+      const state = chain.getHeadState();
+      const exits = chain.deferredVoluntaryExitPool.drainProcessableExits(state);
+      for (const exit of exits) {
+        try {
+          chain.opPool.insertVoluntaryExit(exit);
+          chain.emitter.emit(routes.events.EventType.voluntaryExit, exit);
+          await network.publishVoluntaryExit(exit);
+          logger.info("Voluntary exit successfully published for validator", {
+            validatorIndex: exit.message.validatorIndex,
+          });
+        } catch (e) {
+          logger.warn(
+            "Failed to publish deferred voluntary exit",
+            {validatorIndex: exit.message.validatorIndex},
+            e as Error
+          );
+        }
+      }
+    } catch (e) {
+      logger.warn("Failed to drain deferred voluntary exit pool", {}, e as Error);
+    }
+  };
+
+  chain.clock.addListener(ClockEvent.epoch, onEpoch);
+  signal.addEventListener("abort", () => chain.clock.removeListener(ClockEvent.epoch, onEpoch), {once: true});
+}

package/src/node/nodejs.ts

@@ -1,12 +1,13 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -21,6 +22,7 @@ import {Network, getReqRespHandlers} from "../network/index.js";
 import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
 import {Clock} from "../util/clock.js";
+import {startDeferredVoluntaryExitPublisher} from "./deferredVoluntaryExitPublisher.js";
 import {runNodeNotifier} from "./notifier.js";
 import {IBeaconNodeOptions} from "./options.js";
 
@@ -258,6 +260,11 @@ export class BeaconNode {
       executionBuilder: opts.executionBuilder.enabled
         ? initializeExecutionBuilder(opts.executionBuilder, config, metrics, logger)
         : undefined,
+      builderApiClientOpts: {
+        timeout: opts.executionBuilder.timeout,
+        // Sent with all builder api requests, unless the node runs in private mode
+        userAgent: opts.executionBuilder.userAgent,
+      },
     });
 
     // Load persisted data from disk to in-memory caches
@@ -334,6 +341,8 @@ export class BeaconNode {
 
     void runNodeNotifier({network, chain, sync, config, logger, signal});
 
+    startDeferredVoluntaryExitPublisher({chain, network, logger, signal});
+
     return new BeaconNode({
       opts,
       config,