@lodestar/beacon-node 1.51.0-dev.e21949cfcf → 1.51.0-dev.e2bf489dd3

package/src/network/network.ts

@@ -30,7 +30,7 @@ import {
   isGloasDataColumnSidecar,
   phase0,
 } from "@lodestar/types";
-import {prettyPrintIndices, sleep} from "@lodestar/utils";
+import {defer, prettyPrintIndices, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
 import {ChainEvent, IBeaconChain} from "../chain/index.js";
 import {computeSubnetForDataColumnSidecar} from "../chain/validation/dataColumnSidecar.js";
@@ -128,6 +128,7 @@ export class Network implements INetwork {
 
   private subscribedToCoreTopics = false;
   private connectedPeersSyncMeta = new Map<PeerIdStr, Omit<PeerSyncMeta, "peerId">>();
+  private closePromise: Promise<void> | undefined;
 
   constructor(modules: NetworkModules) {
     this.peerId = peerIdFromPrivateKey(modules.privateKey);
@@ -145,12 +146,8 @@ export class Network implements INetwork {
     this.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
     this.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
     this.chain.emitter.on(routes.events.EventType.head, this.onHead);
-    this.chain.emitter.on(routes.events.EventType.lightClientFinalityUpdate, ({data}) =>
-      this.onLightClientFinalityUpdate(data)
-    );
-    this.chain.emitter.on(routes.events.EventType.lightClientOptimisticUpdate, ({data}) =>
-      this.onLightClientOptimisticUpdate(data)
-    );
+    this.chain.emitter.on(routes.events.EventType.lightClientFinalityUpdate, this.onLightClientFinalityUpdate);
+    this.chain.emitter.on(routes.events.EventType.lightClientOptimisticUpdate, this.onLightClientOptimisticUpdate);
     this.chain.emitter.on(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.on(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.on(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
@@ -240,9 +237,16 @@ export class Network implements INetwork {
     return this.controller.signal.aborted;
   }
 
-  /** Destroy this instance. Can only be called once. */
-  async close(): Promise<void> {
-    if (this.closed) return;
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    this.controller.abort();
 
     this.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
     this.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
@@ -254,10 +258,12 @@ export class Network implements INetwork {
     this.chain.emitter.off(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
     this.chain.emitter.off(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.off(ChainEvent.updateStatus, this.onUpdateStatus);
-    await this.core.close();
-
-    // Used only for sleep() statements
-    this.controller.abort();
+    this.connectedPeersSyncMeta.clear();
+    this.subscribedToCoreTopics = false;
+    const results = await Promise.allSettled([this.networkProcessor.stop(), this.core.close()]);
+    for (const result of results) {
+      if (result.status === "rejected") throw result.reason;
+    }
     this.logger.debug("network core closed");
   }
 
@@ -333,6 +339,7 @@ export class Network implements INetwork {
   async subscribeGossipCoreTopics(): Promise<void> {
     if (!this.subscribedToCoreTopics) {
       await this.core.subscribeGossipCoreTopics();
+      if (this.closed) return;
       // Only mark subscribedToCoreTopics if worker resolved this call
       this.subscribedToCoreTopics = true;
     }
@@ -803,26 +810,32 @@ export class Network implements INetwork {
     return this.core.writeDiscv5HeapSnapshot(prefix, dirpath);
   }
 
-  private onLightClientFinalityUpdate = async (finalityUpdate: LightClientFinalityUpdate): Promise<void> => {
+  private onLightClientFinalityUpdate = async ({
+    data: finalityUpdate,
+  }: routes.events.EventData[routes.events.EventType.lightClientFinalityUpdate]): Promise<void> => {
     // TODO: Review is OK to remove if (this.hasAttachedSyncCommitteeMember())
 
     try {
       // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired
       // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee
       await this.waitForSyncMessageCutoff(finalityUpdate.signatureSlot);
+      if (this.closed) return;
       await this.publishLightClientFinalityUpdate(finalityUpdate);
     } catch (e) {
       this.logger.debug("Error on BeaconGossipHandler.onLightclientFinalityUpdate", {}, e as Error);
     }
   };
 
-  private onLightClientOptimisticUpdate = async (optimisticUpdate: LightClientOptimisticUpdate): Promise<void> => {
+  private onLightClientOptimisticUpdate = async ({
+    data: optimisticUpdate,
+  }: routes.events.EventData[routes.events.EventType.lightClientOptimisticUpdate]): Promise<void> => {
     // TODO: Review is OK to remove if (this.hasAttachedSyncCommitteeMember())
 
     try {
       // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired
       // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee
       await this.waitForSyncMessageCutoff(optimisticUpdate.signatureSlot);
+      if (this.closed) return;
       await this.publishLightClientOptimisticUpdate(optimisticUpdate);
     } catch (e) {
       this.logger.debug("Error on BeaconGossipHandler.onLightclientOptimisticUpdate", {}, e as Error);
@@ -859,10 +872,12 @@ export class Network implements INetwork {
     this.connectedPeersSyncMeta.delete(data.peer);
   };
 
-  private onTargetGroupCountUpdated = (count: number): void => {
-    this.core.setTargetGroupCount(count).catch((e) => {
-      this.logger.debug("Error setting target custody group count", {count}, e as Error);
-    });
+  private onTargetGroupCountUpdated = async (count: number): Promise<void> => {
+    try {
+      await this.core.setTargetGroupCount(count);
+    } catch (error) {
+      this.logger.error("Error updating network custody group count", {count}, error as Error);
+    }
   };
 
   private onPublishDataColumns = async (sidecars: DataColumnSidecar[]): Promise<void> => {
@@ -886,6 +901,10 @@ export class Network implements INetwork {
   };
 
   private onUpdateStatus = async (): Promise<void> => {
-    await this.core.updateStatus(this.chain.getStatus());
+    try {
+      await this.core.updateStatus(this.chain.getStatus());
+    } catch (error) {
+      this.logger.error("Error updating network status", {}, error as Error);
+    }
   };
 }

package/src/network/processor/gossipHandlers.ts

@@ -1377,6 +1377,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                   // Errors might indicate an issue with our node or the connected EL client
                   logLevel = LogLevel.error;
                   break;
+                case PayloadErrorCode.BLOCK_EXECUTION_INVALID:
+                  logLevel = LogLevel.warn;
+                  break;
                 // INVALID_SIGNATURE only happens for an envelope added by another source
                 case PayloadErrorCode.INVALID_SIGNATURE:
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:

package/src/network/processor/index.ts

@@ -244,6 +244,12 @@ type SearchUnknownRootTarget =
  * Such that enough work is processed to fill either one of the queue.
  */
 export class NetworkProcessor {
+  private stopped = false;
+  /**
+   * Replaced by dropAllJobs(). Loops that yield while re-queueing awaiting messages compare it after each yield
+   * and stop, so no message is queued after a drop.
+   */
+  private jobGeneration = Symbol();
   private readonly chain: IBeaconChain;
   private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
@@ -335,16 +341,27 @@ export class NetworkProcessor {
   }
 
   async stop(): Promise<void> {
+    if (this.stopped) return;
+    this.stopped = true;
     this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
-    this.chain.emitter.off(ClockEvent.slot, this.onClockSlot);
+    this.chain.clock.off(ClockEvent.slot, this.onClockSlot);
+    this.dropAllJobs();
   }
 
   dropAllJobs(): void {
+    this.jobGeneration = Symbol();
     for (const topic of executeGossipWorkOrder) {
       this.gossipQueues[topic].clear();
     }
+    this.awaitingMessagesByBlockRoot.clear();
+    this.awaitingBlockMessageCount = 0;
+    this.awaitingMessagesByPayloadBlockRoot.clear();
+    this.awaitingPayloadMessageCount = 0;
+    this.searchedRootsBySlot.clear();
+    this.bufferedRootsBySlot.clear();
+    this.fatalPeers.clear();
   }
 
   dumpGossipQueue(topic: GossipType): PendingGossipsubMessage[] {
@@ -364,7 +381,7 @@ export class NetworkProcessor {
    * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (this.chain.seenBlock(root)) {
+    if (this.stopped || this.chain.seenBlock(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -398,7 +415,7 @@ export class NetworkProcessor {
     peer?: PeerIdStr,
     slotIsPayloadSlot = false
   ): void {
-    if (this.chain.seenPayloadEnvelope(root)) {
+    if (this.stopped || this.chain.seenPayloadEnvelope(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -801,6 +818,7 @@ export class NetworkProcessor {
       this.awaitingBlockMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -819,6 +837,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -843,6 +862,7 @@ export class NetworkProcessor {
       this.awaitingPayloadMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const messages of messagesByTopic.values()) {
@@ -858,6 +878,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -937,6 +958,7 @@ export class NetworkProcessor {
   };
 
   private executeWork(): void {
+    if (this.stopped) return;
     // TODO: Maybe de-bounce by timing the last time executeWork was run
 
     this.metrics?.networkProcessor.executeWorkCalls.inc();

package/src/util/execution.ts

@@ -42,7 +42,7 @@ export enum DataColumnEngineResult {
 
 export async function getBlobSidecarsFromExecution(
   config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getBlobs">,
   metrics: Metrics | null,
   emitter: ChainEventEmitter,
   blockInput: IBlockInput
@@ -140,7 +140,7 @@ export async function getBlobSidecarsFromExecution(
  */
 export async function getDataColumnSidecarsFromExecution(
   config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getBlobs">,
   emitter: ChainEventEmitter,
   input: IBlockInput | PayloadEnvelopeInput,
   metrics: Metrics | null,
@@ -289,7 +289,7 @@ export type RebuildMiss = {slot: Slot; reason: "unavailable" | "mismatch"; error
  */
 export async function* reconstructExecutionPayloadEnvelopesByRange(
   db: IBeaconDb,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   logger: Logger,
   metrics: Metrics | null,
   startSlot: Slot,
@@ -315,7 +315,7 @@ export async function* reconstructExecutionPayloadEnvelopesByRange(
  * RANGE_UNSERVABLE for it
  */
 async function* reconstructBatch(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   logger: Logger,
   metrics: Metrics | null,
   batch: RangeEntry[]
@@ -355,7 +355,7 @@ async function* reconstructBatch(
  * on its path. Throws ENGINE_UNAVAILABLE only.
  */
 export async function reconstructExecutionPayloadEnvelopes(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   metrics: Metrics | null,
   headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
 ): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
@@ -382,7 +382,7 @@ export function isRebuildMiss(result: gloas.SignedExecutionPayloadEnvelope | Reb
  * incremented once, in this function.
  */
 async function reconstructEnvelopesBatch(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   metrics: Metrics | null,
   headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
 ): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {