@lodestar/beacon-node 1.41.0-dev.49f680d9f0 → 1.41.0-dev.4e115ed25d

package/src/network/gossip/encoding.ts

@@ -1,9 +1,9 @@
-import {Message} from "@libp2p/interface";
+import type {Message} from "@libp2p/gossipsub";
+import type {RPC} from "@libp2p/gossipsub/message";
+import type {DataTransform} from "@libp2p/gossipsub/types";
 // snappyjs is better for compression for smaller payloads
 import xxhashFactory from "xxhash-wasm";
 import {digest} from "@chainsafe/as-sha256";
-import {RPC} from "@chainsafe/libp2p-gossipsub/message";
-import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";
 import snappyWasm from "@chainsafe/snappy-wasm";
 import {ForkName} from "@lodestar/params";
 import {intToBytes} from "@lodestar/utils";

package/src/network/gossip/gossipsub.ts

@@ -1,10 +1,18 @@
+import {
+  type GossipSub,
+  type GossipSubEvents,
+  type PublishResult,
+  StrictNoSign,
+  type TopicValidatorResult,
+  gossipsub,
+} from "@libp2p/gossipsub";
+import type {MetricsRegister, TopicLabel, TopicStrToLabel} from "@libp2p/gossipsub/metrics";
+import type {PeerScoreParams, PeerScoreStatsDump} from "@libp2p/gossipsub/score";
+import type {AddrInfo, PublishOpts, TopicStr} from "@libp2p/gossipsub/types";
+import type {PeerId} from "@libp2p/interface";
 import {peerIdFromString} from "@libp2p/peer-id";
 import {multiaddr} from "@multiformats/multiaddr";
 import {ENR} from "@chainsafe/enr";
-import {GossipSub, GossipsubEvents} from "@chainsafe/libp2p-gossipsub";
-import {MetricsRegister, TopicLabel, TopicStrToLabel} from "@chainsafe/libp2p-gossipsub/metrics";
-import {PeerScoreParams} from "@chainsafe/libp2p-gossipsub/score";
-import {AddrInfo, SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types";
 import {routes} from "@lodestar/api";
 import {BeaconConfig, ForkBoundary} from "@lodestar/config";
 import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
@@ -69,6 +77,24 @@ export type Eth2GossipsubOpts = {
 
 export type ForkBoundaryLabel = string;
 
+// Many of the internal properties we need are not available on the public interface,
+// so we create an extended type here to avoid excessive type assertions throughout the codebase.
+// Mind that any updates to the gossipsub package may require updates to this type.
+type GossipSubInternal = GossipSub & {
+  mesh: Map<string, Set<string>>;
+  peers: Map<string, PeerId>;
+  score: {score: (peerIdStr: string) => number};
+  direct: Set<string>;
+  topics: Map<string, Set<string>>;
+  start: () => Promise<void>;
+  stop: () => Promise<void>;
+  publish: (topic: TopicStr, data: Uint8Array, opts?: PublishOpts) => Promise<PublishResult>;
+  getMeshPeers: (topic: TopicStr) => string[];
+  dumpPeerScoreStats: () => PeerScoreStatsDump;
+  getScore: (peerIdStr: string) => number;
+  reportMessageValidationResult: (msgId: string, propagationSource: string, acceptance: TopicValidatorResult) => void;
+};
+
 /**
  * Wrapper around js-libp2p-gossipsub with the following extensions:
  * - Eth2 message id
@@ -82,13 +108,14 @@ export type ForkBoundaryLabel = string;
  *
  * See https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
  */
-export class Eth2Gossipsub extends GossipSub {
+export class Eth2Gossipsub {
   readonly scoreParams: Partial<PeerScoreParams>;
   private readonly config: BeaconConfig;
   private readonly logger: Logger;
   private readonly peersData: PeersData;
   private readonly events: NetworkEventBus;
   private readonly libp2p: Libp2p;
+  private readonly gossipsub: GossipSubInternal;
 
   // Internal caches
   private readonly gossipTopicCache: GossipTopicCache;
@@ -103,9 +130,6 @@ export class Eth2Gossipsub extends GossipSub {
     let metrics: Eth2GossipsubMetrics | null = null;
     if (metricsRegister) {
       metrics = createEth2GossipsubMetrics(metricsRegister);
-      metrics.gossipMesh.peersByType.addCollect(() =>
-        this.onScrapeLodestarMetrics(metrics as Eth2GossipsubMetrics, networkConfig)
-      );
     }
 
     // Parse direct peers from multiaddr strings to AddrInfo objects
@@ -113,8 +137,8 @@ export class Eth2Gossipsub extends GossipSub {
 
     // Gossipsub parameters defined here:
     // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
-    super(modules.libp2p.services.components, {
-      globalSignaturePolicy: SignaturePolicy.StrictNoSign,
+    const gossipsubInstance = gossipsub({
+      globalSignaturePolicy: StrictNoSign,
       allowPublishToZeroTopicPeers: allowPublishToZeroPeers,
       D: gossipsubD ?? GOSSIP_D,
       Dlo: gossipsubDLow ?? GOSSIP_D_LOW,
@@ -155,7 +179,12 @@ export class Eth2Gossipsub extends GossipSub {
       // This should be large enough to not send IDONTWANT for "small" messages
       // See https://github.com/ChainSafe/lodestar/pull/7077#issuecomment-2383679472
       idontwantMinDataSize: 16829,
-    });
+    })(modules.libp2p.services.components) as GossipSubInternal;
+
+    if (metrics) {
+      metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics, networkConfig));
+    }
+    this.gossipsub = gossipsubInstance;
     this.scoreParams = scoreParams;
     this.config = config;
     this.logger = logger;
@@ -164,7 +193,7 @@ export class Eth2Gossipsub extends GossipSub {
     this.libp2p = modules.libp2p;
     this.gossipTopicCache = gossipTopicCache;
 
-    this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this));
+    this.gossipsub.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this));
     this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this));
 
     // Having access to this data is CRUCIAL for debugging. While this is a massive log, it must not be deleted.
@@ -174,6 +203,38 @@ export class Eth2Gossipsub extends GossipSub {
     }
   }
 
+  async start(): Promise<void> {
+    await this.gossipsub.start();
+  }
+
+  async stop(): Promise<void> {
+    await this.gossipsub.stop();
+  }
+
+  get mesh(): Map<string, Set<string>> {
+    return this.gossipsub.mesh;
+  }
+
+  getTopics(): TopicStr[] {
+    return this.gossipsub.getTopics();
+  }
+
+  getMeshPeers(topic: TopicStr): string[] {
+    return this.gossipsub.getMeshPeers(topic);
+  }
+
+  publish(topic: TopicStr, data: Uint8Array, opts?: PublishOpts): Promise<PublishResult> {
+    return this.gossipsub.publish(topic, data, opts);
+  }
+
+  dumpPeerScoreStats(): PeerScoreStatsDump {
+    return this.gossipsub.dumpPeerScoreStats();
+  }
+
+  getScore(peerIdStr: string): number {
+    return this.gossipsub.getScore(peerIdStr);
+  }
+
   /**
    * Subscribe to a `GossipTopic`
    */
@@ -183,7 +244,7 @@ export class Eth2Gossipsub extends GossipSub {
     this.gossipTopicCache.setTopic(topicStr, topic);
 
     this.logger.verbose("Subscribe to gossipsub topic", {topic: topicStr});
-    this.subscribe(topicStr);
+    this.gossipsub.subscribe(topicStr);
   }
 
   /**
@@ -192,15 +253,14 @@ export class Eth2Gossipsub extends GossipSub {
   unsubscribeTopic(topic: GossipTopic): void {
     const topicStr = stringifyGossipTopic(this.config, topic);
     this.logger.verbose("Unsubscribe to gossipsub topic", {topic: topicStr});
-    this.unsubscribe(topicStr);
+    this.gossipsub.unsubscribe(topicStr);
   }
 
   private onScrapeLodestarMetrics(metrics: Eth2GossipsubMetrics, networkConfig: NetworkConfig): void {
-    const mesh = this.mesh;
-    // biome-ignore lint/complexity/useLiteralKeys: `topics` is a private attribute
-    const topics = this["topics"] as Map<string, Set<string>>;
-    const peers = this.peers;
-    const score = this.score;
+    const mesh = this.gossipsub.mesh;
+    const topics = this.gossipsub.topics;
+    const peers = this.gossipsub.peers;
+    const score = this.gossipsub.score;
     const meshPeersByClient = new Map<string, number>();
     const meshPeerIdStrs = new Set<string>();
 
@@ -305,7 +365,7 @@ export class Eth2Gossipsub extends GossipSub {
     metrics.gossipPeer.score.set(gossipScores);
   }
 
-  private onGossipsubMessage(event: GossipsubEvents["gossipsub:message"]): void {
+  private onGossipsubMessage(event: GossipSubEvents["gossipsub:message"]): void {
     const {propagationSource, msgId, msg} = event.detail;
 
     // Also validates that the topicStr is known
@@ -341,7 +401,7 @@ export class Eth2Gossipsub extends GossipSub {
     // Without this we'll have huge event loop lag
     // See https://github.com/ChainSafe/lodestar/issues/5604
     callInNextEventLoop(() => {
-      this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance);
+      this.gossipsub.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance);
     });
   }
 
@@ -379,7 +439,7 @@ export class Eth2Gossipsub extends GossipSub {
     }
 
     // Add to direct peers set only after addresses are stored
-    this.direct.add(peerIdStr);
+    this.gossipsub.direct.add(peerIdStr);
 
     this.logger.info("Added direct peer via API", {peerId: peerIdStr});
     return peerIdStr;
@@ -389,7 +449,7 @@ export class Eth2Gossipsub extends GossipSub {
    * Remove a peer from direct peers.
    */
   removeDirectPeer(peerIdStr: string): boolean {
-    const removed = this.direct.delete(peerIdStr);
+    const removed = this.gossipsub.direct.delete(peerIdStr);
     if (removed) {
       this.logger.info("Removed direct peer via API", {peerId: peerIdStr});
     }
@@ -400,7 +460,7 @@ export class Eth2Gossipsub extends GossipSub {
    * Get list of current direct peer IDs.
    */
   getDirectPeers(): string[] {
-    return Array.from(this.direct);
+    return Array.from(this.gossipsub.direct);
   }
 }
 
@@ -498,7 +558,8 @@ export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], l
       try {
         const ma = multiaddr(peerStr);
 
-        const peerIdStr = ma.getPeerId();
+        const peerIdComponent = ma.getComponents().findLast((component) => component.name === "p2p");
+        const peerIdStr = peerIdComponent?.value;
         if (!peerIdStr) {
           logger.warn("Direct peer multiaddr must contain /p2p/ component with peer ID", {multiaddr: peerStr});
           continue;

package/src/network/gossip/interface.ts

@@ -1,6 +1,6 @@
-import {Message, TopicValidatorResult} from "@libp2p/interface";
-import {Libp2p} from "libp2p";
-import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types";
+import type {Message, TopicValidatorResult} from "@libp2p/gossipsub";
+import type {PeerIdStr} from "@libp2p/gossipsub/types";
+import type {Libp2p} from "libp2p";
 import {BeaconConfig, ForkBoundary} from "@lodestar/config";
 import {
   AttesterSlashing,

package/src/network/gossip/scoringParameters.ts

@@ -1,9 +1,9 @@
 import {
-  PeerScoreParams,
-  PeerScoreThresholds,
-  TopicScoreParams,
+  type PeerScoreParams,
+  type PeerScoreThresholds,
+  type TopicScoreParams,
   defaultTopicScoreParams,
-} from "@chainsafe/libp2p-gossipsub/score";
+} from "@libp2p/gossipsub/score";
 import {BeaconConfig} from "@lodestar/config";
 import {ATTESTATION_SUBNET_COUNT, PTC_SIZE, SLOTS_PER_EPOCH, TARGET_AGGREGATORS_PER_COMMITTEE} from "@lodestar/params";
 import {computeCommitteeCount} from "@lodestar/state-transition";

package/src/network/interface.ts

@@ -1,5 +1,5 @@
-import {Identify} from "@libp2p/identify";
-import {
+import type {Identify} from "@libp2p/identify";
+import type {
   ComponentLogger,
   ConnectionGater,
   ConnectionProtector,
@@ -16,7 +16,7 @@ import {
 } from "@libp2p/interface";
 import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal";
 import type {Datastore} from "interface-datastore";
-import {Libp2p as ILibp2p} from "libp2p";
+import type {Libp2p as ILibp2p} from "libp2p";
 import {
   AttesterSlashing,
   DataColumnSidecar,

package/src/network/libp2p/index.ts

@@ -1,6 +1,6 @@
 import {bootstrap} from "@libp2p/bootstrap";
 import {identify} from "@libp2p/identify";
-import {PrivateKey} from "@libp2p/interface";
+import type {PrivateKey} from "@libp2p/interface";
 import {mdns} from "@libp2p/mdns";
 import {mplex} from "@libp2p/mplex";
 import {prometheusMetrics} from "@libp2p/prometheus-metrics";
@@ -39,6 +39,7 @@ export async function createNodeJsLibp2p(
   nodeJsLibp2pOpts: NodeJsLibp2pOpts = {}
 ): Promise<Libp2p> {
   const localMultiaddrs = networkOpts.localMultiaddrs || defaultNetworkOptions.localMultiaddrs;
+  const disconnectThreshold = networkOpts.disconnectThreshold ?? defaultNetworkOptions.disconnectThreshold;
   const {peerStoreDir, disablePeerDiscovery} = nodeJsLibp2pOpts;
 
   let datastore: undefined | Eth2PeerDataStore = undefined;
@@ -72,8 +73,37 @@ export async function createNodeJsLibp2p(
     noiseCrypto.chaCha20Poly1305Encrypt = asCrypto.chaCha20Poly1305Encrypt;
   }
 
+  const libp2pMetrics = nodeJsLibp2pOpts.metrics
+    ? (components: LodestarComponents) => {
+        const metrics = prometheusMetrics({
+          collectDefaultMetrics: false,
+          preserveExistingMetrics: true,
+          registry: nodeJsLibp2pOpts.metricsRegistry,
+        })(components);
+
+        // Work around identify EOF race:
+        // `trackProtocolStream` attaches a `message` listener immediately after protocol
+        // negotiation. For `/ipfs/id/1.0.0`, identify() adds its own reader later and can
+        // miss the first response frame when metrics listener drains events first.
+        const originalTrackProtocolStream = metrics.trackProtocolStream.bind(metrics);
+        metrics.trackProtocolStream = ((stream) => {
+          if (stream.protocol === "/ipfs/id/1.0.0") {
+            return;
+          }
+          originalTrackProtocolStream(stream);
+        }) as typeof metrics.trackProtocolStream;
+
+        return metrics;
+      }
+    : undefined;
+
   return createLibp2p({
     privateKey,
+    nodeInfo: {
+      name: "lodestar",
+      version: networkOpts.version ?? "unknown",
+      userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
+    },
     addresses: {
       listen: localMultiaddrs,
       announce: [],
@@ -93,15 +123,9 @@ export async function createNodeJsLibp2p(
         },
       }),
     ],
-    streamMuxers: [mplex({maxInboundStreams: 256, disconnectThreshold: networkOpts.disconnectThreshold})],
+    streamMuxers: [mplex({disconnectThreshold})],
     peerDiscovery,
-    metrics: nodeJsLibp2pOpts.metrics
-      ? prometheusMetrics({
-          collectDefaultMetrics: false,
-          preserveExistingMetrics: true,
-          registry: nodeJsLibp2pOpts.metricsRegistry,
-        })
-      : undefined,
+    metrics: libp2pMetrics,
     connectionManager: {
       // dialer config
       maxParallelDials: 100,
@@ -124,7 +148,6 @@ export async function createNodeJsLibp2p(
     datastore,
     services: {
       identify: identify({
-        agentVersion: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
         runOnConnectionOpen: false,
       }),
       // individual components are specified because the components object is a Proxy

package/src/network/network.ts

@@ -1,7 +1,7 @@
-import {PeerId, PrivateKey} from "@libp2p/interface";
+import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score";
+import type {PublishOpts} from "@libp2p/gossipsub/types";
+import type {PeerId, PrivateKey} from "@libp2p/interface";
 import {peerIdFromPrivateKey} from "@libp2p/peer-id";
-import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score";
-import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
 import {LoggerNode} from "@lodestar/logger/node";

package/src/network/options.ts

@@ -72,4 +72,7 @@ export const defaultNetworkOptions: NetworkOptions = {
   //   - for fusaka-devnets, we have 25-30 peers per subnet
   //   - for public testnets or mainnet, average number of peers per group is SAMPLES_PER_SLOT * targetPeers / NUMBER_OF_CUSTODY_GROUPS = 6.25 so this should not be an issue
   targetGroupPeers: 6,
+  // Keep this high enough for normal req/resp bursts on stable connections.
+  // libp2p-mplex default (5) is too low and can cause frequent connection resets.
+  disconnectThreshold: 50,
 };

package/src/network/peers/datastore.ts

@@ -8,6 +8,9 @@ type MemoryItem = {
   data: Uint8Array;
 };
 
+// biome-ignore lint/suspicious/noExplicitAny: used below (copied from upstream)
+type AwaitGenerator<T, TReturn = any, TNext = any> = Generator<T, TReturn, TNext> | AsyncGenerator<T, TReturn, TNext>;
+
 /**
  * Before libp2p 0.35, peerstore stays in memory and periodically write to db after n dirty items
  * This has a memory issue because all peer data stays in memory and loaded at startup time
@@ -93,7 +96,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
    * This throws error if not found
    * see https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L102
    */
-  async get(key: Key): Promise<Uint8Array> {
+  async get(key: Key, options?: AbortOptions): Promise<Uint8Array> {
     const keyStr = key.toString();
     const memoryItem = this._memoryDatastore.get(keyStr);
     if (memoryItem) {
@@ -102,16 +105,16 @@ export class Eth2PeerDataStore extends BaseDatastore {
     }
 
     // this throws error if not found
-    const dbValue = await this._dbDatastore.get(key);
+    const dbValue = await this._dbDatastore.get(key, options);
     // don't call this._memoryDatastore.set directly
     // we want to get through prune() logic with fromDb as true
     await this._put(key, dbValue, true);
     return dbValue;
   }
 
-  async has(key: Key): Promise<boolean> {
+  async has(key: Key, options?: AbortOptions): Promise<boolean> {
     try {
-      await this.get(key);
+      await this.get(key, options);
     } catch (err) {
       // this is the same to how js-datastore-level handles notFound error
       // https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L121
@@ -121,26 +124,26 @@ export class Eth2PeerDataStore extends BaseDatastore {
     return true;
   }
 
-  async delete(key: Key): Promise<void> {
+  async delete(key: Key, options?: AbortOptions): Promise<void> {
     this._memoryDatastore.delete(key.toString());
-    await this._dbDatastore.delete(key);
+    await this._dbDatastore.delete(key, options);
   }
 
-  async *_all(q: Query): AsyncIterable<Pair> {
+  async *_all(q: Query, options?: AbortOptions): AwaitGenerator<Pair> {
     for (const [key, value] of this._memoryDatastore.entries()) {
       yield {
         key: new Key(key),
         value: value.data,
       };
     }
-    yield* this._dbDatastore.query(q);
+    yield* this._dbDatastore.query(q, options);
   }
 
-  async *_allKeys(q: KeyQuery): AsyncIterable<Key> {
+  async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator<Key> {
     for (const key of this._memoryDatastore.keys()) {
       yield new Key(key);
     }
-    yield* this._dbDatastore.queryKeys(q);
+    yield* this._dbDatastore.queryKeys(q, options);
   }
 
   private async _addDirtyItem(keyStr: string): Promise<void> {