@lodestar/beacon-node 1.42.0-dev.1d50253953 → 1.42.0-dev.4118b5b440

package/src/network/processor/index.ts

@@ -2,7 +2,7 @@ import {routes} from "@lodestar/api";
 import {ForkSeq} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {RootHex, Slot, SlotRootHex} from "@lodestar/types";
-import {Logger, MapDef, mapValues, pruneSetToMax, sleep} from "@lodestar/utils";
+import {Logger, MapDef, mapValues, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../../chain/blocks/blockInput/types.js";
 import {ChainEvent} from "../../chain/emitter.js";
 import {GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -87,27 +87,27 @@ const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj) as (keyof
 // TODO: Arbitrary constant, check metrics
 const MAX_JOBS_SUBMITTED_PER_TICK = 128;
 
-// How many attestations (aggregate + unaggregate) we keep before new ones get dropped.
+// How many gossip messages we keep before new ones get dropped.
 const MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS = 16_384;
 
-// We don't want to process too many attestations in a single tick
-// As seen on mainnet, attestation concurrency metric ranges from 1000 to 2000
+// We don't want to process too many gossip messages in a single tick
+// As seen on mainnet, gossip messages concurrency metric ranges from 1000 to 2000
 // so make this constant a little bit conservative
-const MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK = 1024;
+const MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK = 1024;
 
 // Same motivation to JobItemQueue, we don't want to block the event loop
-const PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
+const AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS = 50;
 
 /**
  * Reprocess reject reason for metrics
  */
 export enum ReprocessRejectReason {
   /**
-   * There are too many attestations that have unknown block root.
+   * There are too many gossip messages that have unknown block root.
    */
   reached_limit = "reached_limit",
   /**
-   * The awaiting attestation is pruned per clock slot.
+   * The awaiting gossip message is pruned per clock slot.
    */
   expired = "expired",
 }
@@ -137,7 +137,7 @@ export enum CannotAcceptWorkReason {
  *
  * ### PendingGossipsubMessage beacon_attestation example
  *
- * For attestations, processing the message includes the steps:
+ * For gossip messages, processing the message includes the steps:
  * 1. Pre shuffling sync validation
  * 2. Retrieve shuffling: async + goes into the regen queue and can be expensive
  * 3. Pre sig validation sync validation
@@ -156,11 +156,10 @@ export class NetworkProcessor {
   private readonly gossipQueues: ReturnType<typeof createGossipQueues>;
   private readonly gossipTopicConcurrency: {[K in GossipType]: number};
   private readonly extractBlockSlotRootFns = createExtractBlockSlotRootFns();
-  // we may not receive the block for Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
+  // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
-  private readonly awaitingGossipsubMessagesByRootBySlot: MapDef<Slot, MapDef<RootHex, Set<PendingGossipsubMessage>>>;
-  private unknownBlockGossipsubMessagesCount = 0;
-  private unknownRootsBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
   constructor(
     modules: NetworkProcessorModules,
@@ -184,9 +183,7 @@ export class NetworkProcessor {
     this.chain.emitter.on(routes.events.EventType.block, this.onBlockProcessed.bind(this));
     this.chain.clock.on(ClockEvent.slot, this.onClockSlot.bind(this));
 
-    this.awaitingGossipsubMessagesByRootBySlot = new MapDef(
-      () => new MapDef<RootHex, Set<PendingGossipsubMessage>>(() => new Set())
-    );
+    this.awaitingMessagesByBlockRoot = new MapDef<RootHex, Set<PendingGossipsubMessage>>(() => new Set());
 
     // TODO: Implement queues and priorization for ReqResp incoming requests
     // Listens to NetworkEvent.reqRespIncomingRequest event
@@ -198,7 +195,7 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.reprocessGossipAttestations.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -233,64 +230,77 @@ export class NetworkProcessor {
     return queue.getAll();
   }
 
-  searchUnknownSlotRoot({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (this.chain.seenBlock(root) || this.unknownRootsBySlot.getOrDefault(slot).has(root)) {
+  /**
+   * Search block via `ChainEvent.unknownBlockRoot` event
+   * Note that slot is not necessarily the same to the block's slot but it can be used for a good prune strategy.
+   * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   */
+  searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+    if (
+      this.chain.seenBlock(root) ||
+      this.awaitingMessagesByBlockRoot.has(root) ||
+      this.unknownBlocksBySlot.getOrDefault(slot).has(root)
+    ) {
       return;
     }
     // Search for the unknown block
-    this.unknownRootsBySlot.getOrDefault(slot).add(root);
+    this.unknownBlocksBySlot.getOrDefault(slot).add(root);
     this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
   }
 
   private onPendingGossipsubMessage(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
     const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
-    // check block root of Attestation and SignedAggregateAndProof messages
-    if (extractBlockSlotRootFn) {
-      const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
-      // if slotRoot is null, it means the msg.data is invalid
-      // in that case message will be rejected when deserializing data in later phase (gossipValidatorFn)
-      if (slotRoot) {
-        // DOS protection: avoid processing messages that are too old
-        const {slot, root} = slotRoot;
-        const clockSlot = this.chain.clock.currentSlot;
-        const {fork} = message.topic.boundary;
-        let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
-        if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
-          // post deneb, the attestations could be in current or previous epoch
-          earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
-        }
-        if (slot < earliestPermissableSlot) {
-          // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-          this.metrics?.networkProcessor.gossipValidationError.inc({
-            topic: topicType,
-            error: GossipErrorCode.PAST_SLOT,
-          });
-          return;
-        }
-        message.msgSlot = slot;
-        // check if we processed a block with this root
-        // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
-        if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
-          this.searchUnknownSlotRoot({slot, root}, BlockInputSource.gossip, message.propagationSource.toString());
-
-          if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
-            // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
-            this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.reached_limit});
-            return;
-          }
+    const slotRoot = extractBlockSlotRootFn
+      ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
+      : null;
+    if (slotRoot === null) {
+      // some messages don't have slot and root
+      // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
+      this.pushPendingGossipsubMessageToQueue(message);
+      return;
+    }
 
-          this.metrics?.reprocessGossipAttestations.total.inc();
-          const awaitingGossipsubMessagesByRoot = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-          const awaitingGossipsubMessages = awaitingGossipsubMessagesByRoot.getOrDefault(root);
-          awaitingGossipsubMessages.add(message);
-          this.unknownBlockGossipsubMessagesCount++;
-          return;
-        }
+    // common check for all topics
+    // DOS protection: avoid processing messages that are too old
+    const {slot, root} = slotRoot;
+    const clockSlot = this.chain.clock.currentSlot;
+    const {fork} = message.topic.boundary;
+    let earliestPermissableSlot = clockSlot - DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE;
+    if (ForkSeq[fork] >= ForkSeq.deneb && topicType === GossipType.beacon_attestation) {
+      // post deneb, the attestations could be in current or previous epoch
+      earliestPermissableSlot = computeStartSlotAtEpoch(this.chain.clock.currentEpoch - 1);
+    }
+    if (slot < earliestPermissableSlot) {
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+      this.metrics?.networkProcessor.gossipValidationError.inc({
+        topic: topicType,
+        error: GossipErrorCode.PAST_SLOT,
+      });
+      return;
+    }
+
+    message.msgSlot = slot;
+
+    // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
+    if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
+      this.searchUnknownBlock({slot, root}, BlockInputSource.network_processor, message.propagationSource.toString());
+
+      if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+        // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+        this.metrics?.awaitingBlockGossipMessages.reject.inc({
+          reason: ReprocessRejectReason.reached_limit,
+          topic: topicType,
+        });
+        return;
       }
+
+      this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
+      const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(root);
+      awaitingGossipsubMessages.add(message);
+      return;
     }
 
-    // bypass the check for other messages
     this.pushPendingGossipsubMessageToQueue(message);
   }
 
@@ -298,7 +308,7 @@ export class NetworkProcessor {
     const topicType = message.topic.type;
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
-      // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
       this.metrics?.gossipValidationQueue.droppedJobs.inc({topic: message.topic.type}, droppedCount);
     }
 
@@ -306,58 +316,61 @@ export class NetworkProcessor {
     this.executeWork();
   }
 
-  private async onBlockProcessed({
-    slot,
-    block: rootHex,
-  }: {
-    slot: Slot;
-    block: string;
-    executionOptimistic: boolean;
-  }): Promise<void> {
-    const byRootGossipsubMessages = this.awaitingGossipsubMessagesByRootBySlot.getOrDefault(slot);
-    const waitingGossipsubMessages = byRootGossipsubMessages.getOrDefault(rootHex);
-    if (waitingGossipsubMessages.size === 0) {
+  private async onBlockProcessed({block: rootHex}: {block: string; executionOptimistic: boolean}): Promise<void> {
+    const waitingGossipsubMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+    if (!waitingGossipsubMessages || waitingGossipsubMessages.size === 0) {
       return;
     }
 
-    this.metrics?.reprocessGossipAttestations.resolve.inc(waitingGossipsubMessages.size);
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
     // cache, it may not be necessary at this time
     for (const message of waitingGossipsubMessages) {
-      this.metrics?.reprocessGossipAttestations.waitSecBeforeResolve.set(nowSec - message.seenTimestampSec);
+      const topicType = message.topic.type;
+      this.metrics?.awaitingBlockGossipMessages.waitSecBeforeResolve.set(
+        {topic: topicType},
+        nowSec - message.seenTimestampSec
+      );
+      this.metrics?.awaitingBlockGossipMessages.resolve.inc({topic: topicType});
       this.pushPendingGossipsubMessageToQueue(message);
       count++;
       // don't want to block the event loop, worse case it'd wait for 16_084 / 1024 * 50ms = 800ms which is not a big deal
-      if (count === MAX_UNKNOWN_BLOCK_GOSSIP_OBJECTS_PER_TICK) {
+      if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
         count = 0;
-        await sleep(PROCESS_UNKNOWN_BLOCK_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+        await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
 
-    byRootGossipsubMessages.delete(rootHex);
+    this.awaitingMessagesByBlockRoot.delete(rootHex);
   }
 
   private onClockSlot(clockSlot: Slot): void {
     const nowSec = Date.now() / 1000;
-    for (const [slot, gossipMessagesByRoot] of this.awaitingGossipsubMessagesByRootBySlot.entries()) {
-      if (slot < clockSlot) {
-        for (const gossipMessages of gossipMessagesByRoot.values()) {
+    const minSlot = clockSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE;
+
+    for (const [slot, roots] of this.unknownBlocksBySlot) {
+      if (slot > minSlot) continue;
+      for (const rootHex of roots) {
+        const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
+        if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
-            this.metrics?.reprocessGossipAttestations.reject.inc({reason: ReprocessRejectReason.expired});
-            this.metrics?.reprocessGossipAttestations.waitSecBeforeReject.set(
-              {reason: ReprocessRejectReason.expired},
+            const topicType = message.topic.type;
+            this.metrics?.awaitingBlockGossipMessages.reject.inc({
+              topic: topicType,
+              reason: ReprocessRejectReason.expired,
+            });
+            this.metrics?.awaitingBlockGossipMessages.waitSecBeforeReject.set(
+              {topic: topicType, reason: ReprocessRejectReason.expired},
               nowSec - message.seenTimestampSec
             );
-            // TODO: Should report the dropped job to gossip? It will be eventually pruned from the mcache
+            // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
+          this.awaitingMessagesByBlockRoot.delete(rootHex);
         }
-        this.awaitingGossipsubMessagesByRootBySlot.delete(slot);
       }
+      this.unknownBlocksBySlot.delete(slot);
     }
-    pruneSetToMax(this.unknownRootsBySlot, MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE);
-    this.unknownBlockGossipsubMessagesCount = 0;
   }
 
   private executeWork(): void {
@@ -496,4 +509,12 @@ export class NetworkProcessor {
 
     return null;
   }
+
+  private get unknownBlockGossipsubMessagesCount(): number {
+    let count = 0;
+    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
+      count += messages.size;
+    }
+    return count;
+  }
 }

package/src/sync/unknownBlock.ts

@@ -64,7 +64,7 @@ enum FetchResult {
  *
  * - publishBlock
  * - gossipHandlers
- * - searchUnknownSlotRoot
+ * - searchUnknownBlock
  *    = produceSyncCommitteeContribution
  *    = validateGossipFnRetryUnknownRoot
  *        * submitPoolAttestationsV2
@@ -481,7 +481,7 @@ export class BlockInputSync {
    * From a set of shuffled peers:
    *   - fetch the block
    *   - from deneb, fetch all missing blobs
-   *   - from peerDAS, fetch sampled colmns
+   *   - from peerDAS, fetch sampled columns
    * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name
    * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
    * Also verifies the received block root + returns the peer that provided the block for future downscoring.
@@ -489,10 +489,7 @@ export class BlockInputSync {
   private async fetchBlockInput(cacheItem: BlockInputSyncCacheItem): Promise<PendingBlockInput> {
     const rootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
     const excludedPeers = new Set<PeerIdStr>();
-    const defaultPendingColumns =
-      this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu
-        ? new Set(this.network.custodyConfig.sampledColumns)
-        : null;
+    const defaultPendingColumns = new Set(this.network.custodyConfig.sampledColumns);
 
     const fetchStartSec = Date.now() / 1000;
     let slot = isPendingBlockInput(cacheItem) ? cacheItem.blockInput.slot : undefined;
@@ -506,14 +503,10 @@ export class BlockInputSync {
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
           ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
           : defaultPendingColumns;
-      // pendingDataColumns is null pre-fulu
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
         // no more peer with needed columns to try, throw error
-        let message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer`;
-        if (pendingColumns) {
-          message += ` with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-        }
+        const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
         this.metrics?.blockInputSync.fetchTimeSec.observe(
           {result: FetchResult.FailureTriedAllPeers},
           Date.now() / 1000 - fetchStartSec
@@ -650,7 +643,7 @@ export class BlockInputSync {
       // TODO(fulu): why is this commented out here?
       //
       //   this.knownBadBlocks.add(block.blockRootHex);
-      //   for (const peerIdStr of block.peerIdStrs) {
+      //   for (const peerIdStr of block.peerIdStrings) {
       //     // TODO: Refactor peerRpcScores to work with peerIdStr only
       //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
       //   }
@@ -729,11 +722,11 @@ export class UnknownBlockPeerBalancer {
   }
 
   /**
-   * called from fetchUnknownBlockRoot() where we only have block root and nothing else
+   * called from fetchBlockInput() where we only have block root and nothing else
    * excludedPeers are the peers that we requested already so we don't want to try again
    * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
    */
-  bestPeerForPendingColumns(pendingColumns: Set<number> | null, excludedPeers: Set<PeerIdStr>): PeerSyncMeta | null {
+  bestPeerForPendingColumns(pendingColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerSyncMeta | null {
     const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
     if (eligiblePeers.length === 0) {
       return null;
@@ -750,37 +743,6 @@ export class UnknownBlockPeerBalancer {
     return this.peersMeta.get(bestPeerId) ?? null;
   }
 
-  /**
-   * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
-   * excludedPeers are the peers that we requested already so we don't want to try again
-   */
-  bestPeerForBlockInput(blockInput: IBlockInput, excludedPeers: Set<PeerIdStr>): PeerSyncMeta | null {
-    const eligiblePeers: PeerIdStr[] = [];
-
-    if (isBlockInputColumns(blockInput)) {
-      const pendingDataColumns: Set<number> = new Set(blockInput.getMissingSampledColumnMeta().missing);
-      // there could be no pending column in case when block is still missing
-      eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
-    } else {
-      // prefulu
-      eligiblePeers.push(...this.filterPeers(null, excludedPeers));
-    }
-
-    if (eligiblePeers.length === 0) {
-      return null;
-    }
-
-    const sortedEligiblePeers = sortBy(
-      shuffle(eligiblePeers),
-      // prefer peers with least active req
-      (peerId) => this.activeRequests.get(peerId) ?? 0
-    );
-
-    const bestPeerId = sortedEligiblePeers[0];
-    this.onRequest(bestPeerId);
-    return this.peersMeta.get(bestPeerId) ?? null;
-  }
-
   /**
    * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods
    * make this public for testing
@@ -804,8 +766,7 @@ export class UnknownBlockPeerBalancer {
     return totalActiveRequests;
   }
 
-  // pendingDataColumns could be null for prefulu
-  private filterPeers(pendingDataColumns: Set<number> | null, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
+  private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -820,13 +781,12 @@ export class UnknownBlockPeerBalancer {
         continue;
       }
 
-      if (pendingDataColumns === null || pendingDataColumns.size === 0) {
-        // prefulu, no pending columns
+      if (pendingDataColumns.size === 0) {
         considerPeers.push({peerId, columnCount: 0});
         continue;
       }
 
-      // postfulu, find peers that have custody columns that we need
+      // find peers that have custody columns that we need
       const {custodyColumns: peerColumns} = syncMeta;
       // check if the peer has all needed columns
       // get match