@lodestar/beacon-node 1.51.0-dev.c99cf49add → 1.51.0-dev.cda05d18f1

package/src/sync/sync.ts

@@ -1,13 +1,17 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Slot, Status} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
-import {GENESIS_SLOT} from "../constants/constants.js";
+import {GENESIS_EPOCH, GENESIS_SLOT} from "../constants/constants.js";
+import {GoodByeReasonCode} from "../constants/index.js";
+import {IBeaconDb} from "../db/index.js";
 import {ExecutionEngineState} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
+import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js";
 import {ClockEvent} from "../util/clock.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
+import {PeerIdStr} from "../util/peerId.js";
 import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
 import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
 import {SyncOptions} from "./options.js";
@@ -19,6 +23,7 @@ export class BeaconSync implements IBeaconSync {
   private readonly logger: Logger;
   private readonly network: INetwork;
   private readonly chain: IBeaconChain;
+  private readonly db: IBeaconDb;
   private readonly metrics: Metrics | null;
   private readonly opts: SyncOptions;
 
@@ -28,12 +33,15 @@ export class BeaconSync implements IBeaconSync {
   /** For metrics only */
   private readonly peerSyncType = new Map<string, PeerSyncType>();
   private readonly slotImportTolerance: Slot;
+  /** The requested subscription state, set while a change is being applied */
+  private pendingGossipSubscribed: boolean | undefined;
 
   constructor(opts: SyncOptions, modules: SyncModules) {
-    const {config, chain, metrics, network, logger} = modules;
+    const {config, chain, db, metrics, network, logger} = modules;
     this.opts = opts;
     this.network = network;
     this.chain = chain;
+    this.db = db;
     this.metrics = metrics;
     this.logger = logger;
     this.rangeSync = new RangeSync(modules, opts);
@@ -83,6 +91,7 @@ export class BeaconSync implements IBeaconSync {
   }
 
   close(): void {
+    this.pendingGossipSubscribed = undefined;
     this.network.events.off(NetworkEvent.peerConnected, this.addPeer);
     this.network.events.off(NetworkEvent.peerDisconnected, this.removePeer);
     this.chain.clock.off(ClockEvent.epoch, this.onClockEpoch);
@@ -192,6 +201,11 @@ export class BeaconSync implements IBeaconSync {
    */
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
+    // Genesis and equal finalized epochs are already checked by assertPeerRelevance in the network worker
+    if (data.status.finalizedEpoch > GENESIS_EPOCH && data.status.finalizedEpoch < localStatus.finalizedEpoch) {
+      void this.verifyPastFinalizedCheckpoint(data.peer, data.status);
+    }
+
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
     this.logger.verbose("Peer sync type classified", {
       peer: data.peer,
@@ -216,6 +230,56 @@ export class BeaconSync implements IBeaconSync {
     this.updateSyncState();
   };
 
+  /**
+   * A behind peer's finalized checkpoint must be the canonical block in effect at the start slot of its epoch
+   */
+  async verifyPastFinalizedCheckpoint(peer: PeerIdStr, remote: Status): Promise<void> {
+    const startSlot = computeStartSlotAtEpoch(remote.finalizedEpoch);
+    const root = toRootHex(remote.finalizedRoot);
+    try {
+      let reason: string | null = null;
+      // After a finality jump, finalized blocks stay in fork choice until the archiver migrates them.
+      // Fork choice is pruned only after archiving, so a miss here means the db already has this range.
+      const canonicalBlock = this.chain.forkChoice.getCanonicalBlockClosestLteSlot(startSlot);
+      if (canonicalBlock !== null) {
+        if (canonicalBlock.blockRoot === root) return;
+        reason = "finalized_root_not_canonical";
+      } else {
+        const slot = await this.db.blockArchive.getSlotByRoot(remote.finalizedRoot);
+        if (slot !== null && slot > startSlot) {
+          reason = "finalized_root_after_epoch_start";
+        } else if (slot !== startSlot) {
+          const childSlot = await this.db.blockArchive.getSlotByParentRoot(remote.finalizedRoot);
+          if (childSlot !== null) {
+            if (childSlot <= startSlot) reason = "finalized_root_not_in_effect_at_epoch_start";
+          } else if (slot === null && startSlot > this.chain.earliestAvailableSlot) {
+            // childSlot == null too, neither the root nor a child is canonical
+            // or they are canonical and both pruned but peer's finalized epoch starts after earliestAvailableSlot
+            reason = "unknown_finalized_root";
+          }
+        }
+      }
+
+      if (reason !== null) {
+        this.logger.debug("Peer finalized checkpoint not in our chain", {
+          peer,
+          finalizedEpoch: remote.finalizedEpoch,
+          root,
+          reason,
+        });
+        this.metrics?.syncPeerFinalizedCheckpointMismatch.inc({reason});
+        this.network.reportPeer(peer, PeerAction.Fatal, "PeerFinalizedCheckpointMismatch");
+        await this.network.goodbyeAndDisconnect(peer, GoodByeReasonCode.IRRELEVANT_NETWORK);
+      }
+    } catch (e) {
+      this.logger.debug(
+        "Error verifying peer finalized checkpoint",
+        {peer, finalizedEpoch: remote.finalizedEpoch, root},
+        e as Error
+      );
+    }
+  }
+
   /**
    * Must be called by libp2p when a peer is removed from the peer manager
    */
@@ -235,17 +299,7 @@ export class BeaconSync implements IBeaconSync {
     if (state === SyncState.Synced && this.chain.clock.currentEpoch >= MIN_EPOCH_TO_START_GOSSIP) {
       this.chain.forkChoice.resumeFastConfirmation();
 
-      if (!this.network.isSubscribedToGossipCoreTopics()) {
-        this.network
-          .subscribeGossipCoreTopics()
-          .then(() => {
-            this.metrics?.syncSwitchGossipSubscriptions.inc({action: "subscribed"});
-            this.logger.info("Subscribed gossip core topics");
-          })
-          .catch((e) => {
-            this.logger.error("Error subscribing to gossip core topics", {}, e);
-          });
-      }
+      this.setGossipSubscription(true);
 
       // also start searching for unknown blocks
       if (!this.unknownBlockSync.isSubscribedToNetwork()) {
@@ -261,17 +315,8 @@ export class BeaconSync implements IBeaconSync {
         // Same debounce as gossip: transient blips keep the rule running, only a real gap pauses it
         this.chain.forkChoice.pauseFastConfirmation();
 
-        if (this.network.isSubscribedToGossipCoreTopics()) {
-          this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`);
-          this.network
-            .unsubscribeGossipCoreTopics()
-            .then(() => {
-              this.metrics?.syncSwitchGossipSubscriptions.inc({action: "unsubscribed"});
-              this.logger.info("Un-subscribed gossip core topics");
-            })
-            .catch((e) => {
-              this.logger.error("Error unsubscribing to gossip core topics", {}, e);
-            });
+        if (this.setGossipSubscription(false)) {
+          this.logger.warn("Node sync has fallen behind", {slots: syncDiff});
         }
 
         // also stop searching for unknown blocks
@@ -283,6 +328,43 @@ export class BeaconSync implements IBeaconSync {
     }
   };
 
+  private setGossipSubscription(subscribe: boolean): boolean {
+    if ((this.pendingGossipSubscribed ?? this.network.isSubscribedToGossipCoreTopics()) === subscribe) return false;
+    const applying = this.pendingGossipSubscribed !== undefined;
+    this.pendingGossipSubscribed = subscribe;
+    if (!applying) void this.applyGossipSubscription();
+    return true;
+  }
+
+  /**
+   * Applies the latest requested state one request at a time. Network skips a subscribe issued while an unsubscribe is
+   * in flight, so opposite requests must not overlap.
+   */
+  private async applyGossipSubscription(): Promise<void> {
+    for (;;) {
+      const subscribe = this.pendingGossipSubscribed;
+      if (subscribe === undefined || subscribe === this.network.isSubscribedToGossipCoreTopics()) break;
+      try {
+        await (subscribe ? this.network.subscribeGossipCoreTopics() : this.network.unsubscribeGossipCoreTopics());
+      } catch (e) {
+        this.logger.error(
+          subscribe ? "Error subscribing to gossip core topics" : "Error unsubscribing to gossip core topics",
+          {},
+          e as Error
+        );
+        // Leave a retry of the same request to the next sync state update
+        if (this.pendingGossipSubscribed === subscribe) break;
+        continue;
+      }
+      if (this.pendingGossipSubscribed === undefined || this.network.isSubscribedToGossipCoreTopics() !== subscribe) {
+        break;
+      }
+      this.metrics?.syncSwitchGossipSubscriptions.inc({action: subscribe ? "subscribed" : "unsubscribed"});
+      this.logger.info(subscribe ? "Subscribed gossip core topics" : "Un-subscribed gossip core topics");
+    }
+    this.pendingGossipSubscribed = undefined;
+  }
+
   private onClockEpoch = (): void => {
     // If a node witness the genesis event consider starting gossip
     // Also, ensure that updateSyncState is run at least once per epoch.

package/src/sync/unknownBlock.ts

@@ -6,7 +6,7 @@ import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
 import {Logger, fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
-import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
+import {isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadError, PayloadErrorCode} from "../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
@@ -300,10 +300,19 @@ export class BlockInputSync {
 
   private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
     try {
-      this.addByPayloadInput(data.payloadInput, data.peer);
-      this.triggerUnknownBlockSearch();
-      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
-      this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      const isNewRoot = this.addByPayloadInput(data.payloadInput, data.peer);
+      const pendingPayload = this.pendingPayloads.get(data.payloadInput.blockRootHex);
+      if (
+        pendingPayload &&
+        pendingPayload.status !== PendingPayloadInputStatus.fetching &&
+        pendingPayload.status !== PendingPayloadInputStatus.processing
+      ) {
+        this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
+        this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.INCOMPLETE_PAYLOAD_ENVELOPE});
+        this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      }
     } catch (e) {
       this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e as Error);
     }
@@ -599,43 +608,44 @@ export class BlockInputSync {
     return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
   }
 
-  private addByPayloadInput = (
-    payloadInput: PayloadEnvelopeInput,
-    peerIdStr?: PeerIdStr,
-    envelope?: gloas.SignedExecutionPayloadEnvelope
-  ): void => {
+  private addByPayloadInput = (payloadInput: PayloadEnvelopeInput, peerIdStr?: PeerIdStr): boolean => {
     if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
       this.logger.debug("Ignoring payload input for known bad block root", {
         slot: payloadInput.slot,
         root: payloadInput.blockRootHex,
         peerIdStr,
       });
-      return;
+      return false;
     }
 
-    const pendingPayload = this.toPendingPayloadInput(
-      payloadInput,
-      this.pendingPayloads.get(payloadInput.blockRootHex),
-      envelope
-    );
+    let pendingPayload = this.pendingPayloads.get(payloadInput.blockRootHex);
+    const isNewRoot = pendingPayload === undefined;
+    // Envelope-only entries must go through reconciliation to validate their speculative envelope.
+    if (
+      !pendingPayload ||
+      pendingPayload.status === PendingPayloadInputStatus.pending ||
+      pendingPayload.status === PendingPayloadInputStatus.downloaded
+    ) {
+      pendingPayload = this.toPendingPayloadInput(payloadInput, pendingPayload);
+      this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
+
+      this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
+        slot: payloadInput.slot,
+        root: payloadInput.blockRootHex,
+        delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
+      });
+    }
 
     if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
       pendingPayload.peerIdStrings.add(peerIdStr);
     }
 
-    this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
-
-    this.logger.verbose("Added payloadInput to BlockInputSync.pendingPayloads", {
-      slot: payloadInput.slot,
-      root: payloadInput.blockRootHex,
-      delaySec: this.chain.clock.secFromSlot(payloadInput.slot),
-    });
-
     const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
     if (prunedItemCount > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.capacity}, prunedItemCount);
       this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
     }
+    return isNewRoot;
   };
 
   private onPeerConnected = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
@@ -1267,6 +1277,9 @@ export class BlockInputSync {
     }
 
     const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
+    if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+      return;
+    }
     if (!payloadInput) {
       this.logger.debug("PayloadEnvelopeInput not yet reloadable for imported block, will retry", {
         root: rootHex,
@@ -1278,6 +1291,9 @@ export class BlockInputSync {
       const validationResult = await wrapError(
         validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
       );
+      if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+        return;
+      }
       if (validationResult.err) {
         this.logger.debug(
           "Pending payload envelope failed validation after block import, refetching by root",
@@ -1345,14 +1361,13 @@ export class BlockInputSync {
     payload.status = PendingPayloadInputStatus.fetching;
 
     const res = await wrapError(this.fetchPayloadInput(payload));
+    if (this.pendingPayloads.get(rootHex) !== payload) {
+      this.logger.verbose("Dropping downloaded payload, entry changed during fetch", logCtx);
+      return;
+    }
     if (!res.err) {
       const pendingPayload = res.result;
       const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
-      // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
-      if (!this.pendingPayloads.has(resultRootHex)) {
-        this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
-        return;
-      }
       this.pendingPayloads.set(resultRootHex, pendingPayload);
       const result = isPendingPayloadEnvelope(pendingPayload)
         ? DownloadResult.WaitingForBlock
@@ -1423,12 +1438,16 @@ export class BlockInputSync {
     this.logger.debug("Processing downloaded payload", logCtx);
 
     const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
+    // The executionPayload event emitted during import usually removes the entry before this point
     if (!res.err) {
       this.logger.debug("Processed payload from unknown sync", logCtx);
       this.pendingPayloads.delete(rootHex);
       this.triggerUnknownBlockSearch();
       return;
     }
+    if (this.pendingPayloads.get(rootHex) !== pendingPayload) {
+      return;
+    }
 
     if (res.err instanceof PayloadError) {
       switch (res.err.type.code) {
@@ -1891,13 +1910,6 @@ export class BlockInputSync {
       throw new Error(`${message} Block was not found.`);
     }
 
-    if (isBlockInputBlobs(cacheItem.blockInput)) {
-      const missing = cacheItem.blockInput.getMissingBlobMeta().map((b) => b.index);
-      if (missing.length) {
-        throw new Error(`${message} Missing blob indices=${prettyPrintIndices(missing)}.`);
-      }
-    }
-
     if (isBlockInputColumns(cacheItem.blockInput)) {
       const missing = cacheItem.blockInput.getMissingSampledColumnMeta().missing;
       if (missing.length) {