@lodestar/beacon-node 1.51.0-dev.5e25b73306 → 1.51.0-dev.6484138c50

package/src/network/peers/utils/assertPeerRelevance.ts

@@ -1,5 +1,6 @@
-import {ForkName, isForkPostFulu} from "@lodestar/params";
-import {ForkDigest, Root, Slot, Status, fulu, ssz} from "@lodestar/types";
+import {ForkName, GENESIS_EPOCH, isForkPostFulu} from "@lodestar/params";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {Epoch, ForkDigest, Root, Slot, Status, fulu, ssz} from "@lodestar/types";
 import {toHex, toRootHex} from "@lodestar/utils";
 
 // TODO: Why this value? (From Lighthouse)
@@ -9,6 +10,8 @@ export enum IrrelevantPeerCode {
   INCOMPATIBLE_FORKS = "IRRELEVANT_PEER_INCOMPATIBLE_FORKS",
   DIFFERENT_CLOCKS = "IRRELEVANT_PEER_DIFFERENT_CLOCKS",
   DIFFERENT_FINALIZED = "IRRELEVANT_PEER_DIFFERENT_FINALIZED",
+  FINALIZED_EPOCH_TOO_HIGH = "IRRELEVANT_PEER_FINALIZED_EPOCH_TOO_HIGH",
+  INVALID_FINALIZED_ROOT = "IRRELEVANT_PEER_INVALID_FINALIZED_ROOT",
   NO_EARLIEST_AVAILABLE_SLOT = "NO_EARLIEST_AVAILABLE_SLOT",
 }
 
@@ -16,6 +19,8 @@ type IrrelevantPeerType =
   | {code: IrrelevantPeerCode.INCOMPATIBLE_FORKS; ours: ForkDigest; theirs: ForkDigest}
   | {code: IrrelevantPeerCode.DIFFERENT_CLOCKS; slotDiff: number}
   | {code: IrrelevantPeerCode.NO_EARLIEST_AVAILABLE_SLOT}
+  | {code: IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH; finalizedEpoch: Epoch; maxFinalizedEpoch: Epoch}
+  | {code: IrrelevantPeerCode.INVALID_FINALIZED_ROOT}
   | {code: IrrelevantPeerCode.DIFFERENT_FINALIZED; expectedRoot: Root; remoteRoot: Root};
 
 /**
@@ -45,29 +50,35 @@ export function assertPeerRelevance(
     return {code: IrrelevantPeerCode.DIFFERENT_CLOCKS, slotDiff};
   }
 
-  // The remote's finalized epoch is less than or equal to ours, but the block root is
-  // different to the one in our chain. Therefore, the node is on a different chain and we
-  // should not communicate with them.
+  // A checkpoint can be finalized at the earliest at the start of epoch + 2
+  const maxFinalizedEpoch = Math.max(
+    GENESIS_EPOCH,
+    computeEpochAtSlot(Math.max(currentSlot, 0) + FUTURE_SLOT_TOLERANCE) - 2
+  );
+  if (remote.finalizedEpoch > maxFinalizedEpoch) {
+    return {
+      code: IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH,
+      finalizedEpoch: remote.finalizedEpoch,
+      maxFinalizedEpoch,
+    };
+  }
+
+  if (remote.finalizedEpoch > GENESIS_EPOCH && isZeroRoot(remote.finalizedRoot)) {
+    return {code: IrrelevantPeerCode.INVALID_FINALIZED_ROOT};
+  }
 
+  // The remote's finalized epoch is equal to ours, but the block root is different to the one in our chain.
+  // Therefore, the node is on a different chain and we should not communicate with them.
   if (
-    remote.finalizedEpoch <= local.finalizedEpoch &&
-    !isZeroRoot(remote.finalizedRoot) &&
-    !isZeroRoot(local.finalizedRoot)
+    remote.finalizedEpoch === local.finalizedEpoch &&
+    !isZeroRoot(local.finalizedRoot) &&
+    !ssz.Root.equals(remote.finalizedRoot, local.finalizedRoot)
   ) {
-    // NOTE: due to preferring to not access chain state here, we can't check the finalized root against our history.
-    // The impact of not doing check is low: peers that are behind us we can't confirm they are in the same chain as us.
-    // In the worst case they will attempt to sync from us, fail and disconnect. The ENR fork check should be sufficient
-    // to differentiate most peers in normal network conditions.
-    const remoteRoot = remote.finalizedRoot;
-    const expectedRoot = remote.finalizedEpoch === local.finalizedEpoch ? local.finalizedRoot : null;
-
-    if (expectedRoot !== null && !ssz.Root.equals(remoteRoot, expectedRoot)) {
-      return {
-        code: IrrelevantPeerCode.DIFFERENT_FINALIZED,
-        expectedRoot: expectedRoot, // forkChoice returns Tree BranchNode which the logger prints as {}
-        remoteRoot: remoteRoot,
-      };
-    }
+    return {
+      code: IrrelevantPeerCode.DIFFERENT_FINALIZED,
+      expectedRoot: local.finalizedRoot,
+      remoteRoot: remote.finalizedRoot,
+    };
   }
 
   if (isForkPostFulu(forkName) && (remote as fulu.Status).earliestAvailableSlot === undefined) {
@@ -76,7 +87,8 @@ export function assertPeerRelevance(
     };
   }
 
-  // Note: Accept request status finalized checkpoint in the future, we do not know if it is a true finalized root
+  // Note: for a behind finalized checkpoint, we validate it more on the main thread against our history,
+  // see BeaconSync.verifyPastFinalizedCheckpoint
   return null;
 }
 
@@ -93,6 +105,10 @@ export function renderIrrelevantPeerType(type: IrrelevantPeerType): string {
       return `DIFFERENT_CLOCKS slotDiff: ${type.slotDiff}`;
     case IrrelevantPeerCode.DIFFERENT_FINALIZED:
       return `DIFFERENT_FINALIZED root: ${toRootHex(type.remoteRoot)} expected: ${toRootHex(type.expectedRoot)}`;
+    case IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH:
+      return `FINALIZED_EPOCH_TOO_HIGH finalizedEpoch: ${type.finalizedEpoch} max: ${type.maxFinalizedEpoch}`;
+    case IrrelevantPeerCode.INVALID_FINALIZED_ROOT:
+      return "INVALID_FINALIZED_ROOT zero root for a non-genesis finalized epoch";
     case IrrelevantPeerCode.NO_EARLIEST_AVAILABLE_SLOT:
       return "No earliestAvailableSlot announced via peer Status";
   }

package/src/network/processor/gossipHandlers.ts

@@ -24,6 +24,7 @@ import {
   deneb,
   fulu,
   gloas,
+  isGloasBeaconBlock,
   isGloasDataColumnSidecar,
   ssz,
   sszTypesFor,
@@ -150,6 +151,31 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
  */
 function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): SequentialGossipHandlers {
   const {chain, config, metrics, logger, core} = modules;
+  const incompletePayloadWaits = new WeakSet<PayloadEnvelopeInput>();
+
+  function scheduleIncompletePayloadEnvelope(payloadInput: PayloadEnvelopeInput, peer: string): void {
+    if (payloadInput.isComplete() || incompletePayloadWaits.has(payloadInput)) {
+      return;
+    }
+    incompletePayloadWaits.add(payloadInput);
+
+    const fork = config.getForkName(payloadInput.slot);
+    const cutoffTimeMs = getCutoffTimeMs(chain, payloadInput.slot, config.getPayloadDueMs(fork));
+    payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch(() => {
+      if (
+        payloadInput.isComplete() ||
+        chain.seenPayloadEnvelopeInputCache.get(payloadInput.blockRootHex) !== payloadInput
+      ) {
+        return;
+      }
+      chain.logger.debug("Payload envelope incomplete at payload deadline", payloadInput.getLogMeta());
+      chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
+        payloadInput,
+        peer,
+        source: BlockInputSource.gossip,
+      });
+    });
+  }
 
   /**
    * Add a gossip block to the seen caches. The proposer signature of the block MUST be verified.
@@ -679,10 +705,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
           // Immediately attempt fetch of data columns from execution engine as the bid contains kzg commitments
           // which is all the information we need so there is no reason to delay until execution payload arrives
-          // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
-          // (for example later in the slot). Do not couple retries to incoming gossip columns.
-          // Columns fetched here feed payloadInput.addColumn, which resolves waitForAllData for any
-          // in-flight importExecutionPayload. No processExecutionPayload trigger needed from this path.
+          // Columns fetched here feed payloadInput.addColumn, which resolves the waitForAllData of a
+          // pending payload import. No processExecutionPayload trigger needed from this path.
           chain.getBlobsTracker.triggerGetBlobs(payloadInput);
         }
       })
@@ -692,11 +716,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
         if (e instanceof BlockError) {
           switch (e.type.code) {
-            case BlockErrorCode.DATA_UNAVAILABLE: {
-              // Error is quite frequent and not critical
+            case BlockErrorCode.DATA_UNAVAILABLE:
+              // Data did not arrive in time, BlockInputSync keeps pulling it and retries the import
               logLevel = LogLevel.debug;
               break;
-            }
             // ALREADY_KNOWN should not happen with ignoreIfKnown=true above
             // PARENT_BLOCK_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above
             case BlockErrorCode.ALREADY_KNOWN:
@@ -764,7 +787,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           // roots per proposer and slot, which bounds full imports over gossip to one alternate
           chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root) &&
           // A sibling with an unknown parent cannot be imported, sync fetches it if its branch becomes relevant
-          chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null
+          chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null &&
+          (!isGloasBeaconBlock(signedBlock.message) ||
+            chain.forkChoice.getBlockHexAndBlockHash(
+              toRootHex(signedBlock.message.parentRoot),
+              toRootHex(signedBlock.message.body.signedExecutionPayloadBid.message.parentBlockHash)
+            ) !== null)
         ) {
           const blockInput = cacheSignatureVerifiedBlock(
             signedBlock,
@@ -899,28 +927,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             }
 
             // NOTE: we do NOT call chain.processExecutionPayload here. That is triggered only by
-            // envelope arrival (gossip or API). An in-flight importExecutionPayload is awaiting
+            // envelope arrival (gossip or API). A pending payload import is awaiting
             // payloadInput.waitForAllData(); addColumn above will resolve it once hasAllData flips.
 
-            if (!payloadInput.isComplete()) {
-              const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
-              // do not await here to not delay gossip validation
-              payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch((_e) => {
-                chain.logger.debug(
-                  "Waited for envelope and data after receiving gossip column. Cut-off reached so emitting incompletePayloadEnvelope",
-                  {
-                    dataColumnIndex: index,
-                    ...payloadInputMeta,
-                  }
-                );
-                // TODO GLOAS: UnknownBlockSync to handle this event
-                chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
-                  payloadInput,
-                  peer: peerIdStr,
-                  source: BlockInputSource.gossip,
-                });
-              });
-            }
+            scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
           } catch (e) {
             logger.debug(
               "Error handling gossip data column",
@@ -1330,6 +1340,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             peerIdStr,
           });
 
+          scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
+
           chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
             slot,
             builderIndex: envelope.builderIndex,
@@ -1343,6 +1355,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
             if (e instanceof PayloadError) {
               switch (e.type.code) {
+                case PayloadErrorCode.DATA_UNAVAILABLE:
+                  // Columns did not arrive in time, BlockInputSync keeps pulling them and retries the import
+                  logLevel = LogLevel.debug;
+                  break;
                 // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
                 // already verified the block is in fork choice
                 case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:

package/src/network/processor/index.ts

@@ -867,19 +867,20 @@ export class NetworkProcessor {
     const nowSec = Date.now() / 1000;
     const minSlot = clockSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE;
 
-    for (const [slot, searchedRoots] of this.searchedRootsBySlot) {
+    for (const [slot, bufferedRoots] of this.bufferedRootsBySlot) {
       if (slot > minSlot) continue;
 
-      // expire messages awaiting an unknown block for these roots. Only roots this slot did a BLOCK search
-      // for (blockPeerIds set) have block waits keyed here; skipping the rest avoids deleting a payload wait
-      // that a newer slot created for the same root (block/envelope searches share searchedRootsBySlot).
-      for (const [rootHex, entry] of searchedRoots) {
-        if (entry.blockPeerIds === undefined) continue;
+      for (const rootHex of bufferedRoots) {
         const messagesByTopic = this.awaitingMessagesByBlockRoot.get(rootHex);
         if (messagesByTopic === undefined) continue;
-        let removed = 0;
+        let remaining = 0;
         for (const messages of messagesByTopic.values()) {
           for (const message of messages) {
+            // the same root may be buffered again from a newer slot, keep those messages.
+            // msgSlot is always set before buffering, a missing one expires rather than leaks
+            if ((message.msgSlot ?? minSlot) > minSlot) continue;
+            messages.delete(message);
+            this.awaitingBlockMessageCount--;
             const topicType = message.topic.type;
             this.metrics?.awaitingBlockGossipMessages.reject.inc({
               topic: topicType,
@@ -891,22 +892,22 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          removed += messages.size;
-        }
-        if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
-          this.awaitingBlockMessageCount -= removed;
+          remaining += messages.size;
         }
+        if (remaining === 0) this.awaitingMessagesByBlockRoot.delete(rootHex);
       }
 
-      // expire messages awaiting an unknown payload envelope for these roots. Symmetric to the block pass:
-      // only roots this slot did an ENVELOPE search for (envelopePeerIds set) have payload waits keyed here.
-      for (const [rootHex, entry] of searchedRoots) {
-        if (entry.envelopePeerIds === undefined) continue;
+      for (const rootHex of bufferedRoots) {
         const messagesByTopic = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
         if (messagesByTopic === undefined) continue;
-        let removed = 0;
+        let remaining = 0;
         for (const messages of messagesByTopic.values()) {
           for (const message of messages) {
+            // the same root may be buffered again from a newer slot, keep those messages.
+            // msgSlot is always set before buffering, a missing one expires rather than leaks
+            if ((message.msgSlot ?? minSlot) > minSlot) continue;
+            messages.delete(message);
+            this.awaitingPayloadMessageCount--;
             const topicType = message.topic.type;
             this.metrics?.awaitingPayloadGossipMessages.reject.inc({
               topic: topicType,
@@ -918,18 +919,16 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          removed += messages.size;
-        }
-        if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
-          this.awaitingPayloadMessageCount -= removed;
+          remaining += messages.size;
         }
+        if (remaining === 0) this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
       }
 
-      this.searchedRootsBySlot.delete(slot);
+      this.bufferedRootsBySlot.delete(slot);
     }
 
-    for (const slot of this.bufferedRootsBySlot.keys()) {
-      if (slot <= minSlot) this.bufferedRootsBySlot.delete(slot);
+    for (const slot of this.searchedRootsBySlot.keys()) {
+      if (slot <= minSlot) this.searchedRootsBySlot.delete(slot);
     }
 
     for (const [peer, slot] of this.fatalPeers) {

package/src/network/processor/types.ts

@@ -11,7 +11,7 @@ export type GossipAttestationsWork = {
 export type PendingGossipsubMessage = {
   topic: GossipTopic;
   msg: Message;
-  // only available for beacon_attestation and aggregate_and_proof
+  // set for every topic with a slot, held messages expire by it
   msgSlot?: Slot;
   msgId: string;
   propagationSource: PeerIdStr;

package/src/node/nodejs.ts

@@ -1,23 +1,29 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {hasher} from "@chainsafe/persistent-merkle-tree";
+import {BeaconApiMethods} from "@lodestar/api/beacon/server";
+import {BeaconConfig} from "@lodestar/config";
+import type {LoggerNode} from "@lodestar/logger/node";
 import {
   init as initNativeMetrics,
   registerLocalValidator as registerNativeLocalValidator,
   scrapeMetrics as scrapeNativeMetrics,
   unregisterLocalValidator as unregisterNativeLocalValidator,
-} from "@chainsafe/lodestar-z/metrics";
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
-import {hasher} from "@chainsafe/persistent-merkle-tree";
-import {BeaconApiMethods} from "@lodestar/api/beacon/server";
-import {BeaconConfig} from "@lodestar/config";
-import type {LoggerNode} from "@lodestar/logger/node";
+} from "@lodestar/native/metrics";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
-import {sleep, toRootHex} from "@lodestar/utils";
+import {defer, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -111,6 +117,7 @@ export class BeaconNode {
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
+  private closePromise: Promise<void> | undefined;
 
   constructor({
     opts,
@@ -212,6 +219,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos
@@ -280,6 +298,7 @@ export class BeaconNode {
         : undefined,
       builderApiClientOpts: {
         timeout: opts.executionBuilder.timeout,
+        bidDeadline: opts.executionBuilder.bidDeadline,
         // Sent with all builder api requests, unless the node runs in private mode
         userAgent: opts.executionBuilder.userAgent,
       },
@@ -375,23 +394,40 @@ export class BeaconNode {
   /**
    * Stop beacon node and its sub-components.
    */
-  async close(): Promise<void> {
-    if (this.status === BeaconNodeStatus.started) {
-      this.status = BeaconNodeStatus.closing;
-      this.sync.close();
-      if (this.restApi) await this.restApi.close();
-      await this.network.close();
-      if (this.metricsServer) await this.metricsServer.close();
-      if (this.monitoring) await this.monitoring.close();
-      await this.chain.persistToDisk();
-      await this.chain.close();
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    this.status = BeaconNodeStatus.closing;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const cleanup of [
+      () => this.sync.close(),
+      () => this.restApi?.close(),
+      () => this.network.close(),
+      () => this.metricsServer?.close(),
+      () => this.monitoring?.close(),
+      () => this.chain.persistToDisk(),
+      () => this.chain.close(),
       // Abort signal last: close() calls above clear intervals/timeouts so no new
       // operations get scheduled. If we aborted first, a still-pending interval could
       // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown.
-      if (this.controller) this.controller.abort();
-      await sleep(DELAY_BEFORE_CLOSING_DB_MS);
-      await this.db.close();
-      this.status = BeaconNodeStatus.closed;
+      () => this.controller?.abort(),
+      () => sleep(DELAY_BEFORE_CLOSING_DB_MS),
+      () => this.db.close(),
+    ]) {
+      try {
+        await cleanup();
+      } catch (error) {
+        errors.push(error);
+      }
     }
+    this.status = BeaconNodeStatus.closed;
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) throw new AggregateError(errors, "Beacon node cleanup failed");
   }
 }

package/src/node/options.ts

@@ -1,7 +1,9 @@
+import {BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS} from "../api/impl/validator/index.js";
 import {ApiOptions, defaultApiOptions} from "../api/options.js";
 import {ArchiveMode, DEFAULT_ARCHIVE_MODE, IChainOptions, defaultChainOptions} from "../chain/options.js";
 import {ValidatorMonitorOpts, defaultValidatorMonitorOpts} from "../chain/validatorMonitor.js";
 import {DatabaseOptions, defaultDbOptions} from "../db/options.js";
+import {BUILDER_BID_LAG_BUFFER_MS} from "../execution/builder/apiClient.js";
 import {
   EngineApiMode,
   ExecutionBuilderOpts,
@@ -19,16 +21,17 @@ import {SyncOptions, defaultSyncOptions} from "../sync/options.js";
 
 // Re-export so the CLI doesn't need to depend on lodestar-api
 export {allNamespaces} from "../api/rest/index.js";
-
-// Re-export to use as default values in CLI args
+export type {EngineApiMode};
+// Re-export to use as default values and for validation in CLI args
 export {
-  defaultExecutionEngineHttpOpts,
-  defaultExecutionBuilderHttpOpts,
-  engineApiModes,
   ArchiveMode,
+  BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS,
+  BUILDER_BID_LAG_BUFFER_MS,
   DEFAULT_ARCHIVE_MODE,
+  defaultExecutionBuilderHttpOpts,
+  defaultExecutionEngineHttpOpts,
+  engineApiModes,
 };
-export type {EngineApiMode};
 
 export interface IBeaconNodeOptions {
   api: ApiOptions;

package/src/sync/interface.ts

@@ -7,6 +7,7 @@ import {IBeaconDb} from "../db/index.js";
 import {Metrics} from "../metrics/index.js";
 import {INetwork} from "../network/index.js";
 import {SyncChainDebugState} from "./range/chain.js";
+
 export type {SyncChainDebugState};
 
 export type SyncingStatus = routes.node.SyncingStatus;

package/src/sync/sync.ts

@@ -1,13 +1,17 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Slot, Status} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
-import {GENESIS_SLOT} from "../constants/constants.js";
+import {GENESIS_EPOCH, GENESIS_SLOT} from "../constants/constants.js";
+import {GoodByeReasonCode} from "../constants/index.js";
+import {IBeaconDb} from "../db/index.js";
 import {ExecutionEngineState} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
+import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js";
 import {ClockEvent} from "../util/clock.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
+import {PeerIdStr} from "../util/peerId.js";
 import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
 import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
 import {SyncOptions} from "./options.js";
@@ -19,6 +23,7 @@ export class BeaconSync implements IBeaconSync {
   private readonly logger: Logger;
   private readonly network: INetwork;
   private readonly chain: IBeaconChain;
+  private readonly db: IBeaconDb;
   private readonly metrics: Metrics | null;
   private readonly opts: SyncOptions;
 
@@ -30,10 +35,11 @@ export class BeaconSync implements IBeaconSync {
   private readonly slotImportTolerance: Slot;
 
   constructor(opts: SyncOptions, modules: SyncModules) {
-    const {config, chain, metrics, network, logger} = modules;
+    const {config, chain, db, metrics, network, logger} = modules;
     this.opts = opts;
     this.network = network;
     this.chain = chain;
+    this.db = db;
     this.metrics = metrics;
     this.logger = logger;
     this.rangeSync = new RangeSync(modules, opts);
@@ -192,6 +198,11 @@ export class BeaconSync implements IBeaconSync {
    */
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
+    // Genesis and equal finalized epochs are already checked by assertPeerRelevance in the network worker
+    if (data.status.finalizedEpoch > GENESIS_EPOCH && data.status.finalizedEpoch < localStatus.finalizedEpoch) {
+      void this.verifyPastFinalizedCheckpoint(data.peer, data.status);
+    }
+
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
     this.logger.verbose("Peer sync type classified", {
       peer: data.peer,
@@ -216,6 +227,56 @@ export class BeaconSync implements IBeaconSync {
     this.updateSyncState();
   };
 
+  /**
+   * A behind peer's finalized checkpoint must be the canonical block in effect at the start slot of its epoch
+   */
+  async verifyPastFinalizedCheckpoint(peer: PeerIdStr, remote: Status): Promise<void> {
+    const startSlot = computeStartSlotAtEpoch(remote.finalizedEpoch);
+    const root = toRootHex(remote.finalizedRoot);
+    try {
+      let reason: string | null = null;
+      // After a finality jump, finalized blocks stay in fork choice until the archiver migrates them.
+      // Fork choice is pruned only after archiving, so a miss here means the db already has this range.
+      const canonicalBlock = this.chain.forkChoice.getCanonicalBlockClosestLteSlot(startSlot);
+      if (canonicalBlock !== null) {
+        if (canonicalBlock.blockRoot === root) return;
+        reason = "finalized_root_not_canonical";
+      } else {
+        const slot = await this.db.blockArchive.getSlotByRoot(remote.finalizedRoot);
+        if (slot !== null && slot > startSlot) {
+          reason = "finalized_root_after_epoch_start";
+        } else if (slot !== startSlot) {
+          const childSlot = await this.db.blockArchive.getSlotByParentRoot(remote.finalizedRoot);
+          if (childSlot !== null) {
+            if (childSlot <= startSlot) reason = "finalized_root_not_in_effect_at_epoch_start";
+          } else if (slot === null && startSlot > this.chain.earliestAvailableSlot) {
+            // childSlot == null too, neither the root nor a child is canonical
+            // or they are canonical and both pruned but peer's finalized epoch starts after earliestAvailableSlot
+            reason = "unknown_finalized_root";
+          }
+        }
+      }
+
+      if (reason !== null) {
+        this.logger.debug("Peer finalized checkpoint not in our chain", {
+          peer,
+          finalizedEpoch: remote.finalizedEpoch,
+          root,
+          reason,
+        });
+        this.metrics?.syncPeerFinalizedCheckpointMismatch.inc({reason});
+        this.network.reportPeer(peer, PeerAction.Fatal, "PeerFinalizedCheckpointMismatch");
+        await this.network.goodbyeAndDisconnect(peer, GoodByeReasonCode.IRRELEVANT_NETWORK);
+      }
+    } catch (e) {
+      this.logger.debug(
+        "Error verifying peer finalized checkpoint",
+        {peer, finalizedEpoch: remote.finalizedEpoch, root},
+        e as Error
+      );
+    }
+  }
+
   /**
    * Must be called by libp2p when a peer is removed from the peer manager
    */