@lodestar/beacon-node 1.49.0-dev.b27f498c2c → 1.49.0-dev.c535e94f25

package/src/chain/options.ts

@@ -63,10 +63,6 @@ export type BlockProcessOpts = {
    * Will double processing times. Use only for debugging purposes.
    */
   disableBlsBatchVerify?: boolean;
-  /**
-   * Assert progressive balances the same to EpochTransitionCache
-   */
-  assertCorrectProgressiveBalances?: boolean;
   /** Used for fork_choice spec tests */
   disableOnBlockError?: boolean;
   /** Used for fork_choice spec tests */
@@ -114,7 +110,6 @@ export const defaultChainOptions: IChainOptions = {
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
   graffitiAppend: true,
   serveHistoricalState: false,
-  assertCorrectProgressiveBalances: false,
   archiveStateEpochFrequency: 1024,
   archiveMode: DEFAULT_ARCHIVE_MODE,
   pruneHistory: false,

package/src/chain/produceBlock/computeNewStateRoot.ts

@@ -1,4 +1,4 @@
-import {EMPTY_SIGNATURE, IBeaconStateView} from "@lodestar/state-transition";
+import {EMPTY_SIGNATURE, IBeaconStateView, StateHashTreeRootSource} from "@lodestar/state-transition";
 import {BeaconBlock, BlindedBeaconBlock, Gwei, Root} from "@lodestar/types";
 import {Metrics} from "../../metrics/index.js";
 
@@ -14,5 +14,12 @@ export function computeNewStateRoot(
 ): {newStateRoot: Root; proposerReward: Gwei; postState: IBeaconStateView} {
   // Set signature to zero to re-use stateTransition() function which requires the SignedBeaconBlock type
   const signedBlock = {message: block, signature: EMPTY_SIGNATURE};
-  return state.computeNewStateRoot({block: signedBlock}, {metrics});
+  const {newStateRoot, proposerReward, postState, hashTreeRootTime} = state.computeNewStateRoot(
+    {block: signedBlock},
+    {metrics}
+  );
+
+  metrics?.stateHashTreeRootTime.observe({source: StateHashTreeRootSource.computeNewStateRoot}, hashTreeRootTime);
+
+  return {newStateRoot, proposerReward, postState};
 }

package/src/chain/validation/payloadAttestationMessage.ts

@@ -1,4 +1,5 @@
 import {
+  computeEpochAtSlot,
   createIndexedSignatureSetFromComponents,
   getPayloadAttestationDataSigningRoot,
   isStatePostGloas,
@@ -35,6 +36,14 @@ async function validatePayloadAttestationMessage(
 ): Promise<PayloadAttestationValidationResult> {
   const {data, validatorIndex} = payloadAttestationMessage;
 
+  // [REJECT] The payload attestation slot is at or after the Gloas fork.
+  if (computeEpochAtSlot(data.slot) < chain.config.GLOAS_FORK_EPOCH) {
+    throw new PayloadAttestationError(GossipAction.REJECT, {
+      code: PayloadAttestationErrorCode.PRE_GLOAS_SLOT,
+      slot: data.slot,
+    });
+  }
+
   // [IGNORE] The message's slot is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance), i.e. `data.slot == current_slot`.
   if (!chain.clock.isCurrentSlotGivenGossipDisparity(data.slot)) {
     throw new PayloadAttestationError(GossipAction.IGNORE, {

package/src/db/beacon.ts

@@ -11,7 +11,6 @@ import {CheckpointStateRepository} from "./repositories/checkpointState.js";
 import {
   AttesterSlashingRepository,
   BLSToExecutionChangeRepository,
-  BackfilledRanges,
   BestLightClientUpdateRepository,
   BlobSidecarsArchiveRepository,
   BlobSidecarsRepository,
@@ -61,8 +60,6 @@ export class BeaconDb implements IBeaconDb {
   syncCommittee: SyncCommitteeRepository;
   syncCommitteeWitness: SyncCommitteeWitnessRepository;
 
-  backfilledRanges: BackfilledRanges;
-
   readonly dataColumns: IDataColumnStore;
   lastLegacyArchiveSlot: Slot | null = null;
   private readonly flatFileStore: FlatFileStore;
@@ -97,8 +94,6 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommittee = new SyncCommitteeRepository(config, db);
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
-    this.backfilledRanges = new BackfilledRanges(config, db);
-
     this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
     this.dataColumns = new LegacyDataColumnStore(
       this.flatFileStore,

package/src/db/buckets.ts

@@ -56,7 +56,7 @@ export enum Bucket {
   // altair_lightClientSyncCommitteeProof = 35, // DEPRECATED on v0.32.0
   // index_lightClientInitProof = 36, // DEPRECATED on v0.32.0
 
-  backfilled_ranges = 42, // Backfilled From to To, inclusive of both From, To
+  // backfilled_ranges = 42, // DEPRECATED on v1.48.0
 
   // Buckets to support LightClient server v2
   lightClient_syncCommitteeWitness = 51, // BlockRoot -> SyncCommitteeWitness

package/src/db/interface.ts

@@ -6,7 +6,6 @@ import {CheckpointStateRepository} from "./repositories/checkpointState.js";
 import {
   AttesterSlashingRepository,
   BLSToExecutionChangeRepository,
-  BackfilledRanges,
   BestLightClientUpdateRepository,
   BlobSidecarsArchiveRepository,
   BlobSidecarsRepository,
@@ -60,8 +59,6 @@ export interface IBeaconDb {
   syncCommittee: SyncCommitteeRepository;
   syncCommitteeWitness: SyncCommitteeWitnessRepository;
 
-  backfilledRanges: BackfilledRanges;
-
   dataColumns: IDataColumnStore;
   /** Highest legacy column slot at startup; later archival writes the block-root index first. */
   readonly lastLegacyArchiveSlot: Slot | null;

package/src/db/repositories/blockArchive.ts

@@ -1,8 +1,9 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, DbBatch, FilterOptions, KeyValue, Repository} from "@lodestar/db";
 import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
-import {bytesToInt, intToBytes} from "@lodestar/utils";
+import {bytesToInt, fromHex, intToBytes} from "@lodestar/utils";
 import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {getParentRootFromSignedBeaconBlockSerialized} from "../../util/sszBytes.js";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 import {
   deleteParentRootIndex,
@@ -24,6 +25,10 @@ export type BlockArchiveBatchPutBinaryItem = KeyValue<Slot, Uint8Array> & {
   parentRoot: Root;
 };
 
+// The initial prune can span millions of slots. This bounds the slot index reads in flight and keeps each
+// delete batch at a few thousand keys, both tiny next to the block value batches in archiveBlocks
+const DELETE_RANGE_CHUNK_SIZE = 1000;
+
 /**
  * Stores finalized blocks. Block slot is identifier.
  */
@@ -103,6 +108,38 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
     );
   }
 
+  /**
+   * Delete a contiguous range of archived blocks together with their index entries.
+   * `slots` must be every archived slot in the range, so the parent of each block is the entry before it.
+   */
+  async batchDeleteRange(slots: Slot[]): Promise<void> {
+    const sorted = [...slots].sort((a, b) => a - b);
+    let prevRoot: Root | null = null;
+
+    for (let i = 0; i < sorted.length; i += DELETE_RANGE_CHUNK_SIZE) {
+      const chunk = sorted.slice(i, i + DELETE_RANGE_CHUNK_SIZE);
+      const roots = await Promise.all(chunk.map((slot) => this.getRootBySlot(slot)));
+
+      const keys: Uint8Array[] = [];
+      for (let j = 0; j < chunk.length; j++) {
+        keys.push(this.encodeKey(chunk[j]), getSlotIndexKey(chunk[j]));
+        const root = roots[j];
+        if (root) keys.push(getRootIndexKey(root));
+        // The parent index entry pointing at this block is keyed by its parent root, which is the
+        // previous block's root unless that block is unindexed, then it is read from the block itself
+        let parentRoot = prevRoot;
+        if (parentRoot === null) {
+          const block = await this.getBinary(chunk[j]);
+          const parentRootHex = block ? getParentRootFromSignedBeaconBlockSerialized(block) : null;
+          parentRoot = parentRootHex ? fromHex(parentRootHex) : null;
+        }
+        if (parentRoot) keys.push(getParentRootIndexKey(parentRoot));
+        prevRoot = root;
+      }
+      await this.db.batchDelete(keys, this.dbReqOpts);
+    }
+  }
+
   async batch(operations: DbBatch<Slot, SignedBeaconBlock>): Promise<void> {
     await this.batchBinary(
       operations.map((operation) =>

package/src/db/repositories/index.ts

@@ -1,5 +1,4 @@
 export {AttesterSlashingRepository} from "./attesterSlashing.js";
-export {BackfilledRanges} from "./backfilledRanges.js";
 export {BlobSidecarsRepository} from "./blobSidecars.js";
 export {BlobSidecarsArchiveRepository} from "./blobSidecarsArchive.js";
 export {BlockRepository} from "./block.js";

package/src/metrics/metrics/lodestar.ts

@@ -1,5 +1,6 @@
 /** biome-ignore-all lint/suspicious/noTemplateCurlyInString: The metric templates requires to have `${}` in a normal string */
 import {NotReorgedReason} from "@lodestar/fork-choice";
+import {StateHashTreeRootSource} from "@lodestar/state-transition";
 import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
 import type {LateCanonicalBlockReason} from "../../chain/archiveStore/utils/archiveBlocks.js";
@@ -25,8 +26,13 @@ import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
-import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
-import {DownloadResult, DroppedItemReason, FetchResult, PendingBlockType} from "../../sync/types.js";
+import {
+  DeferredPayloadResult,
+  DownloadResult,
+  DroppedItemReason,
+  FetchResult,
+  PendingBlockType,
+} from "../../sync/types.js";
 import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {AllocSource} from "../../util/bufferPool.js";
 import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -403,6 +409,15 @@ export function createLodestarMetrics(
       labelNames: ["caller"],
     }),
 
+    // this metrics stay here instead of @lodestar/state-transition in order to prepare
+    // for the native zig state-transition in the future
+    stateHashTreeRootTime: register.histogram<{source: StateHashTreeRootSource}>({
+      name: "lodestar_stfn_hash_tree_root_seconds",
+      help: "Time to compute the hash tree root of a post state in seconds",
+      buckets: [0.01, 0.05, 0.1, 0.2, 0.3, 0.5],
+      labelNames: ["source"],
+    }),
+
     // BLS verifier thread pool and queue
 
     bls: {
@@ -655,6 +670,16 @@ export function createLodestarMetrics(
         help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
         labelNames: ["source"],
       }),
+      deferredPayloadResult: register.counter<{result: DeferredPayloadResult}>({
+        name: "lodestar_sync_deferred_payload_result_total",
+        help: "Outcome of a slot's deferred optimistic payload searches, by result",
+        labelNames: ["result"],
+      }),
+      deferredPayloadPolls: register.histogram({
+        name: "lodestar_sync_deferred_payload_polls_count",
+        help: "Poll ticks after PAYLOAD_DUE until a searched payload was imported",
+        buckets: [0, 1, 2, 3, 6],
+      }),
       pendingBlocks: register.gauge({
         name: "lodestar_sync_unknown_block_pending_blocks_size",
         help: "Current size of UnknownBlockSync pending blocks cache",
@@ -986,6 +1011,13 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing execution payload envelopes",
         labelNames: ["error"],
       }),
+      executionPayload: {
+        recvToValidation: register.histogram({
+          name: "lodestar_gossip_execution_payload_envelope_received_to_execution_payload_verification_seconds",
+          help: "Time elapsed between execution payload envelope received and execution payload verification",
+          buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 3, 4, 6, 8],
+        }),
+      },
     },
     gossipExecutionPayloadBid: {
       elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
@@ -1120,30 +1152,6 @@ export function createLodestarMetrics(
       help: "The total result of calling notifyForkchoiceUpdate execution engine api",
       labelNames: ["result"],
     }),
-    backfillSync: {
-      backfilledTillSlot: register.gauge({
-        name: "lodestar_backfill_till_slot",
-        help: "Current lowest backfilled slot",
-      }),
-      prevFinOrWsSlot: register.gauge({
-        name: "lodestar_backfill_prev_fin_or_ws_slot",
-        help: "Slot of previous finalized or wsCheckpoint block to be validated",
-      }),
-      totalBlocks: register.gauge<{method: BackfillSyncMethod}>({
-        name: "lodestar_backfill_sync_blocks_total",
-        help: "Total amount of backfilled blocks",
-        labelNames: ["method"],
-      }),
-      errors: register.gauge({
-        name: "lodestar_backfill_sync_errors_total",
-        help: "Total number of errors while backfilling",
-      }),
-      status: register.gauge({
-        name: "lodestar_backfill_sync_status",
-        help: "Current backfill syncing status: [Aborted, Pending, Syncing, Completed]",
-      }),
-    },
-
     opPool: {
       aggregatedAttestationPool: {
         size: register.gauge({

package/src/metrics/metrics.ts

@@ -12,7 +12,12 @@ export type Metrics = BeaconMetrics &
   BeaconStateTransitionMetrics &
   LodestarMetrics & {register: RegistryMetricCreator; close: () => void};
 
-export function createMetrics(opts: MetricsOptions, genesisTime: number, externalRegistries: Registry[] = []): Metrics {
+export function createMetrics(
+  opts: MetricsOptions,
+  genesisTime: number,
+  externalRegistries: Registry[] = [],
+  {collectNodeMetrics = true}: {collectNodeMetrics?: boolean} = {}
+): Metrics {
   const register = new RegistryMetricCreator();
   const beacon = createBeaconMetrics(register);
   const forkChoice = getForkChoiceMetrics(register);
@@ -24,7 +29,7 @@ export function createMetrics(opts: MetricsOptions, genesisTime: number, externa
   };
   process.on("unhandledRejection", onUnhandledRejection);
 
-  const nodeJsMetricsClose = collectNodeJSMetrics(register);
+  const nodeJsMetricsClose = collectNodeMetrics ? collectNodeJSMetrics(register) : (): void => {};
   const close = (): void => {
     process.removeListener("unhandledRejection", onUnhandledRejection);
     nodeJsMetricsClose();

package/src/network/processor/index.ts

@@ -86,8 +86,13 @@ export const MAX_AWAITING_MESSAGES_PER_ROOT: Partial<Record<GossipType, number>>
 
 /**
  * Track the forwarded peers we already emit to UnknownBlockInput sync.
+ * envelopeSlotEmitted: whether the unknownEnvelopeBlockRootSlot event was already emitted for this root.
  */
-type SearchedRootEntry = {blockPeerIds?: Set<PeerIdStr>; envelopePeerIds?: Set<PeerIdStr>};
+type SearchedRootEntry = {
+  blockPeerIds?: Set<PeerIdStr>;
+  envelopePeerIds?: Set<PeerIdStr>;
+  envelopeSlotEmitted?: boolean;
+};
 
 /**
  * This is respective to gossipsub seenTTL (which is 550 * 0.7 = 385s), also it's respective
@@ -192,6 +197,19 @@ type PreprocessResult =
   | {action: PreprocessAction.AwaitBlock; root: RootHex}
   | {action: PreprocessAction.AwaitEnvelope; root: RootHex};
 
+enum SearchTarget {
+  Block,
+  PayloadEnvelope,
+}
+
+type SearchUnknownRootTarget =
+  | {target: SearchTarget.Block}
+  | {
+      target: SearchTarget.PayloadEnvelope;
+      /** The message slot is the payload's slot, so it can be used for UnknownBlockInput search */
+      slotIsPayloadSlot: boolean;
+    };
+
 /**
  * Network processor handles the gossip queues and throtles processing to not overload the main thread
  * - Decides when to process work and what to process
@@ -347,12 +365,17 @@ export class NetworkProcessor {
   }
 
   /**
-   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
+   * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` / `ChainEvent.unknownEnvelopeBlockRootSlot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
    * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
-  searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+  searchUnknownEnvelope(
+    {slot, root}: SlotRootHex,
+    source: BlockInputSource,
+    peer?: PeerIdStr,
+    slotIsPayloadSlot = false
+  ): void {
     if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
@@ -369,8 +392,17 @@ export class NetworkProcessor {
       forwardedPeers.add(peer);
       shouldEmit = true;
     }
+    // the payload-slot seed must reach the sync once, even if a slot-less trigger started the search first
+    if (slotIsPayloadSlot && !entry.envelopeSlotEmitted) {
+      shouldEmit = true;
+    }
     if (shouldEmit) {
-      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+      if (slotIsPayloadSlot) {
+        entry.envelopeSlotEmitted = true;
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRootSlot, {rootHex: root, slot, peer, source});
+      } else {
+        this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+      }
     }
   }
 
@@ -431,7 +463,7 @@ export class NetworkProcessor {
     // no need to check if root is a descendant of the current finalized block, it will be checked once we validate the message if needed
     if (root && !this.chain.forkChoice.hasBlockHexUnsafe(root)) {
       // starting from GLOAS, unknown root from data_column_sidecar also falls into this case
-      this.searchUnknownRoot({slot, root}, true, false, peerId);
+      this.searchUnknownRoot({slot, root}, peerId, {target: SearchTarget.Block});
       // for beacon_attestation and beacon_aggregate_and_proof messages, this is only temporary.
       // if "index = 1" we need to await for the Envelope instead
       preprocessResult = {action: PreprocessAction.AwaitBlock, root};
@@ -446,27 +478,28 @@ export class NetworkProcessor {
     if (topicType === GossipType.beacon_block) {
       const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
       if (parentRoot) {
-        let searchParentBlock = false;
-        let searchParentEnvelope = false;
+        let search: SearchUnknownRootTarget | null = null;
         if (ForkSeq[fork] >= ForkSeq.gloas) {
           // GLOAS: also check parent envelope, same logic as execution_payload_bid
           const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
           if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
             const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
             if (protoBlock === null) {
-              searchParentBlock = true;
+              search = {target: SearchTarget.Block};
             } else if (
               protoBlock.executionPayloadBlockHash &&
               protoBlock.executionPayloadBlockHash !== parentBlockHash
             ) {
               // only search for the envelope by block root if we're sure there is one. Otherwise UnknownBlockSync will penalize the peer.
-              searchParentEnvelope = true;
+              search = {target: SearchTarget.PayloadEnvelope, slotIsPayloadSlot: false};
             }
           }
         } else if (!this.chain.forkChoice.hasBlockHexUnsafe(parentRoot)) {
-          searchParentBlock = true;
+          search = {target: SearchTarget.Block};
+        }
+        if (search !== null) {
+          this.searchUnknownRoot({slot, root: parentRoot}, peerId, search);
         }
-        this.searchUnknownRoot({slot, root: parentRoot}, searchParentBlock, searchParentEnvelope, peerId);
       }
       preprocessResult = {action: PreprocessAction.PushToQueue};
     }
@@ -485,7 +518,10 @@ export class NetworkProcessor {
               : getDataIndexFromSignedAggregateAndProofSerialized(message.msg.data);
           if (attIndex === 1 && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
             // attestation votes that the payload is available but it is not yet known
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            this.searchUnknownRoot({slot, root}, peerId, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: false,
+            });
             preprocessResult = {action: PreprocessAction.AwaitEnvelope, root};
           }
           break;
@@ -494,8 +530,13 @@ export class NetworkProcessor {
           if (root == null) break;
           const payloadPresent = getPayloadPresentFromPayloadAttestationMessageSerialized(message.msg.data);
           if (payloadPresent && !this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            // payload attestation votes that the payload is available but it is not yet known
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // payload attestation votes that the payload is available but it is not yet known.
+            // this is optimistic search, the peer may not have the payload (only the ptc committee had).
+            // the PTC vote's slot is the payload's slot
+            this.searchUnknownRoot({slot, root}, undefined, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: true,
+            });
             // do not await the envelope, payload attestation processing only requires that the block is known
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -504,7 +545,12 @@ export class NetworkProcessor {
         case GossipType.data_column_sidecar: {
           if (root == null) break;
           if (!this.chain.forkChoice.hasPayloadHexUnsafe(root)) {
-            this.searchUnknownRoot({slot, root}, false, true, peerId);
+            // this is optimistic search, the peer may not have the payload.
+            // the sidecar's slot is the block's (and so the payload's) slot
+            this.searchUnknownRoot({slot, root}, undefined, {
+              target: SearchTarget.PayloadEnvelope,
+              slotIsPayloadSlot: true,
+            });
             // do not await the envelope, we can do gossip validation
             // also do not reset preprocessResult, we may already await for the block
           }
@@ -515,7 +561,7 @@ export class NetworkProcessor {
           // Extract beacon_block_root directly
           const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
           if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
-            this.searchUnknownRoot({slot, root: blockRoot}, true, false, peerId);
+            this.searchUnknownRoot({slot, root: blockRoot}, peerId, {target: SearchTarget.Block});
             // We always want to await the block
             // This allows us to properly forward the payload envelope
             preprocessResult = {action: PreprocessAction.AwaitBlock, root: blockRoot};
@@ -534,13 +580,16 @@ export class NetworkProcessor {
           ) {
             const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentBlockRoot);
             if (protoBlock === null) {
-              this.searchUnknownRoot({slot, root: parentBlockRoot}, true, false, peerId);
+              this.searchUnknownRoot({slot, root: parentBlockRoot}, peerId, {target: SearchTarget.Block});
               preprocessResult = {action: PreprocessAction.AwaitBlock, root: parentBlockRoot};
             } else if (
               protoBlock.executionPayloadBlockHash &&
               protoBlock.executionPayloadBlockHash !== parentBlockHash
             ) {
-              this.searchUnknownRoot({slot, root: parentBlockRoot}, false, true, peerId);
+              this.searchUnknownRoot({slot, root: parentBlockRoot}, peerId, {
+                target: SearchTarget.PayloadEnvelope,
+                slotIsPayloadSlot: false,
+              });
               preprocessResult = {action: PreprocessAction.AwaitEnvelope, root: parentBlockRoot};
             }
           }
@@ -641,16 +690,24 @@ export class NetworkProcessor {
     return true;
   }
 
+  /**
+   * Search block/envelope given a SlotRootHex
+   * undefined peer id means optimistic search
+   */
   private searchUnknownRoot(
     slotRoot: SlotRootHex,
-    searchBlock: boolean,
-    searchEnvelope: boolean,
-    peerId: PeerIdStr
+    peerId: PeerIdStr | undefined,
+    search: SearchUnknownRootTarget
   ): void {
-    if (!searchBlock && !searchEnvelope) return;
     if (this.tooManySearchedRoots(slotRoot.slot, slotRoot.root)) return;
-    if (searchBlock) this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
-    if (searchEnvelope) this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId);
+    switch (search.target) {
+      case SearchTarget.Block:
+        this.searchUnknownBlock(slotRoot, BlockInputSource.network_processor, peerId);
+        break;
+      case SearchTarget.PayloadEnvelope:
+        this.searchUnknownEnvelope(slotRoot, BlockInputSource.network_processor, peerId, search.slotIsPayloadSlot);
+        break;
+    }
   }
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {

package/src/node/nodejs.ts

@@ -8,7 +8,6 @@ import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
 import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
-import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
@@ -19,7 +18,6 @@ import {initializeExecutionBuilder, initializeExecutionEngine} from "../executio
 import {HttpMetricsServer, Metrics, createMetrics, getHttpMetricsServer} from "../metrics/index.js";
 import {MonitoringService} from "../monitoring/index.js";
 import {Network, getReqRespHandlers} from "../network/index.js";
-import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
 import {Clock} from "../util/clock.js";
 import {startDeferredVoluntaryExitPublisher} from "./deferredVoluntaryExitPublisher.js";
@@ -38,7 +36,6 @@ export type BeaconNodeModules = {
   chain: IBeaconChain;
   api: BeaconApiMethods;
   sync: IBeaconSync;
-  backfillSync: BackfillSync | null;
   metricsServer: HttpMetricsServer | null;
   monitoring: MonitoringService | null;
   restApi?: BeaconRestApiServer;
@@ -58,7 +55,6 @@ export type BeaconNodeInitModules = {
   peerStoreDir?: string;
   anchorState: IBeaconStateView;
   isAnchorStateFinalized: boolean;
-  wsCheckpoint?: phase0.Checkpoint;
   metricsRegistries?: Registry[];
 };
 
@@ -70,7 +66,6 @@ export enum BeaconNodeStatus {
 
 enum LoggerModule {
   api = "api",
-  backfill = "backfill",
   chain = "chain",
   execution = "execution",
   metrics = "metrics",
@@ -105,7 +100,6 @@ export class BeaconNode {
   api: BeaconApiMethods;
   restApi?: BeaconRestApiServer;
   sync: IBeaconSync;
-  backfillSync: BackfillSync | null;
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
@@ -123,7 +117,6 @@ export class BeaconNode {
     api,
     restApi,
     sync,
-    backfillSync,
     controller,
   }: BeaconNodeModules) {
     this.opts = opts;
@@ -138,7 +131,6 @@ export class BeaconNode {
     this.restApi = restApi;
     this.network = network;
     this.sync = sync;
-    this.backfillSync = backfillSync;
     this.controller = controller;
 
     this.status = BeaconNodeStatus.started;
@@ -161,7 +153,6 @@ export class BeaconNode {
     peerStoreDir,
     anchorState,
     isAnchorStateFinalized,
-    wsCheckpoint,
     metricsRegistries = [],
   }: BeaconNodeInitModules): Promise<T> {
     if (hasher.name !== "hashtree") {
@@ -295,25 +286,9 @@ export class BeaconNode {
       chain,
       metrics,
       network,
-      wsCheckpoint,
       logger: logger.child({module: LoggerModule.sync}),
     });
 
-    const backfillSync =
-      opts.sync.backfillBatchSize > 0
-        ? await BackfillSync.init(opts.sync, {
-            config,
-            db,
-            chain,
-            metrics,
-            network,
-            wsCheckpoint,
-            anchorState,
-            logger: logger.child({module: LoggerModule.backfill}),
-            signal,
-          })
-        : null;
-
     const api = getApi(opts.api, {
       config,
       logger: logger.child({module: LoggerModule.api}),
@@ -361,7 +336,6 @@ export class BeaconNode {
       api,
       restApi,
       sync,
-      backfillSync,
       controller,
     }) as T;
   }
@@ -373,7 +347,6 @@ export class BeaconNode {
     if (this.status === BeaconNodeStatus.started) {
       this.status = BeaconNodeStatus.closing;
       this.sync.close();
-      this.backfillSync?.close();
       if (this.restApi) await this.restApi.close();
       await this.network.close();
       if (this.metricsServer) await this.metricsServer.close();

package/src/sync/interface.ts

@@ -1,6 +1,6 @@
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
-import {Slot, phase0} from "@lodestar/types";
+import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {IBeaconDb} from "../db/index.js";
@@ -51,5 +51,4 @@ export interface SyncModules {
   metrics: Metrics | null;
   logger: Logger;
   chain: IBeaconChain;
-  wsCheckpoint?: phase0.Checkpoint;
 }

package/src/sync/options.ts

@@ -16,12 +16,6 @@ export type SyncOptions = {
   disableRangeSync?: boolean;
   /** USE FOR TESTING ONLY. Disable unknown block sync completely */
   disableBlockInputSync?: boolean;
-  /**
-   * The batch size of slots for backfill sync can attempt to sync/process before yielding
-   * to sync loop. This number can be increased or decreased to make a suitable resource
-   * allocation to backfill sync. The default of 0 would mean backfill sync will be skipped
-   */
-  backfillBatchSize: number;
   /** For testing only, MAX_PENDING_BLOCKS by default */
   maxPendingBlocks?: number;
 
@@ -39,7 +33,5 @@ export type SyncOptions = {
 export const defaultSyncOptions: SyncOptions = {
   isSingleNode: false,
   disableProcessAsChainSegment: false,
-  /** By default skip the backfill sync */
-  backfillBatchSize: 0,
   slotImportTolerance: SLOTS_PER_EPOCH,
 };