@lodestar/beacon-node 1.48.0-dev.7c6505e8e6 → 1.48.0-dev.82e7577c8b

package/src/api/impl/validator/index.ts

@@ -133,40 +133,37 @@ type BidCandidate = {
 };
 
 /**
- * Return the best bid by boosted total payment. A candidate with the max boost factor is
- * preferred over any other regardless of value, ties prefer a builder api bid over the
- * p2p bid, and the earlier received bid between builder api bids.
+ * Ranking value of a bid, the boosted total payment scaled by 100 as `boostFactor` is a percentage.
+ * Kept scaled so bids that differ by less than a percent are not truncated into a tie.
  */
-function selectBestBid(candidates: BidCandidate[]): BidCandidate | null {
-  const boostedValue = ({totalGwei, boostFactor}: BidCandidate): bigint => boostFactor * totalGwei;
-  let best: BidCandidate | null = null;
-  for (const candidate of candidates) {
-    if (best === null) {
-      best = candidate;
-      continue;
-    }
-    // Preserve max boost preference before comparing bid values
-    const candidateIsMaxBoost = candidate.boostFactor === MAX_BUILDER_BOOST_FACTOR;
-    const bestIsMaxBoost = best.boostFactor === MAX_BUILDER_BOOST_FACTOR;
-    if (candidateIsMaxBoost !== bestIsMaxBoost) {
-      if (candidateIsMaxBoost) {
-        best = candidate;
-      }
-      continue;
-    }
-    const candidateValue = boostedValue(candidate);
-    const bestValue = boostedValue(best);
-    if (candidateValue > bestValue) {
-      best = candidate;
-    } else if (
-      candidateValue === bestValue &&
-      // A tie prefers a builder api bid over the p2p bid, and the earlier received bid otherwise
-      (best.url === undefined || (candidate.url !== undefined && candidate.receivedMs < best.receivedMs))
-    ) {
-      best = candidate;
-    }
+function getBoostedTotalScaled({totalGwei, boostFactor}: BidCandidate): bigint {
+  return boostFactor * totalGwei;
+}
+
+/** Order bid candidates by preference, best first, used to select a bid and to log the ranking */
+function compareBidCandidates(a: BidCandidate, b: BidCandidate): number {
+  // Preserve max boost preference before comparing bid values
+  const aIsMaxBoost = a.boostFactor === MAX_BUILDER_BOOST_FACTOR;
+  const bIsMaxBoost = b.boostFactor === MAX_BUILDER_BOOST_FACTOR;
+  if (aIsMaxBoost !== bIsMaxBoost) {
+    return aIsMaxBoost ? -1 : 1;
   }
-  return best;
+
+  const aBoostedTotal = getBoostedTotalScaled(a);
+  const bBoostedTotal = getBoostedTotalScaled(b);
+  if (aBoostedTotal !== bBoostedTotal) {
+    return aBoostedTotal > bBoostedTotal ? -1 : 1;
+  }
+
+  // A tie prefers a builder api bid over the p2p bid
+  const aIsBuilderApi = a.url !== undefined;
+  const bIsBuilderApi = b.url !== undefined;
+  if (aIsBuilderApi !== bIsBuilderApi) {
+    return aIsBuilderApi ? -1 : 1;
+  }
+
+  // and the earlier received bid between builder api bids
+  return a.receivedMs - b.receivedMs;
 }
 
 type ProduceBlockContentsRes = {executionPayloadValue: Wei; consensusBlockValue: Wei} & {
@@ -1049,20 +1046,24 @@ export function getValidatorApi(
           });
         }
 
-        const best = selectBestBid(candidates);
-        if (candidates.length > 0) {
-          logger.debug("Ranked builder bid candidates", {
+        if (candidates.length === 0) {
+          return null;
+        }
+
+        const rankedCandidates = candidates.toSorted(compareBidCandidates);
+        for (const [index, candidate] of rankedCandidates.entries()) {
+          logger.debug("Builder bid candidate", {
             slot,
-            candidates: candidates
-              .map(
-                (candidate) =>
-                  `${candidate.url ?? "p2p"}:total=${prettyGweiToEth(candidate.totalGwei)}:boost=${candidate.boostFactor}:received=${candidate.receivedMs}ms`
-              )
-              .join(","),
-            bidSource: best?.url ?? "p2p",
+            rank: index + 1,
+            source: candidate.url !== undefined ? toPrintableUrl(candidate.url) : "p2p",
+            builder: candidate.signedBid.message.builderIndex,
+            total: prettyGweiToEth(candidate.totalGwei),
+            boost: candidate.boostFactor,
+            boosted: prettyGweiToEth(getBoostedTotalScaled(candidate) / 100n),
+            receivedMs: candidate.receivedMs,
           });
         }
-        return best;
+        return rankedCandidates[0];
       })();
 
       const commonBlockBodyPromise = chain.produceCommonBlockBody({

package/src/chain/archiveStore/archiveStore.ts

@@ -25,7 +25,11 @@ type ArchiveStoreModules = {
   metrics: Metrics | null;
 };
 
-type ArchiveStoreInitOpts = ArchiveStoreOpts & {dbName: string; anchorState: {finalizedCheckpoint: Checkpoint}};
+type ArchiveStoreInitOpts = ArchiveStoreOpts & {
+  dbName: string;
+  dataColumnDir: string;
+  anchorState: {finalizedCheckpoint: Checkpoint};
+};
 
 export enum ArchiveStoreTask {
   ArchiveBlocks = "archive_blocks",
@@ -121,6 +125,7 @@ export class ArchiveStore {
         opts: {
           genesisTime: this.chain.clock.genesisTime,
           dbLocation: this.opts.dbName,
+          dataColumnDir: this.opts.dataColumnDir,
           nativeStateView: this.opts.nativeStateView ?? false,
         },
         config: this.chain.config,

package/src/chain/archiveStore/historicalState/historicalStateRegen.ts

@@ -33,6 +33,7 @@ export class HistoricalStateRegen implements HistoricalStateWorkerApi {
       maxConcurrency: 1,
       maxLength: 50,
       dbLocation: modules.opts.dbLocation,
+      dataColumnDir: modules.opts.dataColumnDir,
       metricsEnabled: Boolean(modules.metrics),
       loggerOpts: modules.logger.toOpts(),
       nativeStateView: modules.opts.nativeStateView,

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,7 +1,9 @@
 import {ForkName} from "@lodestar/params";
 import {
   BeaconStateTransitionMetrics,
+  BlockProcessStep,
   EpochTransitionStep,
+  ProcessOperationsStep,
   ProposerRewardType,
   StateCloneSource,
   StateHashTreeRootSource,
@@ -70,10 +72,21 @@ export function createHistoricalStateTransitionMetrics(
     processBlockTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_seconds",
       help: "Time to process a single block in seconds",
-      // TODO: Add metrics for each step
       // Block processing can take 5-40ms, 100ms max
       buckets: [0.005, 0.01, 0.02, 0.05, 0.1, 1],
     }),
+    processBlockStepTime: metricsRegister.histogram<{step: BlockProcessStep}>({
+      name: "lodestar_historical_state_stfn_process_block_step_seconds",
+      help: "Time to call each step of process block in seconds",
+      labelNames: ["step"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1],
+    }),
+    processOperationsStepTime: metricsRegister.histogram<{step: ProcessOperationsStep}>({
+      name: "lodestar_historical_state_stfn_process_operations_step_seconds",
+      help: "Time to call each step of process operations in seconds",
+      labelNames: ["step"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1],
+    }),
     processBlockCommitTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_commit_seconds",
       help: "Time to call commit after process a single block in seconds",

package/src/chain/archiveStore/historicalState/types.ts

@@ -7,6 +7,7 @@ export type HistoricalStateRegenInitModules = {
   opts: {
     genesisTime: number;
     dbLocation: string;
+    dataColumnDir: string;
     nativeStateView: boolean;
   };
   config: BeaconConfig;
@@ -25,6 +26,7 @@ export type HistoricalStateWorkerData = {
   maxConcurrency: number;
   maxLength: number;
   dbLocation: string;
+  dataColumnDir: string;
   metricsEnabled: boolean;
   loggerOpts: LoggerNodeOpts;
   nativeStateView: boolean;

package/src/chain/archiveStore/historicalState/worker.ts

@@ -25,7 +25,10 @@ logger.info("Historical state worker started");
 
 const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot);
 
-const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}));
+const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}), {
+  dataColumnDir: workerData.dataColumnDir,
+  logger,
+});
 
 const abortController = new AbortController();
 

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -134,6 +134,8 @@ export async function archiveBlocks(
       logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
 
+    // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
+    // directly to flat files, so this only moves fallback data left by older versions.
     if (finalizedPostFulu) {
       const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
         config,
@@ -142,7 +144,7 @@ export async function archiveBlocks(
         finalizedCanonicalBlocks,
         currentEpoch
       );
-      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {
+      logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
         ...logCtx,
         migratedEntries: migratedSlots.length,
         slotRange: prettyPrintIndices(migratedSlots),
@@ -195,6 +197,23 @@ export async function archiveBlocks(
       slotRange: prettyPrintIndices(nonCanonicalSlots),
     };
 
+    const columnItems = finalizedNonCanonicalBlocks
+      // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
+      // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
+      .filter(
+        (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
+      )
+      .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
+    if (columnItems.length > 0) {
+      // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
+      await db.dataColumns.deleteMany(columnItems);
+      logger.verbose("Deleted non canonical data columns of blocks", {
+        ...logCtx,
+        blocks: columnItems.length,
+        slotRange: prettyPrintIndices(columnItems.map(({slot}) => slot).sort((a, b) => a - b)),
+      });
+    }
+
     await db.block.batchDelete(nonCanonicalBlockRoots);
     logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
 
@@ -203,11 +222,6 @@ export async function archiveBlocks(
       logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
     }
 
-    if (finalizedPostFulu) {
-      await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", nonCanonicalLogCtx);
-    }
-
     if (finalizedPostGloas) {
       await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
       logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
@@ -245,23 +259,16 @@ export async function archiveBlocks(
       );
       const dataColumnSidecarsMinEpoch = currentEpoch - dataColumnSidecarsArchiveWindow;
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
-        const prefixedKeys = await db.dataColumnSidecarArchive.keys({
-          // The `id` value `0` refers to the column index. So we want to fetch all sidecars less than zero column of `dataColumnSidecarsMinEpoch`
-          lt: {prefix: computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch), id: 0},
-        });
-        // for each slot there could be multiple dataColumnSidecar, so we need to deduplicate it
-        const slotsToDelete = [...new Set(prefixedKeys.map(({prefix}) => prefix))].sort((a, b) => a - b);
-
-        if (slotsToDelete.length > 0) {
-          await db.dataColumnSidecarArchive.deleteMany(slotsToDelete);
+        const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
+        const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
+        if (prunedColumnSlots.length > 0) {
           logger.verbose("dataColumnSidecars prune", {
             ...logCtx,
-            slotRange: prettyPrintIndices(slotsToDelete),
-            numOfSlots: slotsToDelete.length,
-            totalNumOfSidecars: prefixedKeys.length,
+            slotRange: prettyPrintIndices(prunedColumnSlots),
+            numOfSlots: prunedColumnSlots.length,
           });
         } else {
-          logger.verbose(`dataColumnSidecars prune: no entries before epoch ${dataColumnSidecarsMinEpoch}`, logCtx);
+          logger.verbose("dataColumnSidecars prune: no entries before slot", {...logCtx, slot: columnsPruneSlot});
         }
       } else {
         logger.verbose(
@@ -389,7 +396,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
       )
     ).filter((e): e is KeyValue<Slot, Uint8Array> => e !== null);
 
-    // put to blockArchive db and delete block db
     await Promise.all([
       db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries),
       db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
@@ -402,11 +408,9 @@ async function migrateBlobSidecarsFromHotToColdDb(
 
 // TODO: This function can be simplified further by reducing layers of promises in a loop
 /**
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope —
- * columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas (Fulu)
- * blocks only have a FULL variant, so the `payloadStatus === FULL` filter passes them all.
- * Blocks whose canonical variant is PENDING/EMPTY are skipped here — their columns will be picked
- * up on a later run once the FULL variant appears in the ancestor walk.
+ * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
+ * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
+ * blocks only have a FULL variant, so the payload-status filter passes them all.
  */
 async function migrateDataColumnSidecarsFromHotToColdDb(
   config: ChainForkConfig,
@@ -428,32 +432,24 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
     if (batch.length === 0) break;
 
     const promises: Promise<void>[] = [];
-
-    // load Buffer instead of ssz deserialized to improve performance
     for (const block of batch) {
-      const blockSlot = block.slot;
-      const blockEpoch = computeEpochAtSlot(blockSlot);
-
+      const blockEpoch = computeEpochAtSlot(block.slot);
       if (
-        config.getForkSeq(blockSlot) < ForkSeq.fulu ||
-        // if block is out of ${config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS}, skip this step
+        config.getForkSeq(block.slot) < ForkSeq.fulu ||
         blockEpoch < currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS
       ) {
         continue;
       }
 
-      // Here we assume the data column sidecars are already in the hot db
       const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
       if (dataColumnSidecarBytes.length === 0) {
-        // Empty stream: either the block has no blobs, or columns were already archived on a
-        // previous run (boundary block). Nothing to migrate.
         logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
           slot: block.slot,
           root: toRootHex(block.root),
         });
         continue;
       }
-      logger.verbose("Migrated dataColumnSidecars for block", {
+      logger.verbose("Migrated legacy dataColumnSidecars for block", {
         currentEpoch,
         slot: block.slot,
         root: toRootHex(block.root),
@@ -462,15 +458,14 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
       promises.push(
         db.dataColumnSidecarArchive.putManyBinary(
           block.slot,
-          dataColumnSidecarBytes.map((p) => ({key: p.id, value: p.value}))
+          dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
         )
       );
       migratedSlots.push(block.slot);
     }
 
-    promises.push(db.dataColumnSidecar.deleteMany(batch.map((block) => block.root)));
-
     await Promise.all(promises);
+    await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
   }
 
   // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.

package/src/chain/blocks/importBlock.ts

@@ -524,19 +524,6 @@ export async function importBlock(
         this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
         this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
       }
-      const finalizedCheckpoint = checkpointState.finalizedCheckpoint;
-      const finalizedEpoch = finalizedCheckpoint.epoch;
-      const preFinalizedEpoch = parentBlockSummary.finalizedEpoch;
-      if (finalizedEpoch > preFinalizedEpoch) {
-        this.emitter.emit(routes.events.EventType.finalizedCheckpoint, {
-          block: toRootHex(finalizedCheckpoint.root),
-          epoch: finalizedCheckpoint.epoch,
-          state: toRootHex(checkpointState.hashTreeRoot()),
-          executionOptimistic: false,
-        });
-        this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
-        this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
-      }
     }
   }
 

package/src/chain/blocks/index.ts

@@ -1,3 +1,4 @@
+import {PayloadStatus} from "@lodestar/fork-choice";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {SignedBeaconBlock, Slot} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
@@ -6,15 +7,17 @@ import {nextEventLoop} from "../../util/eventLoop.js";
 import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.js";
+import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {importBlock, importsBlockAttestations} from "./importBlock.js";
 import {PayloadError, importExecutionPayload} from "./importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "./payloadEnvelopeInput/payloadEnvelopeInput.js";
-import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
-import {assertLinearChainSegment} from "./utils/chainSegment.js";
+import {FullyVerifiedBlock, ImportBlockOpts, ProcessBlocksResult} from "./types.js";
+import {OrphanedPayloadEnvelope, assertLinearChainSegment} from "./utils/chainSegment.js";
 import {verifyBlocksInEpoch} from "./verifyBlock.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
+import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
 
 export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";
 
@@ -24,10 +27,16 @@ const QUEUE_MAX_LENGTH = 256;
  * BlockProcessor processes block jobs in a queued fashion, one after the other.
  */
 export class BlockProcessor {
-  readonly jobQueue: JobItemQueue<[IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts], void>;
+  readonly jobQueue: JobItemQueue<
+    [IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts],
+    ProcessBlocksResult
+  >;
 
   constructor(chain: BeaconChain, metrics: Metrics | null, opts: BlockProcessOpts, signal: AbortSignal) {
-    this.jobQueue = new JobItemQueue<[IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts], void>(
+    this.jobQueue = new JobItemQueue<
+      [IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts],
+      ProcessBlocksResult
+    >(
       (job, payloadEnvelopes, importOpts) => {
         return processBlocks.call(chain, job, payloadEnvelopes, {...opts, ...importOpts});
       },
@@ -40,8 +49,8 @@ export class BlockProcessor {
     job: IBlockInput[],
     payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
     opts: ImportBlockOpts = {}
-  ): Promise<void> {
-    await this.jobQueue.push(job, payloadEnvelopes, opts);
+  ): Promise<ProcessBlocksResult> {
+    return this.jobQueue.push(job, payloadEnvelopes, opts);
   }
 }
 
@@ -60,9 +69,10 @@ export async function processBlocks(
   blocks: IBlockInput[],
   payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
   opts: BlockProcessOpts & ImportBlockOpts
-): Promise<void> {
+): Promise<ProcessBlocksResult> {
+  const emptyResult: ProcessBlocksResult = {orphaned: [], skipped: false};
   if (blocks.length === 0) {
-    return; // TODO: or throw?
+    return emptyResult; // TODO: or throw?
   }
 
   try {
@@ -71,7 +81,7 @@ export async function processBlocks(
     // No relevant blocks, skip verifyBlocksInEpoch()
     if (relevantBlocks.length === 0 || parentBlock === null) {
       // parentBlock can only be null if relevantBlocks are empty
-      return;
+      return await importPayloadEnvelopesOfKnownBlocks.call(this, payloadEnvelopes, opts);
     }
 
     const {warnings: orphanedPayloads} = assertLinearChainSegment(
@@ -86,18 +96,15 @@ export async function processBlocks(
     // status tiebreaker then picks FULL and parks the head there.
     const blockEpoch = computeEpochAtSlot(relevantBlocks[0].getBlock().message.slot);
     const importsAttestations = importsBlockAttestations(opts, blockEpoch, this.clock.currentEpoch);
+    const skipped = orphanedPayloads != null && payloadEnvelopes !== null && !importsAttestations;
     let payloadEnvelopesToImport = payloadEnvelopes;
-    if (orphanedPayloads != null && payloadEnvelopes !== null && !importsAttestations) {
+    if (skipped) {
       payloadEnvelopesToImport = new Map(payloadEnvelopes);
-      for (const orphaned of orphanedPayloads) {
+      for (const orphaned of orphanedPayloads ?? []) {
         payloadEnvelopesToImport.delete(orphaned.slot);
         // never validated, drop it from the shared cache so it is not served to peers, by-root sync reloads the
         // entry from db if the payload is ever needed
         this.seenPayloadEnvelopeInputCache.prune(orphaned.payloadEnvelopeInput.blockRootHex);
-        this.logger.debug("Skipping orphaned payload envelope in chain segment", {
-          slot: orphaned.slot,
-          blockRoot: orphaned.payloadEnvelopeInput.blockRootHex,
-        });
       }
     }
 
@@ -172,9 +179,11 @@ export async function processBlocks(
 
       await nextEventLoop();
     }
+
+    return {orphaned: orphanedPayloads ?? [], skipped};
   } catch (e) {
     if (isErrorAborted(e) || isQueueErrorAborted(e) || isBlockErrorAborted(e)) {
-      return; // Ignore
+      return emptyResult; // Ignore
     }
 
     // above functions should only throw BlockError, or PayloadError from the gloas payload import
@@ -200,8 +209,19 @@ export async function processBlocks(
         const blockSlot = signedBlock.message.slot;
         const {state} = err.type;
         const forkTypes = this.config.getForkTypes(blockSlot);
-        this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
-        this.persistInvalidSszBytes("BeaconState", state.serialize(), `${state.slot}_invalid_signature`);
+        const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
+        this.persistInvalidSszValue(
+          forkTypes.SignedBeaconBlock,
+          signedBlock,
+          `${blockSlot}_invalid_signature`,
+          blockRootHex
+        );
+        this.persistInvalidSszBytes(
+          "BeaconState",
+          state.serialize(),
+          toRootHex(state.hashTreeRoot()),
+          `${state.slot}_invalid_signature`
+        );
       } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
         const {signedBlock} = err;
         const blockSlot = signedBlock.message.slot;
@@ -222,6 +242,70 @@ export async function processBlocks(
   }
 }
 
+/**
+ * A peer can serve a block without its envelope, the block is then imported with a PENDING payload. Later batches
+ * carry the envelope but all their blocks are known, without importing it here every child building on the FULL
+ * variant fails with PARENT_PAYLOAD_UNKNOWN and range sync never progresses. If our head already descends from the
+ * block's EMPTY variant the chain did not build on the payload, importing it would only add a dead FULL leaf. As for
+ * orphaned payloads in a segment this only applies while the block attestations are not imported, a recent payload
+ * is imported anyway so a reorg building on it does not have to fetch it through unknown block sync.
+ */
+async function importPayloadEnvelopesOfKnownBlocks(
+  this: BeaconChain,
+  payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
+  opts: ImportBlockOpts
+): Promise<ProcessBlocksResult> {
+  if (payloadEnvelopes === null) {
+    return {orphaned: [], skipped: false};
+  }
+
+  const head = this.forkChoice.getHead();
+  const currentEpoch = this.clock.currentEpoch;
+  const payloadInputs: PayloadEnvelopeInput[] = [];
+  const orphaned: OrphanedPayloadEnvelope[] = [];
+  for (const payloadInput of payloadEnvelopes.values()) {
+    const {blockRootHex, slot} = payloadInput;
+    if (
+      !payloadInput.hasPayloadEnvelope() ||
+      !this.forkChoice.hasBlockHex(blockRootHex) ||
+      this.forkChoice.getBlockHexAndBlockHash(blockRootHex, payloadInput.getBlockHashHex()) !== null
+    ) {
+      continue;
+    }
+    if (
+      head.blockRoot !== blockRootHex &&
+      !importsBlockAttestations(opts, computeEpochAtSlot(slot), currentEpoch) &&
+      this.forkChoice.isDescendant(blockRootHex, PayloadStatus.EMPTY, head.blockRoot, head.payloadStatus)
+    ) {
+      // never validated, see processBlocks()
+      this.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+      orphaned.push({slot, payloadEnvelopeInput: payloadInput});
+      continue;
+    }
+    payloadInputs.push(payloadInput);
+  }
+
+  // orphaned envelopes on this path are always skipped, range sync logs them with peer/client context
+  const result: ProcessBlocksResult = {orphaned, skipped: orphaned.length > 0};
+
+  if (payloadInputs.length === 0) {
+    return result;
+  }
+
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  for (let i = 0; i < payloadInputs.length; i++) {
+    this.logger.debug("Importing payload envelope of known block", {
+      slot: payloadInputs[i].slot,
+      blockRoot: payloadInputs[i].blockRootHex,
+    });
+    await importExecutionPayload.call(this, payloadInputs[i], dataAvailabilityStatuses[i], {validSignature: false});
+  }
+  // the new FULL variants are not seen by the head cached from the last block import
+  this.recomputeForkChoiceHead(ForkchoiceCaller.importBlock);
+
+  return result;
+}
+
 function getBlockOrPayloadError(e: unknown, block: SignedBeaconBlock): BlockError | PayloadError {
   if (e instanceof BlockError) {
     return e;

package/src/chain/blocks/types.ts

@@ -4,6 +4,7 @@ import {ForkSeq} from "@lodestar/params";
 import {DataAvailabilityStatus, IBeaconStateView, computeEpochAtSlot} from "@lodestar/state-transition";
 import type {IndexedAttestation, Slot, fulu} from "@lodestar/types";
 import {IBlockInput} from "./blockInput/types.js";
+import {OrphanedPayloadEnvelope} from "./utils/chainSegment.js";
 
 export enum GossipedInputType {
   block = "block",
@@ -110,3 +111,11 @@ export type FullyVerifiedBlock = {
   /** If the execution payload couldn't be verified because of EL syncing status, used in optimistic sync */
   executionStatus: BlockExecutionStatus | PayloadExecutionStatus;
 };
+
+/**
+ * Result of processing a chain segment.
+ */
+export type ProcessBlocksResult = {
+  orphaned: OrphanedPayloadEnvelope[];
+  skipped: boolean;
+};

package/src/chain/blocks/utils/chainSegment.ts

@@ -22,7 +22,7 @@ export type ChainSegmentResult = {warnings: OrphanedPayloadEnvelope[] | null};
  * - Verifies parent root + slot linearity
  * - For gloas: verifies bid.parentBlockHash matches the tracked execution hash; if not, the
  *   previous FULL envelope is treated as orphaned (segment continues as if previous slot was EMPTY)
- * - If an envelope exists for this slot: verifies it references this block's root
+ * - If an envelope exists for this slot (or the parent's slot): verifies it references this block's root
  * - Advances the tracked execution hash (FULL if envelope present, EMPTY if not)
  */
 export function assertLinearChainSegment(
@@ -48,6 +48,15 @@ export function assertLinearChainSegment(
   if (parentBlock !== null) {
     const parentPayloadInput = payloadEnvelopes?.get(parentBlock.slot);
     if (parentPayloadInput?.hasPayloadEnvelope()) {
+      // Envelopes are served by slot, the one at the parent's slot must reference the parent block. Without this
+      // check an envelope of a sibling block is taken as orphaned and only fails downstream in the payload import
+      if (parentPayloadInput.blockRootHex !== parentBlock.blockRoot) {
+        throw new BlockError(blocks[0].getBlock(), {
+          code: BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH,
+          envelopeBlockRoot: parentPayloadInput.blockRootHex,
+          blockRoot: parentBlock.blockRoot,
+        });
+      }
       const parentPayloadBlockHash = parentPayloadInput.getBlockHashHex();
       if (currentExecHash === parentPayloadBlockHash) {
         // Checkpoint-sync first batch: the anchor is stored PENDING with the inherited (grandparent)