@lodestar/beacon-node 1.43.0-dev.78c66bac71 → 1.43.0-dev.87cbe69c66

package/src/network/gossip/topic.ts

@@ -73,6 +73,7 @@ function stringifyGossipTopicType(topic: GossipTopic): string {
     case GossipType.execution_payload:
     case GossipType.payload_attestation_message:
     case GossipType.execution_payload_bid:
+    case GossipType.proposer_preferences:
       return topic.type;
     case GossipType.beacon_attestation:
     case GossipType.sync_committee:
@@ -124,6 +125,8 @@ export function getGossipSSZType(topic: GossipTopic) {
       return ssz.gloas.PayloadAttestationMessage;
     case GossipType.execution_payload_bid:
       return ssz.gloas.SignedExecutionPayloadBid;
+    case GossipType.proposer_preferences:
+      return ssz.gloas.SignedProposerPreferences;
   }
 }
 
@@ -203,6 +206,7 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr:
       case GossipType.execution_payload:
       case GossipType.payload_attestation_message:
       case GossipType.execution_payload_bid:
+      case GossipType.proposer_preferences:
         return {type: gossipTypeStr, boundary, encoding};
     }
 
@@ -257,6 +261,7 @@ export function getCoreTopicsAtFork(
     topics.push({type: GossipType.execution_payload});
     topics.push({type: GossipType.payload_attestation_message});
     topics.push({type: GossipType.execution_payload_bid});
+    topics.push({type: GossipType.proposer_preferences});
   }
 
   // After fulu also track data_column_sidecar_{index}
@@ -351,4 +356,5 @@ export const gossipTopicIgnoreDuplicatePublishError: Record<GossipType, boolean>
   [GossipType.execution_payload]: true,
   [GossipType.payload_attestation_message]: true,
   [GossipType.execution_payload_bid]: true,
+  [GossipType.proposer_preferences]: true,
 };

package/src/network/interface.ts

@@ -112,6 +112,7 @@ export interface INetwork extends INetworkCorePublic {
   publishLightClientFinalityUpdate(update: LightClientFinalityUpdate): Promise<number>;
   publishLightClientOptimisticUpdate(update: LightClientOptimisticUpdate): Promise<number>;
   publishSignedExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope): Promise<number>;
+  publishPayloadAttestationMessage(payloadAttestationMessage: gloas.PayloadAttestationMessage): Promise<number>;
 
   // Debug
   dumpGossipQueue(gossipType: GossipType): Promise<PendingGossipsubMessage[]>;

package/src/network/network.ts

@@ -515,6 +515,17 @@ export class Network implements INetwork {
     );
   }
 
+  async publishPayloadAttestationMessage(payloadAttestationMessage: gloas.PayloadAttestationMessage): Promise<number> {
+    const epoch = computeEpochAtSlot(payloadAttestationMessage.data.slot);
+    const boundary = this.config.getForkBoundaryAtEpoch(epoch);
+
+    return this.publishGossip<GossipType.payload_attestation_message>(
+      {type: GossipType.payload_attestation_message, boundary},
+      payloadAttestationMessage,
+      {ignoreDuplicatePublishError: true}
+    );
+  }
+
   private async publishGossip<K extends GossipType>(
     topic: GossipTopicMap[K],
     object: GossipTypeMap[K],

package/src/network/processor/gossipHandlers.ts

@@ -52,6 +52,8 @@ import {
   ExecutionPayloadEnvelopeErrorCode,
   GossipAction,
   GossipActionError,
+  PayloadAttestationError,
+  PayloadAttestationErrorCode,
   SyncCommitteeError,
 } from "../../chain/errors/index.js";
 import {IBeaconChain} from "../../chain/interface.js";
@@ -79,6 +81,7 @@ import {
 import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js";
 import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js";
 import {validateGossipPayloadAttestationMessage} from "../../chain/validation/payloadAttestationMessage.js";
+import {validateGossipProposerPreferences} from "../../chain/validation/proposerPreferences.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
 import {Metrics} from "../../metrics/index.js";
 import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
@@ -182,6 +185,18 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     });
     try {
       await validateGossipBlock(config, chain, signedBlock, fork);
+
+      if (isForkPostGloas(fork)) {
+        chain.seenPayloadEnvelopeInputCache.add({
+          blockRootHex,
+          block: signedBlock as SignedBeaconBlock<ForkPostGloas>,
+          forkName: fork,
+          sampledColumns: chain.custodyConfig.sampledColumns,
+          custodyColumns: chain.custodyConfig.custodyColumns,
+          timeCreatedSec: seenTimestampSec,
+        });
+      }
+
       const blockInputMeta = blockInput.getLogMeta();
 
       const recvToValidation = Date.now() / 1000 - seenTimestampSec;
@@ -198,7 +213,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     } catch (e) {
       if (e instanceof BlockGossipError) {
         logger.debug("Gossip block has error", {slot, root: blockShortHex, code: e.type.code});
-        if (e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput) {
+        if (
+          (e.type.code === BlockErrorCode.PARENT_UNKNOWN || e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) &&
+          blockInput
+        ) {
           chain.emitter.emit(ChainEvent.blockUnknownParent, {
             blockInput,
             peer: peerIdStr,
@@ -1057,10 +1075,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const signedEnvelope = sszDeserialize(topic, serializedData);
       const envelope = signedEnvelope.message;
 
-      // TODO GLOAS: consider optimistically create PayloadEnvelopeInput here similar to how we do that for beacon_block
-      // so that UnknownBlockSync can handle backward sync
-      // the problem now is we cannot create a PayloadEnvelopeInput without the beacon block being known, we need at least the proposer index
-      // we can achieve that by looking into the EpochCache
+      // unlike BlockInput, we send the envelope into UnknownBlockInput sync
+      // inside the sync it'll reconcile into PayloadEnvelopeInput and share the same cache with gossip
       try {
         await validateGossipExecutionPayloadEnvelope(chain, signedEnvelope);
       } catch (e) {
@@ -1068,14 +1084,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           const {beaconBlockRoot} = signedEnvelope.message;
           const slot = signedEnvelope.message.payload.slotNumber;
           logger.debug("Gossip envelope has error", {slot, root: toRootHex(beaconBlockRoot), code: e.type.code});
-          if (e.type.code === ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN) {
-            // TODO GLOAS: UnknownBlockSync to handle this
-            chain.emitter.emit(ChainEvent.envelopeUnknownBlock, {
-              envelope: signedEnvelope,
-              peer: peerIdStr,
-              source: BlockInputSource.gossip,
-            });
-          }
 
           if (e.action === GossipAction.REJECT) {
             chain.persistInvalidSszValue(
@@ -1170,6 +1178,14 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         data: executionPayloadBid,
       });
     },
+    [GossipType.proposer_preferences]: async ({
+      gossipData,
+      topic,
+    }: GossipHandlerParamGeneric<GossipType.proposer_preferences>) => {
+      const {serializedData} = gossipData;
+      const signedProposerPreferences = sszDeserialize(topic, serializedData);
+      await validateGossipProposerPreferences(chain, signedProposerPreferences);
+    },
   };
 }
 
@@ -1292,10 +1308,12 @@ export async function validateGossipFnRetryUnknownRoot<T>(
     try {
       return await fn();
     } catch (e) {
-      if (
-        e instanceof AttestationError &&
-        e.type.code === AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT
-      ) {
+      const isUnknownAttestationRoot =
+        e instanceof AttestationError && e.type.code === AttestationErrorCode.UNKNOWN_OR_PREFINALIZED_BEACON_BLOCK_ROOT;
+      const isUnknownPayloadAttestationRoot =
+        e instanceof PayloadAttestationError && e.type.code === PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT;
+
+      if (isUnknownAttestationRoot || isUnknownPayloadAttestationRoot) {
         if (unknownBlockRootRetries === 0) {
           // Trigger unknown block root search here
           const rootHex = toRootHex(blockRoot);

package/src/network/processor/gossipQueues/index.ts

@@ -83,6 +83,11 @@ const linearGossipQueueOpts: {
     type: QueueType.FIFO,
     dropOpts: {type: DropType.count, count: 1},
   },
+  [GossipType.proposer_preferences]: {
+    maxLength: 1024,
+    type: QueueType.FIFO,
+    dropOpts: {type: DropType.count, count: 1},
+  },
 };
 
 const indexedGossipQueueOpts: {

package/src/network/processor/index.ts

@@ -76,6 +76,7 @@ type WorkOpts = {
  */
 const executeGossipWorkOrderObj: Record<GossipType, WorkOpts> = {
   [GossipType.beacon_block]: {bypassQueue: true},
+  [GossipType.execution_payload]: {bypassQueue: true},
   [GossipType.blob_sidecar]: {bypassQueue: true},
   [GossipType.data_column_sidecar]: {bypassQueue: true},
   [GossipType.beacon_aggregate_and_proof]: {},
@@ -88,9 +89,9 @@ const executeGossipWorkOrderObj: Record<GossipType, WorkOpts> = {
   [GossipType.sync_committee]: {},
   [GossipType.light_client_finality_update]: {},
   [GossipType.light_client_optimistic_update]: {},
-  [GossipType.execution_payload]: {bypassQueue: true},
   [GossipType.payload_attestation_message]: {},
   [GossipType.execution_payload_bid]: {},
+  [GossipType.proposer_preferences]: {},
 };
 const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj) as (keyof typeof executeGossipWorkOrderObj)[];
 
@@ -443,8 +444,7 @@ export class NetworkProcessor {
         }
         case GossipType.execution_payload: {
           // extractBlockSlotRootFn does not return a root for this topic.
-          // Extract beacon_block_root directly and proactively trigger block sync if missing.
-          // Do NOT await the block — the handler runs immediately; BlockInputSync handles recovery.
+          // Extract beacon_block_root directly
           const blockRoot = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(message.msg.data);
           if (blockRoot && !this.chain.forkChoice.hasBlockHexUnsafe(blockRoot)) {
             this.searchUnknownBlock(
@@ -452,9 +452,10 @@ export class NetworkProcessor {
               BlockInputSource.network_processor,
               message.propagationSource.toString()
             );
+            // We always want to await the block
+            // This allows us to properly forward the payload envelope
+            preprocessResult = {action: PreprocessAction.AwaitBlock, root: blockRoot};
           }
-          // do not await the block, we want UnknownBlockSync to handle it.
-          preprocessResult = {action: PreprocessAction.PushToQueue};
           break;
         }
         case GossipType.execution_payload_bid: {

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -24,6 +24,10 @@ export async function* onBeaconBlocksByRange(
   // in the case of initializing from a non-finalized state, we don't have the finalized block so this api does not work
   // chain.forkChoice.getFinalizeBlock().slot
   const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
+  // Blocks are migrated to blockArchive at finalization (including the finalized block itself),
+  // so the archive loop serves up to AND INCLUDING finalizedSlot and the headChain loop
+  // starts above it to avoid duplicate yields. See archiveBlocks.ts for the migration logic.
+  const archiveMaxSlot = finalizedSlot;
 
   const forkName = chain.config.getForkName(startSlot);
   if (isForkPostFulu(forkName) && startSlot < chain.earliestAvailableSlot) {
@@ -35,9 +39,12 @@ export async function* onBeaconBlocksByRange(
   }
 
   // Finalized range of blocks
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     // Chain of blobs won't change
-    for await (const {key, value} of finalized.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
+    for await (const {key, value} of finalized.binaryEntriesStream({
+      gte: startSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
+    })) {
       yield {
         data: value,
         boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(finalized.decodeKey(key))),
@@ -46,19 +53,20 @@ export async function* onBeaconBlocksByRange(
   }
 
   // Non-finalized range of blocks
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
-    // getAllAncestorBlocks response includes the head node, so it's the full chain.
+    // `getAllAncestorBlocks` includes both the head and the previous-finalized boundary.
 
     // Iterate head chain with ascending block numbers
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      // Must include only blocks in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      // Must include only blocks in the range requested, and skip anything the archive loop
+      // above already served via the block.slot > archiveMaxSlot filter.
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of blobs must be consistent, but on
         // re-org there's no need to abort the request

package/src/network/reqresp/handlers/blobSidecarsByRange.ts

@@ -20,31 +20,37 @@ export async function* onBlobSidecarsByRange(
   const finalized = db.blobSidecarsArchive;
   const unfinalized = db.blobSidecars;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // Blobs are migrated to blobSidecarsArchive at finalization (including the finalized block
+  // itself), so the archive loop serves up to AND INCLUDING finalizedSlot and the headChain
+  // loop starts above it to avoid duplicate yields. See archiveBlocks.ts for the migration logic.
+  const archiveMaxSlot = finalizedSlot;
 
   // Finalized range of blobs
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     // Chain of blobs won't change
     for await (const {key, value: blobSideCarsBytesWrapped} of finalized.binaryEntriesStream({
       gte: startSlot,
-      lt: endSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
     })) {
       yield* iterateBlobBytesFromWrapper(chain, blobSideCarsBytesWrapped, finalized.decodeKey(key));
     }
   }
 
   // Non-finalized range of blobs
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
+    // `getAllAncestorBlocks` includes both the head and the previous-finalized boundary.
 
     // Iterate head chain with ascending block numbers
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      // Must include only blobs in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      // Must include only blobs in the range requested, and skip anything the archive loop
+      // above already served via the block.slot > archiveMaxSlot filter.
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of blobs must be consistent, but on
         // re-org there's no need to abort the request

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -1,6 +1,6 @@
 import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
-import {GENESIS_SLOT} from "@lodestar/params";
+import {ForkSeq, GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {ColumnIndex, Epoch, fulu} from "@lodestar/types";
@@ -43,10 +43,19 @@ export async function* onDataColumnSidecarsByRange(
 
   const finalized = db.dataColumnSidecarArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // Columns of the last finalized block live in different DBs depending on fork:
+  // - Pre-gloas (fulu): migrated to dataColumnSidecarArchive in the same finalization run.
+  // - Post-gloas: stay in the hot db (db.dataColumnSidecar) until the next finalization run,
+  //   because the migration filter requires payloadStatus === FULL for gloas blocks.
+  // archiveMaxSlot is the last slot whose columns are served by the archive loop below;
+  // anything above it is served by the headChain loop.
+  const isPostGloasFinalized = chain.config.getForkSeq(finalizedSlot) >= ForkSeq.gloas;
+  const archiveMaxSlot = isPostGloasFinalized ? finalizedSlot - 1 : finalizedSlot;
 
   // Finalized range of columns
-  if (startSlot <= finalizedSlot) {
-    for (let slot = startSlot; slot < endSlot; slot++) {
+  if (startSlot <= archiveMaxSlot) {
+    const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
+    for (let slot = startSlot; slot < archiveEnd; slot++) {
       const dataColumnSidecars = await finalized.getManyBinary(slot, availableColumns);
 
       const unavailableColumnIndices: ColumnIndex[] = [];
@@ -81,9 +90,12 @@ export async function* onDataColumnSidecarsByRange(
   }
 
   // Non-finalized range of columns
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
+    // getAllAncestorBlocks includes the last finalized block as its final element.
+    // Skip anything the archive loop above already served via the block.slot > archiveMaxSlot
+    // filter below (pre-gloas this skips finalizedSlot, post-gloas it keeps it).
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
 
     // Iterate head chain with ascending block numbers
@@ -91,7 +103,7 @@ export async function* onDataColumnSidecarsByRange(
       const block = headChain[i];
 
       // Must include only columns in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of columns must be consistent, but on
         // re-org there's no need to abort the request

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -21,12 +21,15 @@ export async function* onExecutionPayloadEnvelopesByRange(
 
   const finalized = db.executionPayloadEnvelopeArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
+  // The current finalized block's envelope is still in the hot db; archive migration happens
+  // in the next finalization run (see migrateExecutionPayloadEnvelopesFromHotToColdDb).
+  const archiveMaxSlot = finalizedSlot - 1;
 
   // Finalized range of envelopes
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     for await (const {key, value: envelopeBytes} of finalized.binaryEntriesStream({
       gte: startSlot,
-      lt: endSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
     })) {
       const slot = finalized.decodeKey(key);
       yield {
@@ -37,7 +40,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
   }
 
   // Non-finalized range of envelopes
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
@@ -46,7 +49,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Skip EMPTY blocks
         if (block.payloadStatus !== PayloadStatus.FULL) {
           continue;

package/src/node/nodejs.ts

@@ -221,7 +221,7 @@ export class BeaconNode {
 
     let executionEngineOpts = opts.executionEngine;
     if (opts.executionEngine.mode === "mock") {
-      const eth1BlockHash =
+      const latestEth1BlockHash =
         isStatePostBellatrix(anchorState) && anchorState.isExecutionStateType
           ? isStatePostGloas(anchorState)
             ? toRootHex(anchorState.latestBlockHash)
@@ -230,7 +230,7 @@ export class BeaconNode {
       executionEngineOpts = {
         ...opts.executionEngine,
         genesisBlockHash: ZERO_HASH_HEX,
-        eth1BlockHash,
+        eth1BlockHash: opts.executionEngine.eth1BlockHash ?? latestEth1BlockHash,
         genesisTime: anchorState.genesisTime,
         config,
       };