@lodestar/beacon-node 1.44.0-dev.ff43f013ea → 1.44.0-rc.0

package/src/network/reqresp/utils/dataColumnResponseValidation.ts

@@ -1,4 +1,5 @@
 import {LogData} from "@lodestar/logger";
+import {ForkSeq} from "@lodestar/params";
 import {RespStatus, ResponseError} from "@lodestar/reqresp";
 import {ColumnIndex, Slot} from "@lodestar/types";
 import {prettyBytes, prettyPrintIndices, toRootHex} from "@lodestar/utils";
@@ -38,6 +39,13 @@ export async function handleColumnSidecarUnavailability({
 
   chain.logger.debug("dataColumnSidecar requested unavailable", logData);
 
+  // Post-gloas, columns exist only for FULL blocks; a finalized block is FULL if its envelope was
+  // archived. Bid blobsCount is unreliable here since an EMPTY block's bid may still commit to blobs
+  if (blockRoot === undefined && chain.config.getForkSeq(slot) >= ForkSeq.gloas) {
+    const envelopeBytes = await db.executionPayloadEnvelopeArchive.getBinary(slot);
+    if (!envelopeBytes) return;
+  }
+
   const blockBytes = blockRoot ? await db.block.getBinary(blockRoot) : await db.blockArchive.getBinary(slot);
   if (!blockBytes) {
     chain.logger.verbose(
@@ -71,9 +79,19 @@ export function validateRequestedDataColumns(chain: IBeaconChain, requestedColum
     throw new ResponseError(RespStatus.INVALID_REQUEST, "dataColumnSidecar requested without column indices");
   }
 
-  const custodyColumns = chain.custodyConfig.custodyColumns;
-  const availableColumns = requestedColumns.filter((c) => custodyColumns.includes(c));
-  const missingColumns = requestedColumns.filter((c) => !custodyColumns.includes(c));
+  const {custodyColumns, custodyColumnsIndex} = chain.custodyConfig;
+  const availableColumns: ColumnIndex[] = [];
+  const missingColumns: ColumnIndex[] = [];
+  for (const c of requestedColumns) {
+    // `c` is peer-controlled and SSZ-deserialized as `uint64`, so it may exceed
+    // `NUMBER_OF_COLUMNS - 1`; `Uint8Array` returns `undefined` for OOB reads,
+    // and `undefined !== 0` would silently classify OOB indices as custodied.
+    if ((custodyColumnsIndex[c] ?? 0) !== 0) {
+      availableColumns.push(c);
+    } else {
+      missingColumns.push(c);
+    }
+  }
 
   if (missingColumns.length > 0) {
     chain.logger.verbose("Requested dataColumnSidecar for non-custody columns", {

package/src/sync/unknownBlock.ts

@@ -850,25 +850,28 @@ export class BlockInputSync {
     }
 
     const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
-    if (!payloadInput) {
-      if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
-        // Column commitments live on the block body, so an envelope-only entry has to pull the block first.
-        if (!this.pendingBlocks.has(rootHex)) {
-          this.addByRootHex(rootHex);
-        }
+    if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+      // Block not in fork choice yet. payloadInput may be seeded from the block body during download, so a
+      // non-null payloadInput does not imply the block is imported; defer regardless and pull the block first.
+      // onBlockImported re-triggers the search to resume this envelope.
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.addByRootHex(rootHex);
+      }
 
-        const pendingBlock = this.pendingBlocks.get(rootHex);
-        if (pendingBlock && this.network.getConnectedPeers().length > 0) {
-          await this.downloadBlock(pendingBlock);
-        }
-      } else {
-        this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
-          root: rootHex,
-        });
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && this.network.getConnectedPeers().length > 0) {
+        await this.downloadBlock(pendingBlock);
       }
       return;
     }
 
+    if (!payloadInput) {
+      this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+        root: rootHex,
+      });
+      return;
+    }
+
     if (!payloadInput.hasPayloadEnvelope()) {
       const validationResult = await wrapError(
         validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
@@ -1073,11 +1076,11 @@ export class BlockInputSync {
         }
 
         payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
-        if (!payloadInput) {
-          if (this.chain.forkChoice.hasBlockHex(rootHex)) {
-            throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
-          }
-          // Keep the validated envelope around, but wait for the block body before turning it into a full payload input.
+        if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+          // Block not in fork choice yet. Validating now would throw BLOCK_ROOT_UNKNOWN, so keep the downloaded
+          // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
+          // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
+          // imply the block is imported.
           return {
             status: PendingPayloadInputStatus.waitingForBlock,
             envelope,
@@ -1086,6 +1089,11 @@ export class BlockInputSync {
           };
         }
 
+        if (!payloadInput) {
+          // Block is in fork choice but no PayloadEnvelopeInput exists, should have been created during block import.
+          throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
+        }
+
         if (!payloadInput.hasPayloadEnvelope()) {
           await validateGossipExecutionPayloadEnvelope(this.chain, envelope);
         }

package/src/util/dataColumns.ts

@@ -470,15 +470,17 @@ export async function recoverDataColumnSidecars(
     return DataColumnReconstructionCode.SuccessLate;
   }
 
-  // Once the node obtains a column through reconstruction,
-  // the node MUST expose the new column as if it had received it over the network.
-  // If the node is subscribed to the subnet corresponding to the column,
-  // it MUST send the reconstructed DataColumnSidecar to its topic mesh neighbors.
-  // If instead the node is not subscribed to the corresponding subnet,
-  // it SHOULD still expose the availability of the DataColumnSidecar as part of the gossip emission process.
-  // After exposing the reconstructed DataColumnSidecar to the network,
-  // the node MAY delete the DataColumnSidecar if it is not part of the node's custody requirement.
-  const sidecarsToPublish = [];
+  // Per consensus-specs PR #4657, only publish reconstructed columns the node is
+  // subscribed to (custody + sampling). Eagerly cross-seeding non-subscribed
+  // columns floods the network with duplicates because the sender has no
+  // visibility into which peers already saw the message via the topic mesh.
+  // This matches the getBlobsV2 path in `getDataColumnSidecarsFromExecution` and
+  // aligns with Lighthouse/Prysm. Capture missing sampled indices before adding
+  // any reconstructed columns so they are not filtered out by the subsequent
+  // `addColumn` calls.
+  const missingSampledColumns = new Set(input.getMissingSampledColumnMeta().missing);
+  const sidecarsReconstructed: DataColumnSidecar[] = [];
+  const sidecarsToPublish: DataColumnSidecar[] = [];
   for (const columnSidecar of fullSidecars) {
     if (!input.hasColumn(columnSidecar.index)) {
       if (input instanceof PayloadEnvelopeInput) {
@@ -501,11 +503,14 @@ export async function recoverDataColumnSidecars(
           source: BlockInputSource.recovery,
         });
       }
-      sidecarsToPublish.push(columnSidecar);
+      sidecarsReconstructed.push(columnSidecar);
+      if (missingSampledColumns.has(columnSidecar.index)) {
+        sidecarsToPublish.push(columnSidecar);
+      }
     }
   }
-  metrics?.peerDas.reconstructedColumns.inc(sidecarsToPublish.length);
-  metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsToPublish.length);
+  metrics?.peerDas.reconstructedColumns.inc(sidecarsReconstructed.length);
+  metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsReconstructed.length);
   emitter.emit(ChainEvent.publishDataColumns, sidecarsToPublish);
   // TODO: Can we record dataColumns.sentPeersPerSubnet metric somehow
   return DataColumnReconstructionCode.SuccessResolved;