@lodestar/beacon-node 1.44.0-dev.d730eae4b6 → 1.44.0-dev.de5436c9ff

package/src/chain/forkChoice/index.ts

@@ -2,6 +2,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {
   ExecutionStatus,
   ForkChoice,
+  ForkChoiceStateGetter,
   ForkChoiceStore,
   JustifiedBalancesGetter,
   PayloadStatus,
@@ -45,6 +46,7 @@ export function initializeForkChoice(
   isFinalizedState: boolean,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -56,6 +58,7 @@ export function initializeForkChoice(
         state,
         opts,
         justifiedBalancesGetter,
+        stateGetter,
         metrics,
         logger
       )
@@ -66,6 +69,7 @@ export function initializeForkChoice(
         state,
         opts,
         justifiedBalancesGetter,
+        stateGetter,
         metrics,
         logger
       );
@@ -81,6 +85,7 @@ export function initializeForkChoiceFromFinalizedState(
   state: IBeaconStateView,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -112,6 +117,7 @@ export function initializeForkChoiceFromFinalizedState(
       finalizedCheckpoint,
       justifiedBalances,
       justifiedBalancesGetter,
+      stateGetter,
       {
         onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
         onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
@@ -172,6 +178,7 @@ export function initializeForkChoiceFromUnfinalizedState(
   unfinalizedState: IBeaconStateView,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -203,6 +210,7 @@ export function initializeForkChoiceFromUnfinalizedState(
     finalizedCheckpoint,
     justifiedBalances,
     justifiedBalancesGetter,
+    stateGetter,
     {
       onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
       onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),

package/src/chain/options.ts

@@ -105,6 +105,7 @@ export const defaultChainOptions: IChainOptions = {
   proposerBoost: true,
   proposerBoostReorg: true,
   computeUnrealized: true,
+  fastConfirmation: false,
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
   serveHistoricalState: false,
   assertCorrectProgressiveBalances: false,

package/src/network/processor/index.ts

@@ -186,9 +186,11 @@ export class NetworkProcessor {
   // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
   private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingBlockMessageCount = 0;
   // we may not receive the payload for messages that require the FULL payload variant to be processed,
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingPayloadMessageCount = 0;
   private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
   private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
@@ -228,8 +230,8 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
-        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.awaitingBlockMessageCount);
+        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.awaitingPayloadMessageCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -497,7 +499,7 @@ export class NetworkProcessor {
         this.pushPendingGossipsubMessageToQueue(message);
         break;
       case PreprocessAction.AwaitBlock: {
-        if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+        if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
           // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           this.metrics?.awaitingBlockGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
@@ -509,10 +511,11 @@ export class NetworkProcessor {
         this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
         const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
         awaitingGossipsubMessages.add(message);
+        this.awaitingBlockMessageCount++;
         break;
       }
       case PreprocessAction.AwaitEnvelope: {
-        if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+        if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
           this.metrics?.awaitingPayloadGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
             topic: topicType,
@@ -525,6 +528,7 @@ export class NetworkProcessor {
           preprocessResult.root
         );
         awaitingPayloadGossipsubMessages.add(message);
+        this.awaitingPayloadMessageCount++;
         break;
       }
     }
@@ -548,6 +552,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+      this.awaitingBlockMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -567,8 +577,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByBlockRoot.delete(rootHex);
   };
 
   private onPayloadEnvelopeProcessed = async ({blockRoot: rootHex}: {blockRoot: RootHex}): Promise<void> => {
@@ -577,6 +585,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+      this.awaitingPayloadMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const message of waitingGossipsubMessages) {
@@ -593,8 +607,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
   };
 
   private onClockSlot = (clockSlot: Slot): void => {
@@ -618,7 +630,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+            this.awaitingBlockMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownBlocksBySlot.delete(slot);
@@ -641,7 +655,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+            this.awaitingPayloadMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownEnvelopesBySlot.delete(slot);
@@ -784,20 +800,4 @@ export class NetworkProcessor {
 
     return null;
   }
-
-  private get unknownBlockGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
-
-  private get unknownPayloadGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
 }

package/src/util/dataColumns.ts

@@ -470,15 +470,17 @@ export async function recoverDataColumnSidecars(
     return DataColumnReconstructionCode.SuccessLate;
   }
 
-  // Once the node obtains a column through reconstruction,
-  // the node MUST expose the new column as if it had received it over the network.
-  // If the node is subscribed to the subnet corresponding to the column,
-  // it MUST send the reconstructed DataColumnSidecar to its topic mesh neighbors.
-  // If instead the node is not subscribed to the corresponding subnet,
-  // it SHOULD still expose the availability of the DataColumnSidecar as part of the gossip emission process.
-  // After exposing the reconstructed DataColumnSidecar to the network,
-  // the node MAY delete the DataColumnSidecar if it is not part of the node's custody requirement.
-  const sidecarsToPublish = [];
+  // Per consensus-specs PR #4657, only publish reconstructed columns the node is
+  // subscribed to (custody + sampling). Eagerly cross-seeding non-subscribed
+  // columns floods the network with duplicates because the sender has no
+  // visibility into which peers already saw the message via the topic mesh.
+  // This matches the getBlobsV2 path in `getDataColumnSidecarsFromExecution` and
+  // aligns with Lighthouse/Prysm. Capture missing sampled indices before adding
+  // any reconstructed columns so they are not filtered out by the subsequent
+  // `addColumn` calls.
+  const missingSampledColumns = new Set(input.getMissingSampledColumnMeta().missing);
+  const sidecarsReconstructed: DataColumnSidecar[] = [];
+  const sidecarsToPublish: DataColumnSidecar[] = [];
   for (const columnSidecar of fullSidecars) {
     if (!input.hasColumn(columnSidecar.index)) {
       if (input instanceof PayloadEnvelopeInput) {
@@ -501,11 +503,14 @@ export async function recoverDataColumnSidecars(
           source: BlockInputSource.recovery,
         });
       }
-      sidecarsToPublish.push(columnSidecar);
+      sidecarsReconstructed.push(columnSidecar);
+      if (missingSampledColumns.has(columnSidecar.index)) {
+        sidecarsToPublish.push(columnSidecar);
+      }
     }
   }
-  metrics?.peerDas.reconstructedColumns.inc(sidecarsToPublish.length);
-  metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsToPublish.length);
+  metrics?.peerDas.reconstructedColumns.inc(sidecarsReconstructed.length);
+  metrics?.dataColumns.bySource.inc({source: BlockInputSource.recovery}, sidecarsReconstructed.length);
   emitter.emit(ChainEvent.publishDataColumns, sidecarsToPublish);
   // TODO: Can we record dataColumns.sentPeersPerSubnet metric somehow
   return DataColumnReconstructionCode.SuccessResolved;