@lodestar/beacon-node 1.42.0-dev.b2b4af3e21 → 1.42.0-dev.bc0be71fb0

package/src/db/repositories/dataColumnSidecarArchive.ts

@@ -16,8 +16,10 @@ const SLOT_BYTE_SIZE = 8;
  */
 export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
   constructor(config: ChainForkConfig, db: Db) {
-    const bucket = Bucket.fulu_dataColumnSidecarsArchive;
-    super(config, db, bucket, ssz.fulu.DataColumnSidecar, getBucketNameByValue(bucket));
+    const bucket = Bucket.allForks_dataColumnSidecarsArchive;
+    // Type won't be used since we select it dynamically based on fork
+    const type = ssz.fulu.DataColumnSidecar;
+    super(config, db, bucket, type, getBucketNameByValue(bucket));
   }
 
   /**

package/src/metrics/metrics/lodestar.ts

@@ -1746,6 +1746,40 @@ export function createLodestarMetrics(
       }),
     },
 
+    // some gossip messages need to wait for payload to be processed before they can be processed
+    awaitingPayloadGossipMessages: {
+      queue: register.gauge<{topic: GossipType}>({
+        name: "lodestar_awaiting_payload_gossip_messages_total",
+        help: "Total number of gossip messages waiting for payload to be processed",
+        labelNames: ["topic"],
+      }),
+      countPerSlot: register.gauge({
+        name: "lodestar_awaiting_payload_gossip_messages_per_slot_total",
+        help: "Total number of gossip messages waiting for payload to be processed per slot",
+      }),
+      resolve: register.gauge<{topic: GossipType}>({
+        name: "lodestar_awaiting_payload_gossip_messages_resolve_total",
+        help: "Total number of gossip messages are reprocessed",
+        labelNames: ["topic"],
+      }),
+      waitSecBeforeResolve: register.gauge<{topic: GossipType}>({
+        name: "lodestar_awaiting_payload_gossip_messages_wait_time_resolve_seconds",
+        help: "Time to wait for unknown payload in seconds",
+        labelNames: ["topic"],
+      }),
+      // having 2 labels here is not great for performance, however it's rarely happening and having the reason label is important for debugging
+      reject: register.gauge<{reason: ReprocessRejectReason; topic: GossipType}>({
+        name: "lodestar_awaiting_payload_gossip_messages_reject_total",
+        help: "Total number of gossip messages are rejected to reprocess",
+        labelNames: ["reason", "topic"],
+      }),
+      waitSecBeforeReject: register.gauge<{reason: ReprocessRejectReason; topic: GossipType}>({
+        name: "lodestar_awaiting_payload_gossip_messages_wait_time_reject_seconds",
+        help: "Time to wait for unknown payload before being rejected",
+        labelNames: ["reason", "topic"],
+      }),
+    },
+
     lightclientServer: {
       onSyncAggregate: register.gauge<{event: string}>({
         name: "lodestar_lightclient_server_on_sync_aggregate_event_total",

package/src/network/interface.ts

@@ -74,6 +74,7 @@ export interface INetwork extends INetworkCorePublic {
   shouldAggregate(subnet: SubnetID, slot: Slot): boolean;
   reStatusPeers(peers: PeerIdStr[]): Promise<void>;
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
+  searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
   // ReqResp
   sendBeaconBlocksByRange(peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest): Promise<SignedBeaconBlock[]>;
   sendBeaconBlocksByRoot(peerId: PeerIdStr, request: BeaconBlocksByRootRequest): Promise<SignedBeaconBlock[]>;
@@ -82,11 +83,11 @@ export interface INetwork extends INetworkCorePublic {
   sendDataColumnSidecarsByRange(
     peerId: PeerIdStr,
     request: fulu.DataColumnSidecarsByRangeRequest
-  ): Promise<fulu.DataColumnSidecar[]>;
+  ): Promise<DataColumnSidecar[]>;
   sendDataColumnSidecarsByRoot(
     peerId: PeerIdStr,
     request: DataColumnSidecarsByRootRequest
-  ): Promise<fulu.DataColumnSidecar[]>;
+  ): Promise<DataColumnSidecar[]>;
   sendExecutionPayloadEnvelopesByRange(
     peerId: PeerIdStr,
     request: gloas.ExecutionPayloadEnvelopesByRangeRequest

package/src/network/network.ts

@@ -11,7 +11,6 @@ import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {
   AttesterSlashing,
   DataColumnSidecar,
-  DataColumnSidecars,
   LightClientBootstrap,
   LightClientFinalityUpdate,
   LightClientOptimisticUpdate,
@@ -286,6 +285,10 @@ export class Network implements INetwork {
     this.networkProcessor.searchUnknownBlock(slotRoot, source, peer);
   }
 
+  searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
+    this.networkProcessor.searchUnknownEnvelope(slotRoot, source, peer);
+  }
+
   async reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): Promise<void> {
     return this.core.reportPeer(peer, action, actionName);
   }
@@ -621,7 +624,7 @@ export class Network implements INetwork {
   async sendDataColumnSidecarsByRange(
     peerId: PeerIdStr,
     request: fulu.DataColumnSidecarsByRangeRequest
-  ): Promise<fulu.DataColumnSidecar[]> {
+  ): Promise<DataColumnSidecar[]> {
     return collectMaxResponseTyped(
       this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRange, [Version.V1], request),
       request.count * request.columns.length,
@@ -632,7 +635,7 @@ export class Network implements INetwork {
   async sendDataColumnSidecarsByRoot(
     peerId: PeerIdStr,
     request: DataColumnSidecarsByRootRequest
-  ): Promise<fulu.DataColumnSidecar[]> {
+  ): Promise<DataColumnSidecar[]> {
     return collectMaxResponseTyped(
       this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRoot, [Version.V1], request),
       request.reduce((total, {columns}) => total + columns.length, 0),
@@ -812,7 +815,7 @@ export class Network implements INetwork {
     this.core.setTargetGroupCount(count);
   };
 
-  private onPublishDataColumns = (sidecars: DataColumnSidecars): Promise<number[]> => {
+  private onPublishDataColumns = (sidecars: DataColumnSidecar[]): Promise<number[]> => {
     return promiseAllMaybeAsync(sidecars.map((sidecar) => () => this.publishDataColumnSidecar(sidecar)));
   };
 

package/src/network/processor/extractSlotRootFns.ts

@@ -1,23 +1,25 @@
-import {ForkName, isForkPostGloas} from "@lodestar/params";
+import {ForkName, ForkSeq} from "@lodestar/params";
 import {SlotOptionalRoot, SlotRootHex} from "@lodestar/types";
 import {
   getBeaconBlockRootFromDataColumnSidecarSerialized,
-  getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized,
   getBlockRootFromBeaconAttestationSerialized,
+  getBlockRootFromPayloadAttestationMessageSerialized,
   getBlockRootFromSignedAggregateAndProofSerialized,
   getSlotFromBeaconAttestationSerialized,
   getSlotFromBlobSidecarSerialized,
   getSlotFromDataColumnSidecarSerialized,
   getSlotFromExecutionPayloadEnvelopeSerialized,
+  getSlotFromPayloadAttestationMessageSerialized,
   getSlotFromSignedAggregateAndProofSerialized,
   getSlotFromSignedBeaconBlockSerialized,
+  getSlotFromSignedExecutionPayloadBidSerialized,
 } from "../../util/sszBytes.js";
 import {GossipType} from "../gossip/index.js";
 import {ExtractSlotRootFns} from "./types.js";
 
 /**
  * Extract the slot and block root of a gossip message form serialized data.
- * Not applicable for all topics.
+ * Only do it for messages that have a slot and block root, and we want to await the block if the block root is not known.
  */
 export function createExtractBlockSlotRootFns(): ExtractSlotRootFns {
   return {
@@ -57,21 +59,45 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns {
     },
     [GossipType.data_column_sidecar]: (data: Uint8Array, fork: ForkName): SlotOptionalRoot | null => {
       const slot = getSlotFromDataColumnSidecarSerialized(data, fork);
+
       if (slot === null) {
         return null;
       }
 
-      const root = isForkPostGloas(fork) ? getBeaconBlockRootFromDataColumnSidecarSerialized(data) : null;
+      if (ForkSeq[fork] < ForkSeq.gloas) {
+        return {slot};
+      }
+
+      const root = getBeaconBlockRootFromDataColumnSidecarSerialized(data);
+      // null root means the message is invalid here and will be ignored in gossip handler later
+      // returning the slot here helps check the earliest permissable slot in the network processor
       return root !== null ? {slot, root} : {slot};
     },
-    [GossipType.execution_payload]: (data: Uint8Array): SlotRootHex | null => {
+    [GossipType.execution_payload]: (data: Uint8Array): SlotOptionalRoot | null => {
       const slot = getSlotFromExecutionPayloadEnvelopeSerialized(data);
-      const root = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data);
+      // Do not extract the root here; the network processor will extract it in the 2nd round to trigger block search without awaiting.
+      if (slot === null) {
+        return null;
+      }
+      return {slot};
+    },
+    [GossipType.payload_attestation_message]: (data: Uint8Array): SlotRootHex | null => {
+      const slot = getSlotFromPayloadAttestationMessageSerialized(data);
+      const root = getBlockRootFromPayloadAttestationMessageSerialized(data);
 
       if (slot === null || root === null) {
         return null;
       }
       return {slot, root};
     },
+    [GossipType.execution_payload_bid]: (data: Uint8Array): SlotOptionalRoot | null => {
+      const slot = getSlotFromSignedExecutionPayloadBidSerialized(data);
+
+      if (slot === null) {
+        return null;
+      }
+
+      return {slot};
+    },
   };
 }