@lodestar/beacon-node 1.51.0-dev.83da04fef3 → 1.51.0-dev.a1662c7de7

package/src/network/peers/utils/enrSubnetsDeserialize.ts

@@ -17,10 +17,13 @@ export function deserializeEnrSubnets(bytes: Uint8Array, subnetCount: number): b
     return getUint8ByteToBitBooleanArray(bytes[0] ?? 0);
   }
 
-  let boolsArr: boolean[] = [];
   const byteCount = Math.ceil(subnetCount / 8);
+  const boolsArr = new Array<boolean>(byteCount * 8);
   for (let i = 0; i < byteCount; i++) {
-    boolsArr = boolsArr.concat(getUint8ByteToBitBooleanArray(bytes[i] ?? 0));
+    const bits = getUint8ByteToBitBooleanArray(bytes[i] ?? 0);
+    for (let j = 0; j < 8; j++) {
+      boolsArr[i * 8 + j] = bits[j];
+    }
   }
 
   return boolsArr;

package/src/network/processor/gossipHandlers.ts

@@ -38,7 +38,7 @@ import {
 } from "../../chain/blocks/blockInput/index.js";
 import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
-import {BlobSidecarValidation} from "../../chain/blocks/types.js";
+import {BlobSidecarValidation, InclusionListSource} from "../../chain/blocks/types.js";
 import {ChainEvent} from "../../chain/emitter.js";
 import {
   AttestationError,
@@ -66,6 +66,7 @@ import {
 } from "../../chain/validation/dataColumnSidecar.js";
 import {validateGossipExecutionPayloadBid} from "../../chain/validation/executionPayloadBid.js";
 import {validateGossipExecutionPayloadEnvelope} from "../../chain/validation/executionPayloadEnvelope.js";
+import {InclusionListValidationResult, validateGossipInclusionList} from "../../chain/validation/inclusionList.js";
 import {
   AggregateAndProofValidationResult,
   GossipAttestation,
@@ -1501,6 +1502,56 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         }
       });
     },
+
+    [GossipType.inclusion_list]: async ({
+      gossipData,
+      topic,
+      seenTimestampSec,
+    }: GossipHandlerParamGeneric<GossipType.inclusion_list>) => {
+      const {serializedData} = gossipData;
+      const signedInclusionList = sszDeserialize(topic, serializedData);
+      const {slot} = signedInclusionList.message;
+
+      metrics?.inclusionListsReceived.inc({source: InclusionListSource.gossip});
+      metrics?.inclusionListTransactionsReceived.inc(
+        {source: InclusionListSource.gossip},
+        signedInclusionList.message.transactions.length
+      );
+
+      const timer = metrics?.inclusionListsValidationTime.startTimer();
+      let validationResult: InclusionListValidationResult;
+      try {
+        validationResult = await validateGossipInclusionList(chain, signedInclusionList);
+      } finally {
+        timer?.({source: InclusionListSource.gossip});
+      }
+      const {committeeIndices, totalBytes} = validationResult;
+
+      // Handler - deferred to next event loop so the validation result propagates first
+      callInNextEventLoop(() => {
+        try {
+          metrics?.inclusionListsValid.inc({source: InclusionListSource.gossip});
+          metrics?.inclusionListsValidSize.inc(totalBytes);
+          // Timely only if received within its own slot before the deadline, so a list for the next slot that
+          // passed validation under clock disparity is untimely. Late lists are still stored, marked untimely
+          const secFromSlot = chain.clock.secFromSlot(slot, seenTimestampSec);
+          metrics?.inclusionListArrivalTime.observe(secFromSlot);
+          const isTimely =
+            secFromSlot >= 0 && secFromSlot * 1000 < config.getInclusionListDueMs(config.getForkName(slot));
+
+          const insertOutcome = chain.inclusionListStore.process(signedInclusionList, committeeIndices, isTimely);
+          metrics?.inclusionListStore.insertOutcome.inc({insertOutcome});
+          metrics?.inclusionListStore.size.set(chain.inclusionListStore.size);
+
+          chain.emitter.emit(routes.events.EventType.inclusionList, {
+            version: config.getForkName(slot),
+            data: signedInclusionList,
+          });
+        } catch (e) {
+          logger.debug("Error handling gossip inclusion list", {slot}, e as Error);
+        }
+      });
+    },
   };
 }
 

package/src/network/processor/gossipQueues/index.ts

@@ -88,6 +88,12 @@ const linearGossipQueueOpts: {
     type: QueueType.FIFO,
     dropOpts: {type: DropType.count, count: 1},
   },
+  [GossipType.inclusion_list]: {
+    // 16 committee members, two lists each under the first-or-second-message rule, 32 per slot
+    maxLength: 1024,
+    type: QueueType.FIFO,
+    dropOpts: {type: DropType.count, count: 1},
+  },
 };
 
 const indexedGossipQueueOpts: {

package/src/network/processor/gossipValidatorFn.ts

@@ -13,6 +13,7 @@ import {
   ExecutionPayloadEnvelopeErrorCode,
   GossipAction,
   GossipActionError,
+  InclusionListErrorCode,
   PayloadAttestationErrorCode,
   ProposerPreferencesErrorCode,
   ProposerSlashingErrorCode,
@@ -86,6 +87,10 @@ const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
     default: PeerAction.MidToleranceError,
     byCode: {[PayloadAttestationErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
   },
+  [GossipType.inclusion_list]: {
+    default: PeerAction.MidToleranceError,
+    byCode: {[InclusionListErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},
+  },
   [GossipType.execution_payload_bid]: {
     default: PeerAction.HighToleranceError,
     byCode: {[ExecutionPayloadBidErrorCode.INVALID_SIGNATURE]: PeerAction.Fatal},

package/src/network/processor/index.ts

@@ -63,7 +63,7 @@ export type NetworkProcessorOpts = GossipHandlerOpts & {
 /**
  * Keep up to 3 slot of unknown roots, so we don't always emit to UnknownBlock sync.
  */
-const MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE = 3;
+export const MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE = 3;
 
 /**
  * Max distinct roots we BUFFER messages for, per slot. Kept tight to avoid the OOM
@@ -140,6 +140,7 @@ const executeGossipWorkOrderObj: Record<GossipType, WorkOpts> = {
   [GossipType.payload_attestation_message]: {},
   [GossipType.execution_payload_bid]: {},
   [GossipType.proposer_preferences]: {},
+  [GossipType.inclusion_list]: {},
 };
 const executeGossipWorkOrder = Object.keys(executeGossipWorkOrderObj) as (keyof typeof executeGossipWorkOrderObj)[];
 
@@ -665,6 +666,7 @@ export class NetworkProcessor {
    *
    * For DOS protection, the message is ignored (reject metric incremented) when any cap is hit:
    * - MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS: global awaiting-block message count
+   * - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE: message slot already pruned by onClockSlot, so not bounded by the per-slot root budget
    * - MAX_BUFFERED_ROOTS_PER_SLOT: distinct roots we buffer messages for, per slot
    * - MAX_AWAITING_MESSAGES_PER_ROOT[topic]: messages buffered per (root, topic)
    */
@@ -675,6 +677,10 @@ export class NetworkProcessor {
       metric?.reject.inc({reason: ReprocessRejectReason.reached_limit, topic: topicType});
       return;
     }
+    if (slot <= this.chain.clock.currentSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE) {
+      metric?.reject.inc({reason: ReprocessRejectReason.expired, topic: topicType});
+      return;
+    }
     // per-(root, topic) message cap: e.g. <= NUMBER_OF_COLUMNS columns per root
     const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
     if (
@@ -701,6 +707,10 @@ export class NetworkProcessor {
       metric?.reject.inc({reason: ReprocessRejectReason.reached_limit, topic: topicType});
       return;
     }
+    if (slot <= this.chain.clock.currentSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE) {
+      metric?.reject.inc({reason: ReprocessRejectReason.expired, topic: topicType});
+      return;
+    }
     const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
     if (
       perRootCap !== undefined &&

package/src/network/reqresp/ReqRespBeaconNode.ts

@@ -311,6 +311,13 @@ export class ReqRespBeaconNode extends ReqResp {
       );
     }
 
+    if (ForkSeq[fork] >= ForkSeq.heze) {
+      protocolsAtFork.push([
+        protocols.InclusionListsByIndices(fork, this.config),
+        this.getHandler(ReqRespMethod.InclusionListsByIndices),
+      ]);
+    }
+
     return protocolsAtFork;
   }
 

package/src/network/reqresp/handlers/inclusionListsByIndices.ts

@@ -0,0 +1,44 @@
+import {PeerId} from "@libp2p/interface";
+import {ResponseOutgoing} from "@lodestar/reqresp";
+import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {heze, ssz} from "@lodestar/types";
+import {toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../../chain/index.js";
+import {prettyPrintPeerId} from "../../util.js";
+
+/**
+ * Only lists that passed gossip validation are stored and the store skips equivocators, so the spec's
+ * "SHOULD NOT respond with lists that fail gossip validation or are from equivocators" holds by construction.
+ */
+export async function* onInclusionListsByIndices(
+  requestBody: heze.InclusionListsByIndicesRequest,
+  chain: IBeaconChain,
+  peerId: PeerId,
+  peerClient: string
+): AsyncIterable<ResponseOutgoing> {
+  const {slot, dependentRoot, indices} = requestBody;
+
+  const minimumRequestSlot = Math.max(
+    chain.clock.currentSlot - chain.config.MIN_SLOTS_FOR_INCLUSION_LISTS_REQUESTS,
+    computeStartSlotAtEpoch(chain.config.HEZE_FORK_EPOCH)
+  );
+  if (slot < minimumRequestSlot) {
+    chain.logger.debug("Cannot serve InclusionListsByIndices: slot below minimum request slot", {
+      slot,
+      minimumRequestSlot,
+      peer: prettyPrintPeerId(peerId),
+      client: peerClient,
+    });
+    return;
+  }
+
+  const signedInclusionLists = chain.inclusionListStore.getByIndices(slot, toRootHex(dependentRoot), indices);
+
+  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot));
+  for (const signedInclusionList of signedInclusionLists.slice(0, chain.config.MAX_REQUEST_INCLUSION_LIST)) {
+    yield {
+      data: ssz.heze.SignedInclusionList.serialize(signedInclusionList),
+      boundary,
+    };
+  }
+}

package/src/network/reqresp/handlers/index.ts

@@ -19,6 +19,7 @@ import {onDataColumnSidecarsByRange} from "./dataColumnSidecarsByRange.js";
 import {onDataColumnSidecarsByRoot} from "./dataColumnSidecarsByRoot.js";
 import {onExecutionPayloadEnvelopesByRange} from "./executionPayloadEnvelopesByRange.js";
 import {onExecutionPayloadEnvelopesByRoot} from "./executionPayloadEnvelopesByRoot.js";
+import {onInclusionListsByIndices} from "./inclusionListsByIndices.js";
 import {onLightClientBootstrap} from "./lightClientBootstrap.js";
 import {onLightClientFinalityUpdate} from "./lightClientFinalityUpdate.js";
 import {onLightClientOptimisticUpdate} from "./lightClientOptimisticUpdate.js";
@@ -142,6 +143,10 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.LightClientFinalityUpdate]: () => onLightClientFinalityUpdate(chain),
     [ReqRespMethod.LightClientOptimisticUpdate]: () => onLightClientOptimisticUpdate(chain),
+    [ReqRespMethod.InclusionListsByIndices]: (req, peerId, peerClient) => {
+      const body = deserializeRequestBody(ssz.heze.InclusionListsByIndicesRequest, req.data);
+      return onInclusionListsByIndices(body, chain, peerId, peerClient);
+    },
   };
 
   return (method) => handlers[method];

package/src/network/reqresp/protocols.ts

@@ -113,6 +113,12 @@ export const ExecutionPayloadEnvelopesByRange = toProtocol({
   contextBytesType: ContextBytesType.ForkDigest,
 });
 
+export const InclusionListsByIndices = toProtocol({
+  method: ReqRespMethod.InclusionListsByIndices,
+  version: Version.V1,
+  contextBytesType: ContextBytesType.ForkDigest,
+});
+
 export const LightClientBootstrap = toProtocol({
   method: ReqRespMethod.LightClientBootstrap,
   version: Version.V1,

package/src/network/reqresp/rateLimit.ts

@@ -109,6 +109,10 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
     // Allow 2 per slot and a very safe bound until there's more testing of real usage.
     byPeer: {quota: 2, quotaTimeMs: config.getSlotDurationMs(fork)},
   },
+  [ReqRespMethod.InclusionListsByIndices]: {
+    // Peers only backfill what gossip missed, so a small per-slot allowance is enough
+    byPeer: {quota: 5, quotaTimeMs: config.getSlotDurationMs(fork)},
+  },
   [ReqRespMethod.LightClientOptimisticUpdate]: {
     // Optimistic updates should not be requested more than once per slot.
     // Allow 2 per slot and a very safe bound until there's more testing of real usage.

package/src/network/reqresp/types.ts

@@ -16,6 +16,7 @@ import {
   deneb,
   fulu,
   gloas,
+  heze,
   phase0,
   ssz,
   sszTypesFor,
@@ -53,6 +54,7 @@ export enum ReqRespMethod {
   LightClientUpdatesByRange = "light_client_updates_by_range",
   LightClientFinalityUpdate = "light_client_finality_update",
   LightClientOptimisticUpdate = "light_client_optimistic_update",
+  InclusionListsByIndices = "inclusion_lists_by_indices",
 }
 
 // To typesafe events to network
@@ -74,6 +76,7 @@ export type RequestBodyByMethod = {
   [ReqRespMethod.LightClientUpdatesByRange]: altair.LightClientUpdatesByRange;
   [ReqRespMethod.LightClientFinalityUpdate]: null;
   [ReqRespMethod.LightClientOptimisticUpdate]: null;
+  [ReqRespMethod.InclusionListsByIndices]: heze.InclusionListsByIndicesRequest;
 };
 
 type ResponseBodyByMethod = {
@@ -96,6 +99,7 @@ type ResponseBodyByMethod = {
   [ReqRespMethod.LightClientUpdatesByRange]: LightClientUpdate;
   [ReqRespMethod.LightClientFinalityUpdate]: LightClientFinalityUpdate;
   [ReqRespMethod.LightClientOptimisticUpdate]: LightClientOptimisticUpdate;
+  [ReqRespMethod.InclusionListsByIndices]: heze.SignedInclusionList;
 };
 
 /** Request SSZ type for each method and ForkName */
@@ -126,6 +130,7 @@ export const requestSszTypeByMethod: (
   [ReqRespMethod.LightClientUpdatesByRange]: ssz.altair.LightClientUpdatesByRange,
   [ReqRespMethod.LightClientFinalityUpdate]: null,
   [ReqRespMethod.LightClientOptimisticUpdate]: null,
+  [ReqRespMethod.InclusionListsByIndices]: ssz.heze.InclusionListsByIndicesRequest,
 });
 
 export type ResponseTypeGetter<T> = (fork: ForkName, version: number) => Type<T>;
@@ -158,6 +163,7 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter<
   [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: () => ssz.gloas.SignedExecutionPayloadEnvelope,
   [ReqRespMethod.LightClientOptimisticUpdate]: (fork) =>
     sszTypesFor(onlyPostAltairFork(fork)).LightClientOptimisticUpdate,
+  [ReqRespMethod.InclusionListsByIndices]: () => ssz.heze.SignedInclusionList,
 };
 
 function onlyPostAltairFork(fork: ForkName): ForkPostAltair {