@lodestar/beacon-node 1.51.0-dev.3ff306e42b → 1.51.0-dev.41b1b87394

package/src/network/peers/utils/prioritizePeers.ts

@@ -49,6 +49,8 @@ const syncnetsZero = BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT);
 
 type SubnetDiscvQuery = {subnet: SubnetID; toSlot: number; maxPeersToDiscover: number};
 
+export type RequestedSubnetWithPeerCount = RequestedSubnet & {peerCount: number};
+
 /**
  * A map of das custody group index to maxPeersToDiscover
  */
@@ -145,8 +147,8 @@ export function prioritizePeers(
     samplingGroups: CustodyIndex[] | null;
     score: number;
   }[],
-  activeAttnets: RequestedSubnet[],
-  activeSyncnets: RequestedSubnet[],
+  activeAttnets: RequestedSubnetWithPeerCount[],
+  activeSyncnets: RequestedSubnetWithPeerCount[],
   samplingGroups: CustodyIndex[] | undefined,
   opts: PrioritizePeersOpts,
   config: ChainConfig,
@@ -231,8 +233,8 @@ export function prioritizePeers(
  */
 function requestSubnetPeers(
   connectedPeers: PeerInfo[],
-  activeAttnets: RequestedSubnet[],
-  activeSyncnets: RequestedSubnet[],
+  activeAttnets: RequestedSubnetWithPeerCount[],
+  activeSyncnets: RequestedSubnetWithPeerCount[],
   ourSamplingGroups: CustodyIndex[] | undefined,
   opts: PrioritizePeersOpts,
   config: ChainConfig,
@@ -252,54 +254,44 @@ function requestSubnetPeers(
 
   // attnets, do we need queries for more peers
   if (activeAttnets.length > 0) {
-    /** Map of peers per subnet, peer may be in multiple arrays */
-    const peersPerSubnet = new Map<number, number>();
-
     for (const peer of connectedPeers) {
       const trueBitIndices = peer.attnetsTrueBitIndices;
       let dutyCount = 0;
       for (const {subnet} of activeAttnets) {
         if (trueBitIndices.includes(subnet)) {
           dutyCount += 1;
-          peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0));
         }
       }
       dutiesByPeer.set(peer, dutyCount);
     }
 
-    for (const {subnet, toSlot} of activeAttnets) {
-      const peersInSubnet = peersPerSubnet.get(subnet) ?? 0;
-      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.attnets}, peersInSubnet);
-      if (peersInSubnet < targetSubnetPeers) {
+    for (const {subnet, toSlot, peerCount} of activeAttnets) {
+      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.attnets}, peerCount);
+      if (peerCount < targetSubnetPeers) {
         // We need more peers
-        attnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peersInSubnet});
+        attnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peerCount});
       }
     }
   }
 
   // syncnets, do we need queries for more peers
   if (activeSyncnets.length > 0) {
-    /** Map of peers per subnet, peer may be in multiple arrays */
-    const peersPerSubnet = new Map<number, number>();
-
     for (const peer of connectedPeers) {
       const trueBitIndices = peer.syncnetsTrueBitIndices;
       let dutyCount = dutiesByPeer.get(peer) ?? 0;
       for (const {subnet} of activeSyncnets) {
         if (trueBitIndices.includes(subnet)) {
           dutyCount += 1;
-          peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0));
         }
       }
       dutiesByPeer.set(peer, dutyCount);
     }
 
-    for (const {subnet, toSlot} of activeSyncnets) {
-      const peersInSubnet = peersPerSubnet.get(subnet) ?? 0;
-      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.syncnets}, peersInSubnet);
-      if (peersInSubnet < targetSubnetPeers) {
+    for (const {subnet, toSlot, peerCount} of activeSyncnets) {
+      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.syncnets}, peerCount);
+      if (peerCount < targetSubnetPeers) {
         // We need more peers
-        syncnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peersInSubnet});
+        syncnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peerCount});
       }
     }
   }

package/src/network/processor/gossipHandlers.ts

@@ -66,7 +66,10 @@ import {
   validateGossipGloasDataColumnSidecar,
 } from "../../chain/validation/dataColumnSidecar.js";
 import {validateGossipExecutionPayloadBid} from "../../chain/validation/executionPayloadBid.js";
-import {validateGossipExecutionPayloadEnvelope} from "../../chain/validation/executionPayloadEnvelope.js";
+import {
+  validateGossipExecutionPayloadEnvelope,
+  validateGossipExecutionPayloadEnvelopeSerialized,
+} from "../../chain/validation/executionPayloadEnvelope.js";
 import {InclusionListValidationResult, validateGossipInclusionList} from "../../chain/validation/inclusionList.js";
 import {
   AggregateAndProofValidationResult,
@@ -1276,6 +1279,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload>) => {
       const {serializedData} = gossipData;
+      validateGossipExecutionPayloadEnvelopeSerialized(chain, serializedData);
       const signedEnvelope = sszDeserialize(topic, serializedData);
       const envelope = signedEnvelope.message;
 
@@ -1328,18 +1332,22 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         });
       }
 
+      // Add the envelope in the same tick as validation, otherwise a by-root download that completes
+      // before the next event loop sees no envelope, validates it again and adds it first
+      if (!payloadInput.hasPayloadEnvelope()) {
+        chain.serializedCache.set(signedEnvelope, serializedData);
+
+        payloadInput.addPayloadEnvelope({
+          envelope: signedEnvelope,
+          source: PayloadEnvelopeInputSource.gossip,
+          seenTimestampSec,
+          peerIdStr,
+        });
+      }
+
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.serializedCache.set(signedEnvelope, serializedData);
-
-          payloadInput.addPayloadEnvelope({
-            envelope: signedEnvelope,
-            source: PayloadEnvelopeInputSource.gossip,
-            seenTimestampSec,
-            peerIdStr,
-          });
-
           scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
 
           chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
@@ -1349,7 +1357,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             blockRoot: blockRootHex,
           });
 
-          chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
+          // Signature is verified above only for this envelope, one added by another source is verified at import
+          const validSignature = payloadInput.getPayloadEnvelope() === signedEnvelope;
+          chain.processExecutionPayload(payloadInput, {validSignature}).catch((e) => {
             // Adjust verbosity based on error type
             let logLevel: LogLevel;
 
@@ -1367,11 +1377,14 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                   // Errors might indicate an issue with our node or the connected EL client
                   logLevel = LogLevel.error;
                   break;
-                // INVALID_SIGNATURE should not happen, signature is verified during gossip validation
+                // INVALID_SIGNATURE only happens for an envelope added by another source
                 case PayloadErrorCode.INVALID_SIGNATURE:
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
-                  core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  // Penalize only when the stored envelope is the one this peer sent, not one added by another source
+                  if (validSignature) {
+                    core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  }
                   // The builder may have signed another envelope that is valid, keeping this one would make
                   // by-root and range sync reuse it and never import the payload
                   chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);

package/src/network/processor/index.ts

@@ -244,6 +244,12 @@ type SearchUnknownRootTarget =
  * Such that enough work is processed to fill either one of the queue.
  */
 export class NetworkProcessor {
+  private stopped = false;
+  /**
+   * Replaced by dropAllJobs(). Loops that yield while re-queueing awaiting messages compare it after each yield
+   * and stop, so no message is queued after a drop.
+   */
+  private jobGeneration = Symbol();
   private readonly chain: IBeaconChain;
   private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
@@ -335,16 +341,27 @@ export class NetworkProcessor {
   }
 
   async stop(): Promise<void> {
+    if (this.stopped) return;
+    this.stopped = true;
     this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
-    this.chain.emitter.off(ClockEvent.slot, this.onClockSlot);
+    this.chain.clock.off(ClockEvent.slot, this.onClockSlot);
+    this.dropAllJobs();
   }
 
   dropAllJobs(): void {
+    this.jobGeneration = Symbol();
     for (const topic of executeGossipWorkOrder) {
       this.gossipQueues[topic].clear();
     }
+    this.awaitingMessagesByBlockRoot.clear();
+    this.awaitingBlockMessageCount = 0;
+    this.awaitingMessagesByPayloadBlockRoot.clear();
+    this.awaitingPayloadMessageCount = 0;
+    this.searchedRootsBySlot.clear();
+    this.bufferedRootsBySlot.clear();
+    this.fatalPeers.clear();
   }
 
   dumpGossipQueue(topic: GossipType): PendingGossipsubMessage[] {
@@ -364,7 +381,7 @@ export class NetworkProcessor {
    * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (this.chain.seenBlock(root)) {
+    if (this.stopped || this.chain.seenBlock(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -398,7 +415,7 @@ export class NetworkProcessor {
     peer?: PeerIdStr,
     slotIsPayloadSlot = false
   ): void {
-    if (this.chain.seenPayloadEnvelope(root)) {
+    if (this.stopped || this.chain.seenPayloadEnvelope(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -801,6 +818,7 @@ export class NetworkProcessor {
       this.awaitingBlockMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -819,6 +837,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -843,6 +862,7 @@ export class NetworkProcessor {
       this.awaitingPayloadMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const messages of messagesByTopic.values()) {
@@ -858,6 +878,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -937,6 +958,7 @@ export class NetworkProcessor {
   };
 
   private executeWork(): void {
+    if (this.stopped) return;
     // TODO: Maybe de-bounce by timing the last time executeWork was run
 
     this.metrics?.networkProcessor.executeWorkCalls.inc();

package/src/node/nodejs.ts

@@ -20,7 +20,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
-import {sleep, toRootHex} from "@lodestar/utils";
+import {defer, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
 import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
@@ -117,6 +117,7 @@ export class BeaconNode {
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
+  private closePromise: Promise<void> | undefined;
 
   constructor({
     opts,
@@ -393,23 +394,40 @@ export class BeaconNode {
   /**
    * Stop beacon node and its sub-components.
    */
-  async close(): Promise<void> {
-    if (this.status === BeaconNodeStatus.started) {
-      this.status = BeaconNodeStatus.closing;
-      this.sync.close();
-      if (this.restApi) await this.restApi.close();
-      await this.network.close();
-      if (this.metricsServer) await this.metricsServer.close();
-      if (this.monitoring) await this.monitoring.close();
-      await this.chain.persistToDisk();
-      await this.chain.close();
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    this.status = BeaconNodeStatus.closing;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const cleanup of [
+      () => this.sync.close(),
+      () => this.restApi?.close(),
+      () => this.network.close(),
+      () => this.metricsServer?.close(),
+      () => this.monitoring?.close(),
+      () => this.chain.persistToDisk(),
+      () => this.chain.close(),
       // Abort signal last: close() calls above clear intervals/timeouts so no new
       // operations get scheduled. If we aborted first, a still-pending interval could
       // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown.
-      if (this.controller) this.controller.abort();
-      await sleep(DELAY_BEFORE_CLOSING_DB_MS);
-      await this.db.close();
-      this.status = BeaconNodeStatus.closed;
+      () => this.controller?.abort(),
+      () => sleep(DELAY_BEFORE_CLOSING_DB_MS),
+      () => this.db.close(),
+    ]) {
+      try {
+        await cleanup();
+      } catch (error) {
+        errors.push(error);
+      }
     }
+    this.status = BeaconNodeStatus.closed;
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) throw new AggregateError(errors, "Beacon node cleanup failed");
   }
 }

package/src/util/sszBytes.ts

@@ -652,6 +652,46 @@ export function getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data: U
   return "0x" + blockRootBuf.toString("hex");
 }
 
+// transactions offset within ExecutionPayload fixed portion, see SLOT_NUMBER_OFFSET_IN_EXECUTION_PAYLOAD:
+// everything up to and including blockHash(32) = 504, followed by withdrawals_offset(4)
+const PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  ENVELOPE_START_IN_SIGNED + EXECUTION_PAYLOAD_ENVELOPE_FIXED_SIZE; // 180
+const TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 504; // 684
+const WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 4; // 688
+
+/**
+ * Transaction count of the payload, read from the SSZ offsets without deserializing the envelope.
+ * Returns null on malformed offsets, deserialization reports the exact error.
+ */
+export function getTransactionCountFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): number | null {
+  if (data.length < WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 4) {
+    return null;
+  }
+
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const transactionsOffset = view.getUint32(TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, true);
+  const withdrawalsOffset = view.getUint32(WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, true);
+  if (transactionsOffset > withdrawalsOffset) {
+    return null;
+  }
+  if (transactionsOffset === withdrawalsOffset) {
+    return 0;
+  }
+
+  const transactionsStart = PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + transactionsOffset;
+  if (transactionsStart + 4 > data.length) {
+    return null;
+  }
+  // first offset of a non-empty list is the size of its offset table, 4 bytes per element
+  const firstTransactionOffset = view.getUint32(transactionsStart, true);
+  if (firstTransactionOffset % 4 !== 0 || firstTransactionOffset > withdrawalsOffset - transactionsOffset) {
+    return null;
+  }
+  return firstTransactionOffset / 4;
+}
+
 /**
  * BeaconState of all forks (up until Electra, check with new forks)
  * class BeaconState(Container):