@lodestar/beacon-node 1.51.0-dev.ab56d90f0c → 1.51.0-dev.ab8f3f6381

package/src/network/processor/gossipHandlers.ts

@@ -24,6 +24,7 @@ import {
   deneb,
   fulu,
   gloas,
+  isGloasBeaconBlock,
   isGloasDataColumnSidecar,
   ssz,
   sszTypesFor,
@@ -65,7 +66,10 @@ import {
   validateGossipGloasDataColumnSidecar,
 } from "../../chain/validation/dataColumnSidecar.js";
 import {validateGossipExecutionPayloadBid} from "../../chain/validation/executionPayloadBid.js";
-import {validateGossipExecutionPayloadEnvelope} from "../../chain/validation/executionPayloadEnvelope.js";
+import {
+  validateGossipExecutionPayloadEnvelope,
+  validateGossipExecutionPayloadEnvelopeSerialized,
+} from "../../chain/validation/executionPayloadEnvelope.js";
 import {InclusionListValidationResult, validateGossipInclusionList} from "../../chain/validation/inclusionList.js";
 import {
   AggregateAndProofValidationResult,
@@ -150,6 +154,31 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
  */
 function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHandlerOpts): SequentialGossipHandlers {
   const {chain, config, metrics, logger, core} = modules;
+  const incompletePayloadWaits = new WeakSet<PayloadEnvelopeInput>();
+
+  function scheduleIncompletePayloadEnvelope(payloadInput: PayloadEnvelopeInput, peer: string): void {
+    if (payloadInput.isComplete() || incompletePayloadWaits.has(payloadInput)) {
+      return;
+    }
+    incompletePayloadWaits.add(payloadInput);
+
+    const fork = config.getForkName(payloadInput.slot);
+    const cutoffTimeMs = getCutoffTimeMs(chain, payloadInput.slot, config.getPayloadDueMs(fork));
+    payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch(() => {
+      if (
+        payloadInput.isComplete() ||
+        chain.seenPayloadEnvelopeInputCache.get(payloadInput.blockRootHex) !== payloadInput
+      ) {
+        return;
+      }
+      chain.logger.debug("Payload envelope incomplete at payload deadline", payloadInput.getLogMeta());
+      chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
+        payloadInput,
+        peer,
+        source: BlockInputSource.gossip,
+      });
+    });
+  }
 
   /**
    * Add a gossip block to the seen caches. The proposer signature of the block MUST be verified.
@@ -679,10 +708,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
           // Immediately attempt fetch of data columns from execution engine as the bid contains kzg commitments
           // which is all the information we need so there is no reason to delay until execution payload arrives
-          // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
-          // (for example later in the slot). Do not couple retries to incoming gossip columns.
-          // Columns fetched here feed payloadInput.addColumn, which resolves waitForAllData for any
-          // in-flight importExecutionPayload. No processExecutionPayload trigger needed from this path.
+          // Columns fetched here feed payloadInput.addColumn, which resolves the waitForAllData of a
+          // pending payload import. No processExecutionPayload trigger needed from this path.
           chain.getBlobsTracker.triggerGetBlobs(payloadInput);
         }
       })
@@ -692,11 +719,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
         if (e instanceof BlockError) {
           switch (e.type.code) {
-            case BlockErrorCode.DATA_UNAVAILABLE: {
-              // Error is quite frequent and not critical
+            case BlockErrorCode.DATA_UNAVAILABLE:
+              // Data did not arrive in time, BlockInputSync keeps pulling it and retries the import
               logLevel = LogLevel.debug;
               break;
-            }
             // ALREADY_KNOWN should not happen with ignoreIfKnown=true above
             // PARENT_BLOCK_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above
             case BlockErrorCode.ALREADY_KNOWN:
@@ -764,7 +790,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           // roots per proposer and slot, which bounds full imports over gossip to one alternate
           chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root) &&
           // A sibling with an unknown parent cannot be imported, sync fetches it if its branch becomes relevant
-          chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null
+          chain.forkChoice.getBlockHexDefaultStatus(toRootHex(signedBlock.message.parentRoot)) !== null &&
+          (!isGloasBeaconBlock(signedBlock.message) ||
+            chain.forkChoice.getBlockHexAndBlockHash(
+              toRootHex(signedBlock.message.parentRoot),
+              toRootHex(signedBlock.message.body.signedExecutionPayloadBid.message.parentBlockHash)
+            ) !== null)
         ) {
           const blockInput = cacheSignatureVerifiedBlock(
             signedBlock,
@@ -899,28 +930,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             }
 
             // NOTE: we do NOT call chain.processExecutionPayload here. That is triggered only by
-            // envelope arrival (gossip or API). An in-flight importExecutionPayload is awaiting
+            // envelope arrival (gossip or API). A pending payload import is awaiting
             // payloadInput.waitForAllData(); addColumn above will resolve it once hasAllData flips.
 
-            if (!payloadInput.isComplete()) {
-              const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
-              // do not await here to not delay gossip validation
-              payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch((_e) => {
-                chain.logger.debug(
-                  "Waited for envelope and data after receiving gossip column. Cut-off reached so emitting incompletePayloadEnvelope",
-                  {
-                    dataColumnIndex: index,
-                    ...payloadInputMeta,
-                  }
-                );
-                // TODO GLOAS: UnknownBlockSync to handle this event
-                chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
-                  payloadInput,
-                  peer: peerIdStr,
-                  source: BlockInputSource.gossip,
-                });
-              });
-            }
+            scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
           } catch (e) {
             logger.debug(
               "Error handling gossip data column",
@@ -1266,6 +1279,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload>) => {
       const {serializedData} = gossipData;
+      validateGossipExecutionPayloadEnvelopeSerialized(chain, serializedData);
       const signedEnvelope = sszDeserialize(topic, serializedData);
       const envelope = signedEnvelope.message;
 
@@ -1318,17 +1332,23 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         });
       }
 
+      // Add the envelope in the same tick as validation, otherwise a by-root download that completes
+      // before the next event loop sees no envelope, validates it again and adds it first
+      if (!payloadInput.hasPayloadEnvelope()) {
+        chain.serializedCache.set(signedEnvelope, serializedData);
+
+        payloadInput.addPayloadEnvelope({
+          envelope: signedEnvelope,
+          source: PayloadEnvelopeInputSource.gossip,
+          seenTimestampSec,
+          peerIdStr,
+        });
+      }
+
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.serializedCache.set(signedEnvelope, serializedData);
-
-          payloadInput.addPayloadEnvelope({
-            envelope: signedEnvelope,
-            source: PayloadEnvelopeInputSource.gossip,
-            seenTimestampSec,
-            peerIdStr,
-          });
+          scheduleIncompletePayloadEnvelope(payloadInput, peerIdStr);
 
           chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
             slot,
@@ -1337,12 +1357,18 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             blockRoot: blockRootHex,
           });
 
-          chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
+          // Signature is verified above only for this envelope, one added by another source is verified at import
+          const validSignature = payloadInput.getPayloadEnvelope() === signedEnvelope;
+          chain.processExecutionPayload(payloadInput, {validSignature}).catch((e) => {
             // Adjust verbosity based on error type
             let logLevel: LogLevel;
 
             if (e instanceof PayloadError) {
               switch (e.type.code) {
+                case PayloadErrorCode.DATA_UNAVAILABLE:
+                  // Columns did not arrive in time, BlockInputSync keeps pulling them and retries the import
+                  logLevel = LogLevel.debug;
+                  break;
                 // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
                 // already verified the block is in fork choice
                 case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
@@ -1351,11 +1377,17 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                   // Errors might indicate an issue with our node or the connected EL client
                   logLevel = LogLevel.error;
                   break;
-                // INVALID_SIGNATURE should not happen, signature is verified during gossip validation
+                case PayloadErrorCode.BLOCK_EXECUTION_INVALID:
+                  logLevel = LogLevel.warn;
+                  break;
+                // INVALID_SIGNATURE only happens for an envelope added by another source
                 case PayloadErrorCode.INVALID_SIGNATURE:
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
-                  core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  // Penalize only when the stored envelope is the one this peer sent, not one added by another source
+                  if (validSignature) {
+                    core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  }
                   // The builder may have signed another envelope that is valid, keeping this one would make
                   // by-root and range sync reuse it and never import the payload
                   chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
@@ -1630,6 +1662,32 @@ function getBatchHandlers(modules: ValidatorFnsModules, options: GossipHandlerOp
               logger.debug("Error adding gossip unaggregated attestation to pool", {subnet}, e as Error);
             }
 
+            // Collected only for vote slots that an upcoming block of a connected validator can include
+            if (isForkPostElectra(fork) && chain.proposerTracker.shouldCollect(indexedAttestation.data.slot)) {
+              try {
+                const insertOutcome = chain.singleAttestationPool.add(
+                  attestation as SingleAttestation<ForkPostElectra>,
+                  attDataRootHex,
+                  validatorCommitteeIndex,
+                  committeeSize
+                );
+                metrics?.opPool.singleAttestationPool.gossipInsertOutcome.inc({insertOutcome});
+              } catch (e) {
+                logger.debug(
+                  "Error adding gossip single attestation to pool",
+                  {
+                    slot: indexedAttestation.data.slot,
+                    committeeIndex,
+                    validatorCommitteeIndex,
+                    validatorIndex: (attestation as SingleAttestation<ForkPostElectra>).attesterIndex,
+                    subnet,
+                    attDataRoot: attDataRootHex,
+                  },
+                  e as Error
+                );
+              }
+            }
+
             // Separate boundary: a pool insertion error above must not skip the fork-choice vote
             if (!options.dontSendGossipAttestationsToForkchoice) {
               try {

package/src/network/processor/index.ts

@@ -244,6 +244,12 @@ type SearchUnknownRootTarget =
  * Such that enough work is processed to fill either one of the queue.
  */
 export class NetworkProcessor {
+  private stopped = false;
+  /**
+   * Replaced by dropAllJobs(). Loops that yield while re-queueing awaiting messages compare it after each yield
+   * and stop, so no message is queued after a drop.
+   */
+  private jobGeneration = Symbol();
   private readonly chain: IBeaconChain;
   private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
@@ -335,16 +341,27 @@ export class NetworkProcessor {
   }
 
   async stop(): Promise<void> {
+    if (this.stopped) return;
+    this.stopped = true;
     this.events.off(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockProcessed);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadEnvelopeProcessed);
-    this.chain.emitter.off(ClockEvent.slot, this.onClockSlot);
+    this.chain.clock.off(ClockEvent.slot, this.onClockSlot);
+    this.dropAllJobs();
   }
 
   dropAllJobs(): void {
+    this.jobGeneration = Symbol();
     for (const topic of executeGossipWorkOrder) {
       this.gossipQueues[topic].clear();
     }
+    this.awaitingMessagesByBlockRoot.clear();
+    this.awaitingBlockMessageCount = 0;
+    this.awaitingMessagesByPayloadBlockRoot.clear();
+    this.awaitingPayloadMessageCount = 0;
+    this.searchedRootsBySlot.clear();
+    this.bufferedRootsBySlot.clear();
+    this.fatalPeers.clear();
   }
 
   dumpGossipQueue(topic: GossipType): PendingGossipsubMessage[] {
@@ -364,7 +381,7 @@ export class NetworkProcessor {
    * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (this.chain.seenBlock(root)) {
+    if (this.stopped || this.chain.seenBlock(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -398,7 +415,7 @@ export class NetworkProcessor {
     peer?: PeerIdStr,
     slotIsPayloadSlot = false
   ): void {
-    if (this.chain.seenPayloadEnvelope(root)) {
+    if (this.stopped || this.chain.seenPayloadEnvelope(root)) {
       return;
     }
     const entry = this.searchedRootsBySlot.getOrDefault(slot).getOrDefault(root);
@@ -801,6 +818,7 @@ export class NetworkProcessor {
       this.awaitingBlockMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -819,6 +837,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -843,6 +862,7 @@ export class NetworkProcessor {
       this.awaitingPayloadMessageCount -= total;
     }
 
+    const generation = this.jobGeneration;
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const messages of messagesByTopic.values()) {
@@ -858,6 +878,7 @@ export class NetworkProcessor {
         if (count === MAX_AWAITING_GOSSIP_OBJECTS_PER_TICK) {
           count = 0;
           await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
+          if (generation !== this.jobGeneration) return;
         }
       }
     }
@@ -937,6 +958,7 @@ export class NetworkProcessor {
   };
 
   private executeWork(): void {
+    if (this.stopped) return;
     // TODO: Maybe de-bounce by timing the last time executeWork was run
 
     this.metrics?.networkProcessor.executeWorkCalls.inc();

package/src/network/subnets/attnetsService.ts

@@ -20,6 +20,7 @@ const gossipType = GossipType.beacon_attestation;
 export enum SubnetSource {
   committee = "committee",
   longLived = "long_lived",
+  beforeProposal = "before_proposal",
 }
 
 // map of subnet to time to form stable mesh as seconds, null if not yet formed
@@ -40,7 +41,7 @@ export class AttnetsService implements IAttnetsService {
   /** Committee subnets - PeerManager must find peers for those */
   private committeeSubnets = new SubnetMap();
   /**
-   * All currently subscribed short-lived subnets, for attestation aggregation
+   * All currently subscribed short-lived subnets, for attestation aggregation and before local block proposals
    * This class will tell gossip to subscribe and un-subscribe
    * If a value exists for `SubscriptionId` it means that gossip subscription is active in network.gossip
    */
@@ -52,6 +53,11 @@ export class AttnetsService implements IAttnetsService {
    * Used to determine if we should process an attestation.
    */
   private aggregatorSlotSubnet = new MapDef<Slot, AggregatorDutyInfo>(() => new Map());
+  /**
+   * Block proposal slots of validators connected to this node.
+   * Used to subscribe to all subnets `slotsToSubscribeBeforeProposalDuty` slots before, see onSlot
+   */
+  private localProposalSlots = new Set<Slot>();
 
   constructor(
     private readonly config: BeaconConfig,
@@ -119,6 +125,22 @@ export class AttnetsService implements IAttnetsService {
     }
   }
 
+  /**
+   * Called from the main thread with upcoming block proposal slots of connected validators.
+   * Tracked here, the subscription happens in onSlot like aggregator duties, or right away for a proposal learned
+   * after its subscription slot (node restart, newly connected validator)
+   */
+  addLocalProposalSlots(slots: Slot[]): void {
+    const clockSlot = this.clock.currentSlot;
+    for (const proposalSlot of slots) {
+      this.localProposalSlots.add(proposalSlot);
+      // not from the slot before the proposal, its votes are published before a mesh could form
+      if (proposalSlot - this.opts.slotsToSubscribeBeforeProposalDuty <= clockSlot && clockSlot < proposalSlot - 1) {
+        this.subscribeAllSubnetsUntilProposal(proposalSlot);
+      }
+    }
+  }
+
   /**
    * Check if a subscription is still active before handling a gossip object
    */
@@ -158,7 +180,8 @@ export class AttnetsService implements IAttnetsService {
 
   /**
    * Run per slot.
-   * - Subscribe to gossip subnets 2 slots in advance
+   * - Subscribe to gossip subnets 2 slots in advance of aggregator duties
+   * - Subscribe to all subnets `slotsToSubscribeBeforeProposalDuty` slots in advance of local block proposals
    * - Unsubscribe from expired subnets
    * - Track time to stable mesh if not yet formed
    */
@@ -186,19 +209,76 @@ export class AttnetsService implements IAttnetsService {
         this.trackTimeToStableMesh(clockSlot, dutiedSlot, dutiedInfo);
       }
 
+      // Must run before unsubscribeExpiredCommitteeSubnets so that a subnet whose subscription expires in this slot
+      // gets its ttl extended instead of being unsubscribed and subscribed again
+      this.subscribeAllSubnetsBeforeProposal(clockSlot);
+
       this.unsubscribeExpiredCommitteeSubnets(clockSlot);
       this.pruneExpiredAggregator(clockSlot);
+      this.pruneExpiredLocalProposals(clockSlot);
     } catch (e) {
       this.logger.error("Error on AttnetsService.onSlot", {slot: clockSlot}, e as Error);
     }
   };
 
   private onHalfSlot = (clockSlot: Slot): void => {
-    for (const [dutiedSlot, dutiedInfo] of this.aggregatorSlotSubnet.entries()) {
-      this.trackTimeToStableMesh(clockSlot, dutiedSlot, dutiedInfo);
+    try {
+      for (const [dutiedSlot, dutiedInfo] of this.aggregatorSlotSubnet.entries()) {
+        this.trackTimeToStableMesh(clockSlot, dutiedSlot, dutiedInfo);
+      }
+
+      this.unsubscribeAllSubnetsBeforeProposal(clockSlot);
+    } catch (e) {
+      this.logger.error("Error on AttnetsService.onHalfSlot", {slot: clockSlot}, e as Error);
     }
   };
 
+  /**
+   * Subscribe to all subnets `slotsToSubscribeBeforeProposalDuty` slots before a block proposal of a connected
+   * validator, so that single attestations of those slots can be packed into the block. The subscription expires in
+   * the slot before the proposal, see unsubscribeAllSubnetsBeforeProposal
+   */
+  private subscribeAllSubnetsBeforeProposal(clockSlot: Slot): void {
+    for (const proposalSlot of this.localProposalSlots) {
+      if (proposalSlot === clockSlot + this.opts.slotsToSubscribeBeforeProposalDuty) {
+        this.subscribeAllSubnetsUntilProposal(proposalSlot);
+      }
+    }
+  }
+
+  private subscribeAllSubnetsUntilProposal(proposalSlot: Slot): void {
+    if (this.opts.slotsToSubscribeBeforeProposalDuty === 0 || this.opts.subscribeAllSubnets) {
+      return;
+    }
+
+    const subnets = Array.from({length: ATTESTATION_SUBNET_COUNT}, (_, subnet) => subnet);
+    // Same as aggregator duties: trigger gossip subscription first, then register the subscriptions.
+    // Do not add to committeeSubnets: it drives peer discovery, we only use the peers we already have
+    this.subscribeToSubnets(subnets, SubnetSource.beforeProposal);
+    for (const subnet of subnets) {
+      this.shortLivedSubscriptions.request({subnet, toSlot: proposalSlot - 1});
+    }
+  }
+
+  /**
+   * Unsubscribe from subnets of subscribeAllSubnetsBeforeProposal at half of the slot before the proposal instead of
+   * the proposal slot start, so the proposal slot does not carry their traffic. Single attestations are published at
+   * 1/3 of the slot so they have arrived by then.
+   * Short-lived subscriptions are either aggregator duties or these, an aggregator duty ends in its dutied slot and is
+   * kept here (aggregates are due later), it expires at the next slot start in onSlot as before. So only subnets of
+   * subscribeAllSubnetsBeforeProposal are unsubscribed here.
+   */
+  private unsubscribeAllSubnetsBeforeProposal(clockSlot: Slot): void {
+    const aggregatorSubnets = this.aggregatorSlotSubnet.get(clockSlot);
+    const expiring = this.shortLivedSubscriptions.getExpired(
+      clockSlot + 1,
+      (subnet) => aggregatorSubnets?.has(subnet) === true
+    );
+    if (expiring.length > 0) {
+      this.unsubscribeSubnets(expiring, clockSlot + 1, SubnetSource.beforeProposal);
+    }
+  }
+
   /**
    * Track time to form stable mesh if not yet formed
    */
@@ -291,8 +371,8 @@ export class AttnetsService implements IAttnetsService {
   }
 
   /**
-   * Unsubscribe to a committee subnet from subscribedCommitteeSubnets.
-   * If a random subnet is present, we do not unsubscribe from it.
+   * Unsubscribe from short-lived subnets whose subscription expired.
+   * If a long-lived subnet is present, we do not unsubscribe from it.
    */
   private unsubscribeExpiredCommitteeSubnets(slot: Slot): void {
     const expired = this.shortLivedSubscriptions.getExpired(slot);
@@ -313,6 +393,14 @@ export class AttnetsService implements IAttnetsService {
     }
   }
 
+  private pruneExpiredLocalProposals(currentSlot: Slot): void {
+    for (const proposalSlot of this.localProposalSlots) {
+      if (currentSlot > proposalSlot) {
+        this.localProposalSlots.delete(proposalSlot);
+      }
+    }
+  }
+
   /** Update ENR */
   private updateMetadata(): void {
     const subnets = ssz.phase0.AttestationSubnets.defaultValue();

package/src/network/subnets/interface.ts

@@ -25,6 +25,8 @@ export type SubnetsService = {
 
 export interface IAttnetsService extends SubnetsService {
   shouldProcess(subnet: SubnetID, slot: Slot): boolean;
+  /** Block proposal slots of validators connected to this node, see `slotsToSubscribeBeforeProposalDuty` */
+  addLocalProposalSlots(slots: Slot[]): void;
 }
 
 export type RandBetweenFn = (min: number, max: number) => number;
@@ -33,6 +35,11 @@ export type ShuffleFn = <T>(arr: T[]) => T[];
 export type SubnetsServiceOpts = {
   subscribeAllSubnets?: boolean;
   slotsToSubscribeBeforeAggregatorDuty: number;
+  /**
+   * Subscribe to all attestation subnets this many slots before a block proposal of a connected validator, so that
+   * its single attestations can be packed into the block. In `[0, SLOTS_PER_EPOCH]`, 0 disables
+   */
+  slotsToSubscribeBeforeProposalDuty: number;
 };
 
 export type SubnetsServiceTestOpts = {

package/src/node/nodejs.ts

@@ -1,23 +1,29 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {hasher} from "@chainsafe/persistent-merkle-tree";
+import {BeaconApiMethods} from "@lodestar/api/beacon/server";
+import {BeaconConfig} from "@lodestar/config";
+import type {LoggerNode} from "@lodestar/logger/node";
 import {
   init as initNativeMetrics,
   registerLocalValidator as registerNativeLocalValidator,
   scrapeMetrics as scrapeNativeMetrics,
   unregisterLocalValidator as unregisterNativeLocalValidator,
-} from "@chainsafe/lodestar-z/metrics";
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
-import {hasher} from "@chainsafe/persistent-merkle-tree";
-import {BeaconApiMethods} from "@lodestar/api/beacon/server";
-import {BeaconConfig} from "@lodestar/config";
-import type {LoggerNode} from "@lodestar/logger/node";
+} from "@lodestar/native/metrics";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
-import {sleep, toRootHex} from "@lodestar/utils";
+import {defer, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -111,6 +117,7 @@ export class BeaconNode {
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
+  private closePromise: Promise<void> | undefined;
 
   constructor({
     opts,
@@ -212,6 +219,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos
@@ -280,6 +298,7 @@ export class BeaconNode {
         : undefined,
       builderApiClientOpts: {
         timeout: opts.executionBuilder.timeout,
+        bidDeadline: opts.executionBuilder.bidDeadline,
         // Sent with all builder api requests, unless the node runs in private mode
         userAgent: opts.executionBuilder.userAgent,
       },
@@ -375,23 +394,40 @@ export class BeaconNode {
   /**
    * Stop beacon node and its sub-components.
    */
-  async close(): Promise<void> {
-    if (this.status === BeaconNodeStatus.started) {
-      this.status = BeaconNodeStatus.closing;
-      this.sync.close();
-      if (this.restApi) await this.restApi.close();
-      await this.network.close();
-      if (this.metricsServer) await this.metricsServer.close();
-      if (this.monitoring) await this.monitoring.close();
-      await this.chain.persistToDisk();
-      await this.chain.close();
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    this.status = BeaconNodeStatus.closing;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const cleanup of [
+      () => this.sync.close(),
+      () => this.restApi?.close(),
+      () => this.network.close(),
+      () => this.metricsServer?.close(),
+      () => this.monitoring?.close(),
+      () => this.chain.persistToDisk(),
+      () => this.chain.close(),
       // Abort signal last: close() calls above clear intervals/timeouts so no new
       // operations get scheduled. If we aborted first, a still-pending interval could
       // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown.
-      if (this.controller) this.controller.abort();
-      await sleep(DELAY_BEFORE_CLOSING_DB_MS);
-      await this.db.close();
-      this.status = BeaconNodeStatus.closed;
+      () => this.controller?.abort(),
+      () => sleep(DELAY_BEFORE_CLOSING_DB_MS),
+      () => this.db.close(),
+    ]) {
+      try {
+        await cleanup();
+      } catch (error) {
+        errors.push(error);
+      }
     }
+    this.status = BeaconNodeStatus.closed;
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) throw new AggregateError(errors, "Beacon node cleanup failed");
   }
 }