@lodestar/beacon-node 1.51.0-dev.ab56d90f0c → 1.51.0-dev.ab8f3f6381

package/src/network/network.ts

@@ -19,6 +19,7 @@ import {
   SignedAggregateAndProof,
   SignedBeaconBlock,
   SingleAttestation,
+  Slot,
   SlotRootHex,
   SubnetID,
   altair,
@@ -30,10 +31,11 @@ import {
   isGloasDataColumnSidecar,
   phase0,
 } from "@lodestar/types";
-import {prettyPrintIndices, sleep} from "@lodestar/utils";
+import {defer, prettyPrintIndices, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
 import {ChainEvent, IBeaconChain} from "../chain/index.js";
 import {computeSubnetForDataColumnSidecar} from "../chain/validation/dataColumnSidecar.js";
+import {GoodByeReasonCode} from "../constants/index.js";
 import {IBeaconDb} from "../db/interface.js";
 import {Metrics, RegistryMetricCreator} from "../metrics/index.js";
 import {IClock} from "../util/clock.js";
@@ -127,6 +129,7 @@ export class Network implements INetwork {
 
   private subscribedToCoreTopics = false;
   private connectedPeersSyncMeta = new Map<PeerIdStr, Omit<PeerSyncMeta, "peerId">>();
+  private closePromise: Promise<void> | undefined;
 
   constructor(modules: NetworkModules) {
     this.peerId = peerIdFromPrivateKey(modules.privateKey);
@@ -144,17 +147,14 @@ export class Network implements INetwork {
     this.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
     this.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
     this.chain.emitter.on(routes.events.EventType.head, this.onHead);
-    this.chain.emitter.on(routes.events.EventType.lightClientFinalityUpdate, ({data}) =>
-      this.onLightClientFinalityUpdate(data)
-    );
-    this.chain.emitter.on(routes.events.EventType.lightClientOptimisticUpdate, ({data}) =>
-      this.onLightClientOptimisticUpdate(data)
-    );
+    this.chain.emitter.on(routes.events.EventType.lightClientFinalityUpdate, this.onLightClientFinalityUpdate);
+    this.chain.emitter.on(routes.events.EventType.lightClientOptimisticUpdate, this.onLightClientOptimisticUpdate);
     this.chain.emitter.on(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.on(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.on(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
     this.chain.emitter.on(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.on(ChainEvent.updateStatus, this.onUpdateStatus);
+    this.chain.emitter.on(ChainEvent.localProposalSlots, this.onLocalProposalSlots);
   }
 
   static async init({
@@ -239,9 +239,16 @@ export class Network implements INetwork {
     return this.controller.signal.aborted;
   }
 
-  /** Destroy this instance. Can only be called once. */
-  async close(): Promise<void> {
-    if (this.closed) return;
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    this.controller.abort();
 
     this.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
     this.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
@@ -253,10 +260,13 @@ export class Network implements INetwork {
     this.chain.emitter.off(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
     this.chain.emitter.off(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.off(ChainEvent.updateStatus, this.onUpdateStatus);
-    await this.core.close();
-
-    // Used only for sleep() statements
-    this.controller.abort();
+    this.chain.emitter.off(ChainEvent.localProposalSlots, this.onLocalProposalSlots);
+    this.connectedPeersSyncMeta.clear();
+    this.subscribedToCoreTopics = false;
+    const results = await Promise.allSettled([this.networkProcessor.stop(), this.core.close()]);
+    for (const result of results) {
+      if (result.status === "rejected") throw result.reason;
+    }
     this.logger.debug("network core closed");
   }
 
@@ -278,6 +288,13 @@ export class Network implements INetwork {
     return this.core.prepareBeaconCommitteeSubnets(subscriptions);
   }
 
+  /** The core subscribes to all attestation subnets before these proposals, see `slotsToSubscribeBeforeProposalDuty` */
+  private onLocalProposalSlots = (proposalSlots: Slot[]): void => {
+    this.core.addLocalProposalSlots(proposalSlots).catch((e) => {
+      this.logger.debug("Error adding local proposal slots", {slots: proposalSlots.join(",")}, e as Error);
+    });
+  };
+
   async prepareSyncCommitteeSubnets(subscriptions: CommitteeSubscription[]): Promise<void> {
     return this.core.prepareSyncCommitteeSubnets(subscriptions);
   }
@@ -289,6 +306,10 @@ export class Network implements INetwork {
     return this.core.reStatusPeers(peers);
   }
 
+  async goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    return this.core.goodbyeAndDisconnect(peer, reason);
+  }
+
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
     this.networkProcessor.searchUnknownBlock(slotRoot, source, peer);
   }
@@ -328,6 +349,7 @@ export class Network implements INetwork {
   async subscribeGossipCoreTopics(): Promise<void> {
     if (!this.subscribedToCoreTopics) {
       await this.core.subscribeGossipCoreTopics();
+      if (this.closed) return;
       // Only mark subscribedToCoreTopics if worker resolved this call
       this.subscribedToCoreTopics = true;
     }
@@ -798,26 +820,32 @@ export class Network implements INetwork {
     return this.core.writeDiscv5HeapSnapshot(prefix, dirpath);
   }
 
-  private onLightClientFinalityUpdate = async (finalityUpdate: LightClientFinalityUpdate): Promise<void> => {
+  private onLightClientFinalityUpdate = async ({
+    data: finalityUpdate,
+  }: routes.events.EventData[routes.events.EventType.lightClientFinalityUpdate]): Promise<void> => {
     // TODO: Review is OK to remove if (this.hasAttachedSyncCommitteeMember())
 
     try {
       // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired
       // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee
       await this.waitForSyncMessageCutoff(finalityUpdate.signatureSlot);
+      if (this.closed) return;
       await this.publishLightClientFinalityUpdate(finalityUpdate);
     } catch (e) {
       this.logger.debug("Error on BeaconGossipHandler.onLightclientFinalityUpdate", {}, e as Error);
     }
   };
 
-  private onLightClientOptimisticUpdate = async (optimisticUpdate: LightClientOptimisticUpdate): Promise<void> => {
+  private onLightClientOptimisticUpdate = async ({
+    data: optimisticUpdate,
+  }: routes.events.EventData[routes.events.EventType.lightClientOptimisticUpdate]): Promise<void> => {
     // TODO: Review is OK to remove if (this.hasAttachedSyncCommitteeMember())
 
     try {
       // messages SHOULD be broadcast after SYNC_MESSAGE_DUE_BPS of slot has transpired
       // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#sync-committee
       await this.waitForSyncMessageCutoff(optimisticUpdate.signatureSlot);
+      if (this.closed) return;
       await this.publishLightClientOptimisticUpdate(optimisticUpdate);
     } catch (e) {
       this.logger.debug("Error on BeaconGossipHandler.onLightclientOptimisticUpdate", {}, e as Error);
@@ -854,8 +882,12 @@ export class Network implements INetwork {
     this.connectedPeersSyncMeta.delete(data.peer);
   };
 
-  private onTargetGroupCountUpdated = (count: number): void => {
-    this.core.setTargetGroupCount(count);
+  private onTargetGroupCountUpdated = async (count: number): Promise<void> => {
+    try {
+      await this.core.setTargetGroupCount(count);
+    } catch (error) {
+      this.logger.error("Error updating network custody group count", {count}, error as Error);
+    }
   };
 
   private onPublishDataColumns = async (sidecars: DataColumnSidecar[]): Promise<void> => {
@@ -879,6 +911,10 @@ export class Network implements INetwork {
   };
 
   private onUpdateStatus = async (): Promise<void> => {
-    await this.core.updateStatus(this.chain.getStatus());
+    try {
+      await this.core.updateStatus(this.chain.getStatus());
+    } catch (error) {
+      this.logger.error("Error updating network status", {}, error as Error);
+    }
   };
 }

package/src/network/options.ts

@@ -25,6 +25,8 @@ export interface NetworkOptions
   mdns?: boolean;
   connectToDiscv5Bootnodes?: boolean;
   version?: string;
+  /** Include the version in libp2p Identify. Private mode takes precedence. */
+  advertiseVersion?: boolean;
   private?: boolean;
   useWorker?: boolean;
   maxYoungGenerationSizeMb?: number;
@@ -72,6 +74,8 @@ export const defaultNetworkOptions: NetworkOptions = {
   maxYoungGenerationSizeMb: 152,
   // subscribe 2 slots before aggregator dutied slot to get stable mesh peers as monitored on goerli
   slotsToSubscribeBeforeAggregatorDuty: 2,
+  // up to the max number of attestations per block
+  slotsToSubscribeBeforeProposalDuty: 8,
   // This will enable the light client server by default
   disableLightClientServer: false,
   quic: true,

package/src/network/peers/peerManager.ts

@@ -14,6 +14,7 @@ import {NetworkCoreMetrics} from "../core/metrics.js";
 import {LodestarDiscv5Opts} from "../discv5/types.js";
 import {INetworkEventBus, NetworkEvent, NetworkEventData} from "../events.js";
 import {Eth2Gossipsub} from "../gossip/gossipsub.js";
+import {GossipType} from "../gossip/interface.js";
 import {Libp2p} from "../interface.js";
 import {SubnetType} from "../metadata.js";
 import {NetworkConfig} from "../networkConfig.js";
@@ -577,6 +578,8 @@ export class PeerManager {
     this.lastStatus = status;
     this.metrics?.peerManager.starved.set(starved ? 1 : 0);
     const forkSeq = this.config.getForkSeq(this.clock.currentSlot);
+    const boundary = this.config.getForkBoundaryAtEpoch(this.clock.currentEpoch);
+    const connectedHealthyPeerIds = new Set(connectedHealthyPeers.map((peer) => peer.toString()));
 
     const prioritizeTimer = this.metrics?.peerManager.prioritizePeersDuration.startTimer();
     const {peersToDisconnect, peersToConnect, attnetQueries, syncnetQueries, custodyGroupQueries} = prioritizePeers(
@@ -594,8 +597,30 @@ export class PeerManager {
         };
       }),
       // Collect subnets which we need peers for in the current slot
-      this.attnetsService.getActiveSubnets(),
-      this.syncnetsService.getActiveSubnets(),
+      this.attnetsService.getActiveSubnets().map((request) => {
+        const subscribers = this.gossipsub.getSubscribersWithOutboundStream({
+          type: GossipType.beacon_attestation,
+          boundary,
+          subnet: request.subnet,
+        });
+        let peerCount = 0;
+        for (const peer of subscribers) {
+          if (connectedHealthyPeerIds.has(peer)) peerCount++;
+        }
+        return {...request, peerCount};
+      }),
+      this.syncnetsService.getActiveSubnets().map((request) => {
+        const subscribers = this.gossipsub.getSubscribersWithOutboundStream({
+          type: GossipType.sync_committee,
+          boundary,
+          subnet: request.subnet,
+        });
+        let peerCount = 0;
+        for (const peer of subscribers) {
+          if (connectedHealthyPeerIds.has(peer)) peerCount++;
+        }
+        return {...request, peerCount};
+      }),
       // ignore samplingGroups for pre-fulu forks
       forkSeq >= ForkSeq.fulu ? this.networkConfig.custodyConfig.sampleGroups : undefined,
       {
@@ -893,7 +918,7 @@ export class PeerManager {
     }
   }
 
-  private async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise<void> {
+  async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise<void> {
     const reason = GOODBYE_KNOWN_CODES[goodbye.toString()] || "";
     const peerIdStr = peer.toString();
     try {

package/src/network/peers/utils/assertPeerRelevance.ts

@@ -1,5 +1,6 @@
-import {ForkName, isForkPostFulu} from "@lodestar/params";
-import {ForkDigest, Root, Slot, Status, fulu, ssz} from "@lodestar/types";
+import {ForkName, GENESIS_EPOCH, isForkPostFulu} from "@lodestar/params";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {Epoch, ForkDigest, Root, Slot, Status, fulu, ssz} from "@lodestar/types";
 import {toHex, toRootHex} from "@lodestar/utils";
 
 // TODO: Why this value? (From Lighthouse)
@@ -9,6 +10,8 @@ export enum IrrelevantPeerCode {
   INCOMPATIBLE_FORKS = "IRRELEVANT_PEER_INCOMPATIBLE_FORKS",
   DIFFERENT_CLOCKS = "IRRELEVANT_PEER_DIFFERENT_CLOCKS",
   DIFFERENT_FINALIZED = "IRRELEVANT_PEER_DIFFERENT_FINALIZED",
+  FINALIZED_EPOCH_TOO_HIGH = "IRRELEVANT_PEER_FINALIZED_EPOCH_TOO_HIGH",
+  INVALID_FINALIZED_ROOT = "IRRELEVANT_PEER_INVALID_FINALIZED_ROOT",
   NO_EARLIEST_AVAILABLE_SLOT = "NO_EARLIEST_AVAILABLE_SLOT",
 }
 
@@ -16,6 +19,8 @@ type IrrelevantPeerType =
   | {code: IrrelevantPeerCode.INCOMPATIBLE_FORKS; ours: ForkDigest; theirs: ForkDigest}
   | {code: IrrelevantPeerCode.DIFFERENT_CLOCKS; slotDiff: number}
   | {code: IrrelevantPeerCode.NO_EARLIEST_AVAILABLE_SLOT}
+  | {code: IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH; finalizedEpoch: Epoch; maxFinalizedEpoch: Epoch}
+  | {code: IrrelevantPeerCode.INVALID_FINALIZED_ROOT}
   | {code: IrrelevantPeerCode.DIFFERENT_FINALIZED; expectedRoot: Root; remoteRoot: Root};
 
 /**
@@ -45,29 +50,35 @@ export function assertPeerRelevance(
     return {code: IrrelevantPeerCode.DIFFERENT_CLOCKS, slotDiff};
   }
 
-  // The remote's finalized epoch is less than or equal to ours, but the block root is
-  // different to the one in our chain. Therefore, the node is on a different chain and we
-  // should not communicate with them.
+  // A checkpoint can be finalized at the earliest at the start of epoch + 2
+  const maxFinalizedEpoch = Math.max(
+    GENESIS_EPOCH,
+    computeEpochAtSlot(Math.max(currentSlot, 0) + FUTURE_SLOT_TOLERANCE) - 2
+  );
+  if (remote.finalizedEpoch > maxFinalizedEpoch) {
+    return {
+      code: IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH,
+      finalizedEpoch: remote.finalizedEpoch,
+      maxFinalizedEpoch,
+    };
+  }
+
+  if (remote.finalizedEpoch > GENESIS_EPOCH && isZeroRoot(remote.finalizedRoot)) {
+    return {code: IrrelevantPeerCode.INVALID_FINALIZED_ROOT};
+  }
 
+  // The remote's finalized epoch is equal to ours, but the block root is different to the one in our chain.
+  // Therefore, the node is on a different chain and we should not communicate with them.
   if (
-    remote.finalizedEpoch <= local.finalizedEpoch &&
-    !isZeroRoot(remote.finalizedRoot) &&
-    !isZeroRoot(local.finalizedRoot)
+    remote.finalizedEpoch === local.finalizedEpoch &&
+    !isZeroRoot(local.finalizedRoot) &&
+    !ssz.Root.equals(remote.finalizedRoot, local.finalizedRoot)
   ) {
-    // NOTE: due to preferring to not access chain state here, we can't check the finalized root against our history.
-    // The impact of not doing check is low: peers that are behind us we can't confirm they are in the same chain as us.
-    // In the worst case they will attempt to sync from us, fail and disconnect. The ENR fork check should be sufficient
-    // to differentiate most peers in normal network conditions.
-    const remoteRoot = remote.finalizedRoot;
-    const expectedRoot = remote.finalizedEpoch === local.finalizedEpoch ? local.finalizedRoot : null;
-
-    if (expectedRoot !== null && !ssz.Root.equals(remoteRoot, expectedRoot)) {
-      return {
-        code: IrrelevantPeerCode.DIFFERENT_FINALIZED,
-        expectedRoot: expectedRoot, // forkChoice returns Tree BranchNode which the logger prints as {}
-        remoteRoot: remoteRoot,
-      };
-    }
+    return {
+      code: IrrelevantPeerCode.DIFFERENT_FINALIZED,
+      expectedRoot: local.finalizedRoot,
+      remoteRoot: remote.finalizedRoot,
+    };
   }
 
   if (isForkPostFulu(forkName) && (remote as fulu.Status).earliestAvailableSlot === undefined) {
@@ -76,7 +87,8 @@ export function assertPeerRelevance(
     };
   }
 
-  // Note: Accept request status finalized checkpoint in the future, we do not know if it is a true finalized root
+  // Note: for a behind finalized checkpoint, we validate it more on the main thread against our history,
+  // see BeaconSync.verifyPastFinalizedCheckpoint
   return null;
 }
 
@@ -93,6 +105,10 @@ export function renderIrrelevantPeerType(type: IrrelevantPeerType): string {
       return `DIFFERENT_CLOCKS slotDiff: ${type.slotDiff}`;
     case IrrelevantPeerCode.DIFFERENT_FINALIZED:
       return `DIFFERENT_FINALIZED root: ${toRootHex(type.remoteRoot)} expected: ${toRootHex(type.expectedRoot)}`;
+    case IrrelevantPeerCode.FINALIZED_EPOCH_TOO_HIGH:
+      return `FINALIZED_EPOCH_TOO_HIGH finalizedEpoch: ${type.finalizedEpoch} max: ${type.maxFinalizedEpoch}`;
+    case IrrelevantPeerCode.INVALID_FINALIZED_ROOT:
+      return "INVALID_FINALIZED_ROOT zero root for a non-genesis finalized epoch";
     case IrrelevantPeerCode.NO_EARLIEST_AVAILABLE_SLOT:
       return "No earliestAvailableSlot announced via peer Status";
   }

package/src/network/peers/utils/prioritizePeers.ts

@@ -49,6 +49,8 @@ const syncnetsZero = BitArray.fromBitLen(SYNC_COMMITTEE_SUBNET_COUNT);
 
 type SubnetDiscvQuery = {subnet: SubnetID; toSlot: number; maxPeersToDiscover: number};
 
+export type RequestedSubnetWithPeerCount = RequestedSubnet & {peerCount: number};
+
 /**
  * A map of das custody group index to maxPeersToDiscover
  */
@@ -145,8 +147,8 @@ export function prioritizePeers(
     samplingGroups: CustodyIndex[] | null;
     score: number;
   }[],
-  activeAttnets: RequestedSubnet[],
-  activeSyncnets: RequestedSubnet[],
+  activeAttnets: RequestedSubnetWithPeerCount[],
+  activeSyncnets: RequestedSubnetWithPeerCount[],
   samplingGroups: CustodyIndex[] | undefined,
   opts: PrioritizePeersOpts,
   config: ChainConfig,
@@ -231,8 +233,8 @@ export function prioritizePeers(
  */
 function requestSubnetPeers(
   connectedPeers: PeerInfo[],
-  activeAttnets: RequestedSubnet[],
-  activeSyncnets: RequestedSubnet[],
+  activeAttnets: RequestedSubnetWithPeerCount[],
+  activeSyncnets: RequestedSubnetWithPeerCount[],
   ourSamplingGroups: CustodyIndex[] | undefined,
   opts: PrioritizePeersOpts,
   config: ChainConfig,
@@ -252,54 +254,44 @@ function requestSubnetPeers(
 
   // attnets, do we need queries for more peers
   if (activeAttnets.length > 0) {
-    /** Map of peers per subnet, peer may be in multiple arrays */
-    const peersPerSubnet = new Map<number, number>();
-
     for (const peer of connectedPeers) {
       const trueBitIndices = peer.attnetsTrueBitIndices;
       let dutyCount = 0;
       for (const {subnet} of activeAttnets) {
         if (trueBitIndices.includes(subnet)) {
           dutyCount += 1;
-          peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0));
         }
       }
       dutiesByPeer.set(peer, dutyCount);
     }
 
-    for (const {subnet, toSlot} of activeAttnets) {
-      const peersInSubnet = peersPerSubnet.get(subnet) ?? 0;
-      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.attnets}, peersInSubnet);
-      if (peersInSubnet < targetSubnetPeers) {
+    for (const {subnet, toSlot, peerCount} of activeAttnets) {
+      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.attnets}, peerCount);
+      if (peerCount < targetSubnetPeers) {
         // We need more peers
-        attnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peersInSubnet});
+        attnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peerCount});
       }
     }
   }
 
   // syncnets, do we need queries for more peers
   if (activeSyncnets.length > 0) {
-    /** Map of peers per subnet, peer may be in multiple arrays */
-    const peersPerSubnet = new Map<number, number>();
-
     for (const peer of connectedPeers) {
       const trueBitIndices = peer.syncnetsTrueBitIndices;
       let dutyCount = dutiesByPeer.get(peer) ?? 0;
       for (const {subnet} of activeSyncnets) {
         if (trueBitIndices.includes(subnet)) {
           dutyCount += 1;
-          peersPerSubnet.set(subnet, 1 + (peersPerSubnet.get(subnet) ?? 0));
         }
       }
       dutiesByPeer.set(peer, dutyCount);
     }
 
-    for (const {subnet, toSlot} of activeSyncnets) {
-      const peersInSubnet = peersPerSubnet.get(subnet) ?? 0;
-      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.syncnets}, peersInSubnet);
-      if (peersInSubnet < targetSubnetPeers) {
+    for (const {subnet, toSlot, peerCount} of activeSyncnets) {
+      metrics?.peerManager.peersPerActiveSubnet.observe({type: SubnetType.syncnets}, peerCount);
+      if (peerCount < targetSubnetPeers) {
         // We need more peers
-        syncnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peersInSubnet});
+        syncnetQueries.push({subnet, toSlot, maxPeersToDiscover: targetSubnetPeers - peerCount});
       }
     }
   }

package/src/network/peers/utils/subnetMap.ts

@@ -66,11 +66,14 @@ export class SubnetMap {
     return subnets;
   }
 
-  /** Return subnetIds with a `toSlot` less than `currentSlot`. Also deletes expired entries */
-  getExpired(currentSlot: Slot): SubnetID[] {
+  /**
+   * Return subnetIds with a `toSlot` less than `currentSlot`. Also deletes expired entries
+   * @param keep expired subnets for which it returns true are neither returned nor deleted
+   */
+  getExpired(currentSlot: Slot, keep?: (subnet: SubnetID) => boolean): SubnetID[] {
     const subnetIds: SubnetID[] = [];
     for (const [subnet, toSlot] of this.subnets.entries()) {
-      if (toSlot < currentSlot) {
+      if (toSlot < currentSlot && keep?.(subnet) !== true) {
         subnetIds.push(subnet);
         this.subnets.delete(subnet);
       }