@aztec-labs/p2p 6.0.0-nightly.20260829

package/dest/services/peer-manager/peer_manager.js

@@ -0,0 +1,1015 @@
+import { compactArray } from '@aztec-labs/foundation/collection';
+import { makeEthSignDigest, tryRecoverAddress } from '@aztec-labs/foundation/crypto/secp256k1-signer';
+import { Fr } from '@aztec-labs/foundation/curves/bn254';
+import { createLogger } from '@aztec-labs/foundation/log';
+import { bufferToHex } from '@aztec-labs/foundation/string';
+import { DateProvider } from '@aztec-labs/foundation/timer';
+import { peerIdFromString } from '@libp2p/peer-id';
+import { ENR } from '@nethermindeth/enr';
+import { inspect } from 'util';
+import { PeerEvent } from '../../types/index.js';
+import { ReqRespSubProtocol } from '../reqresp/interface.js';
+import { AuthRequest, AuthResponse } from '../reqresp/protocols/auth.js';
+import { GoodByeReason, prettyGoodbyeReason } from '../reqresp/protocols/goodbye.js';
+import { StatusMessage } from '../reqresp/protocols/status.js';
+import { ReqRespStatus } from '../reqresp/status.js';
+import { PeerManagerMetrics } from './metrics.js';
+import { PeerScoreState } from './peer_scoring.js';
+const MAX_DIAL_ATTEMPTS = 3;
+const MAX_CACHED_PEERS = 100;
+const MAX_CACHED_PEER_AGE_MS = 5 * 60 * 1000; // 5 minutes
+const DEFAULT_FAILED_PEER_BAN_TIME_MS = 5 * 60 * 1000; // 5 minutes timeout after failing MAX_DIAL_ATTEMPTS
+const GOODBYE_DIAL_TIMEOUT_MS = 1000;
+const FAILED_AUTH_HANDSHAKE_EXPIRY_MS = 60 * 60 * 1000; // 1 hour
+const ZERO_PEER_WARNING_THRESHOLD_HEARTBEATS = 3; // tolerate transient dips, such as at startup
+const ZERO_PEER_WARNING_REPEAT_INTERVAL_MS = 60 * 1000;
+export class PeerManager {
+    libP2PNode;
+    peerDiscoveryService;
+    config;
+    logger;
+    peerScoring;
+    reqresp;
+    worldStateSynchronizer;
+    protocolVersion;
+    epochCache;
+    dateProvider;
+    cachedPeers;
+    heartbeatCounter;
+    displayPeerCountsPeerHeartbeat;
+    timedOutPeers;
+    trustedPeers;
+    trustedPeersInitialized;
+    privatePeers;
+    privatePeersInitialized;
+    preferredPeers;
+    authenticatedPeerIdToValidatorAddress;
+    authenticatedValidatorAddressToPeerId;
+    peersToBeDisconnected;
+    failedAuthHandshakes;
+    validatorAddresses;
+    initializedPreferredPeers;
+    consecutiveZeroPeerHeartbeats;
+    zeroPeerWarningHeartbeats;
+    zeroPeerWarningIssued;
+    lastConnectedPeerAtMs;
+    metrics;
+    handlers;
+    constructor(libP2PNode, peerDiscoveryService, config, telemetryClient, logger = createLogger('p2p:peer-manager'), peerScoring, reqresp, worldStateSynchronizer, protocolVersion, epochCache, dateProvider = new DateProvider()){
+        this.libP2PNode = libP2PNode;
+        this.peerDiscoveryService = peerDiscoveryService;
+        this.config = config;
+        this.logger = logger;
+        this.peerScoring = peerScoring;
+        this.reqresp = reqresp;
+        this.worldStateSynchronizer = worldStateSynchronizer;
+        this.protocolVersion = protocolVersion;
+        this.epochCache = epochCache;
+        this.dateProvider = dateProvider;
+        this.cachedPeers = new Map();
+        this.heartbeatCounter = 0;
+        this.displayPeerCountsPeerHeartbeat = 0;
+        this.timedOutPeers = new Map();
+        this.trustedPeers = new Set();
+        this.trustedPeersInitialized = false;
+        this.privatePeers = new Set();
+        this.privatePeersInitialized = false;
+        this.preferredPeers = new Set();
+        this.authenticatedPeerIdToValidatorAddress = new Map();
+        this.authenticatedValidatorAddressToPeerId = new Map();
+        this.peersToBeDisconnected = new Set();
+        this.failedAuthHandshakes = new Map();
+        this.validatorAddresses = [];
+        this.initializedPreferredPeers = false;
+        this.consecutiveZeroPeerHeartbeats = 0;
+        this.zeroPeerWarningHeartbeats = 0;
+        this.zeroPeerWarningIssued = false;
+        if (this.config.p2pDisableStatusHandshake && this.config.p2pAllowOnlyValidators) {
+            throw new Error('Status handshake disabled but is required to allow only validators to connect.');
+        }
+        this.metrics = new PeerManagerMetrics(telemetryClient, 'PeerManager');
+        // Handle Discovered peers
+        this.handlers = {
+            handleConnectedPeerEvent: this.handleConnectedPeerEvent.bind(this),
+            handleDisconnectedPeerEvent: this.handleDisconnectedPeerEvent.bind(this),
+            handleDiscoveredPeer: (enr)=>this.handleDiscoveredPeer(enr).catch((e)=>this.logger.error('Error handling discovered peer', e))
+        };
+        // Handle new established connections
+        this.libP2PNode.addEventListener(PeerEvent.CONNECTED, this.handlers.handleConnectedPeerEvent);
+        // Handle lost connections
+        this.libP2PNode.addEventListener(PeerEvent.DISCONNECTED, this.handlers.handleDisconnectedPeerEvent);
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        this.peerDiscoveryService?.on(PeerEvent.DISCOVERED, this.handlers.handleDiscoveredPeer);
+        // Display peer counts every 60 seconds
+        this.displayPeerCountsPeerHeartbeat = Math.floor(60_000 / this.config.peerCheckIntervalMS);
+        this.zeroPeerWarningHeartbeats = Math.max(1, Math.floor(ZERO_PEER_WARNING_REPEAT_INTERVAL_MS / this.config.peerCheckIntervalMS));
+    }
+    /**
+   * Initializes the trusted peers.
+   *
+   * This function is called when the peer manager is initialized.
+   */ async initializePeers() {
+        if (this.config.trustedPeers) {
+            const trustedPeersEnrs = this.config.trustedPeers.map((enr)=>ENR.decodeTxt(enr));
+            await Promise.all(trustedPeersEnrs.map((enr)=>enr.peerId())).then((peerIds)=>peerIds.forEach((peerId)=>this.trustedPeers.add(peerId.toString()))).finally(()=>{
+                this.trustedPeersInitialized = true;
+            }).catch((e)=>this.logger.error('Error initializing trusted peers', e));
+        }
+        if (this.config.privatePeers) {
+            const privatePeersEnrs = this.config.privatePeers.map((enr)=>ENR.decodeTxt(enr));
+            await Promise.all(privatePeersEnrs.map((enr)=>enr.peerId())).then((peerIds)=>peerIds.forEach((peerId)=>{
+                    this.trustedPeers.add(peerId.toString());
+                    this.privatePeers.add(peerId.toString());
+                })).finally(()=>{
+                if (!this.config.trustedPeers) {
+                    this.trustedPeersInitialized = true;
+                }
+                this.privatePeersInitialized = true;
+            }).catch((e)=>this.logger.error('Error initializing private peers', e));
+        }
+        if (this.config.preferredPeers) {
+            const preferredPeersEnrs = this.config.preferredPeers.map((enr)=>ENR.decodeTxt(enr));
+            await Promise.all(preferredPeersEnrs.map((enr)=>enr.peerId())).then((peerIds)=>peerIds.forEach((peerId)=>this.preferredPeers.add(peerId.toString()))).catch((e)=>this.logger.error('Error initializing preferred peers', e));
+        }
+    }
+    get tracer() {
+        return this.metrics.tracer;
+    }
+    async heartbeat() {
+        this.heartbeatCounter++;
+        this.peerScoring.decayAllScores();
+        this.peerScoring.pruneExpiredBans();
+        this.cleanupExpiredTimeouts();
+        await this.setupDirectPeersIfValidator();
+        await this.updateAuthenticatedPeers();
+        await this.processScheduledDisconnects();
+        this.discover();
+        this.checkPeerConnectivity();
+    }
+    /**
+   * Flags a node that has been left without any connected peers, since it can neither gossip nor propagate txs.
+   *
+   * Warnings only start after a few consecutive zero-peer heartbeats so that startup or a transient dip stays quiet,
+   * and then repeat roughly once a minute instead of on every heartbeat. Recovery is logged once.
+   */ checkPeerConnectivity() {
+        const connectedPeerCount = this.libP2PNode.getPeers().length;
+        const now = this.dateProvider.now();
+        if (connectedPeerCount > 0) {
+            if (this.zeroPeerWarningIssued) {
+                this.logger.info(`Peer connectivity restored with ${connectedPeerCount} connected peers`, {
+                    connectedPeerCount,
+                    zeroPeerHeartbeats: this.consecutiveZeroPeerHeartbeats
+                });
+            }
+            this.consecutiveZeroPeerHeartbeats = 0;
+            this.zeroPeerWarningIssued = false;
+            this.lastConnectedPeerAtMs = now;
+            return;
+        }
+        this.consecutiveZeroPeerHeartbeats++;
+        const heartbeatsSinceThreshold = this.consecutiveZeroPeerHeartbeats - ZERO_PEER_WARNING_THRESHOLD_HEARTBEATS;
+        if (heartbeatsSinceThreshold < 0 || heartbeatsSinceThreshold % this.zeroPeerWarningHeartbeats !== 0) {
+            return;
+        }
+        this.zeroPeerWarningIssued = true;
+        this.logger.warn('Node has no connected peers; gossip and tx propagation are unavailable', {
+            zeroPeerHeartbeats: this.consecutiveZeroPeerHeartbeats,
+            peerCheckIntervalMS: this.config.peerCheckIntervalMS,
+            msSinceLastConnectedPeer: this.lastConnectedPeerAtMs === undefined ? undefined : now - this.lastConnectedPeerAtMs,
+            cachedPeers: this.cachedPeers.size
+        });
+    }
+    /*
+   * If this node is connecting to preferred peers, make sure it is registered validator */ async setupDirectPeersIfValidator() {
+        if (!this.config.preferredPeers) {
+            return;
+        }
+        // Already initialized preferred peers, don't wastefully repeat the same work
+        if (this.initializedPreferredPeers) {
+            return;
+        }
+        const registeredValidators = await this.epochCache.getRegisteredValidators();
+        const validatorSet = new Set(registeredValidators.map((v)=>v.toString()));
+        const isThisNodePartOfValidatorSet = this.validatorAddresses.some((v)=>validatorSet.has(v.toString()));
+        if (!isThisNodePartOfValidatorSet) {
+            return;
+        }
+        const preferredPeersEnrs = this.config.preferredPeers.map((enr)=>ENR.decodeTxt(enr));
+        await Promise.all(preferredPeersEnrs.map((enr)=>enr.peerId())).then((peerIds)=>peerIds.forEach((peerId)=>this.preferredPeers.add(peerId.toString()))).catch((e)=>this.logger.error('Error initializing preferred peers', e));
+        const directPeers = compactArray(await Promise.all(preferredPeersEnrs.map(async (enr)=>{
+            try {
+                const peerId = await enr.peerId();
+                const address = enr.getLocationMultiaddr('tcp');
+                if (address === undefined) {
+                    throw new Error(`Direct peer ${peerId.toString()} has no TCP address, ENR: ${enr.encodeTxt()}`);
+                }
+                return {
+                    id: peerId,
+                    addrs: [
+                        address
+                    ]
+                };
+            } catch (err) {
+                // A malformed configured ENR shouldn't abort preferred-peer setup — skip it and log.
+                this.logger.warn(`Skipping preferred peer with invalid ENR`, {
+                    err
+                });
+                return undefined;
+            }
+        })));
+        await Promise.all(directPeers.map((peer)=>{
+            this.libP2PNode.services.pubsub.direct.add(peer.id.toString());
+            return this.libP2PNode.peerStore.merge(peer.id, {
+                multiaddrs: peer.addrs
+            });
+        }));
+        this.initializedPreferredPeers = true;
+    }
+    /**
+   * Cleans up expired timeouts and stale failed-auth-handshake entries.
+   *
+   * When peers fail to dial after a number of retries, they are temporarily timed out.
+   * This function removes any peers that have been in the timed out state for too long.
+   * To give them a chance to reconnect.
+   *
+   * Also evicts entries from the failed-auth-handshake map whose expiry window has passed.
+   * Without this, peers that probe once and never reconnect would leave their entries in the
+   * map forever, causing an unbounded memory leak.
+   */ cleanupExpiredTimeouts() {
+        const now = this.dateProvider.now();
+        for (const [peerId, timedOutPeer] of this.timedOutPeers.entries()){
+            if (now >= timedOutPeer.timeoutUntilMs) {
+                this.timedOutPeers.delete(peerId);
+            }
+        }
+        for (const [id, entry] of this.failedAuthHandshakes.entries()){
+            if (now - entry.lastFailureTimestamp > FAILED_AUTH_HANDSHAKE_EXPIRY_MS) {
+                this.failedAuthHandshakes.delete(id);
+            }
+        }
+    }
+    /**
+   * Processes scheduled disconnects during heartbeat.
+   *
+   * This batch processes all peers that have been marked for disconnect.
+   * preventing immediate disconnects that could cause libp2p state corruption.
+   */ async processScheduledDisconnects() {
+        if (this.peersToBeDisconnected.size === 0) {
+            return;
+        }
+        const peersToDisconnect = Array.from(this.peersToBeDisconnected);
+        this.logger.debug(`Processing ${peersToDisconnect.length} scheduled disconnects`);
+        try {
+            await Promise.all(peersToDisconnect.map(async (peerIdStr)=>{
+                if (await this.disconnectPeer(peerIdFromString(peerIdStr))) {
+                    this.peersToBeDisconnected.delete(peerIdStr);
+                }
+            }));
+            this.logger.verbose(`Disconnected ${peersToDisconnect.length} peers`, {
+                peersToDisconnect
+            });
+        } catch (error) {
+            this.logger.error('Error when disconnecting from peers', error);
+        }
+    }
+    /**
+   * Performs Status Handshake with a connected peer.
+   * @param e - The connected peer event.
+   */ handleConnectedPeerEvent(e) {
+        const peerId = e.detail;
+        this.logger.verbose(`Connected to peer ${peerId.toString()}`);
+        this.metrics.peerConnected(peerId);
+        if (this.config.p2pDisableStatusHandshake) {
+            return;
+        }
+        // If we are not configured to only allow validators then perform a status handshake
+        if (!this.config.p2pAllowOnlyValidators) {
+            void this.exchangeStatusHandshake(peerId);
+            return;
+        }
+        // We are configured to only allow validators, but this doesn't apply to trusted, private peers or preferred peers
+        if (this.isProtectedPeer(peerId)) {
+            void this.exchangeStatusHandshake(peerId);
+            return;
+        }
+        // Initiate auth handshake
+        void this.exchangeAuthHandshake(peerId);
+    }
+    /**
+   * Simply logs the type of disconnected peer.
+   * @param e - The disconnected peer event.
+   */ handleDisconnectedPeerEvent(e) {
+        const peerId = e.detail;
+        const peerIdStr = peerId.toString();
+        this.metrics.peerDisconnected(peerId);
+        this.logger.verbose(`Disconnected from peer ${peerIdStr}`);
+        const validatorAddress = this.authenticatedPeerIdToValidatorAddress.get(peerIdStr);
+        if (validatorAddress !== undefined) {
+            this.logger.info(`Removing authentication for validator ${validatorAddress} at peer id ${peerIdStr} due to disconnection`);
+            this.authenticatedValidatorAddressToPeerId.delete(validatorAddress.toString());
+            this.authenticatedPeerIdToValidatorAddress.delete(peerIdStr);
+        }
+        if (this.peerScoring.getScoreState(peerIdStr) === PeerScoreState.Healthy) {
+            this.peerScoring.removePeer(peerIdStr);
+        }
+    }
+    registerThisValidatorAddresses(address) {
+        this.validatorAddresses = [
+            ...address
+        ];
+    }
+    /**
+   * Checks if a peer is trusted.
+   * @param peerId - The peer ID.
+   * @returns True if the peer is trusted, false otherwise.
+   * Note: This function will return false and log a warning if the trusted peers are not initialized.
+   */ isTrustedPeer(peerId) {
+        if (!this.trustedPeersInitialized) {
+            this.logger.warn('Trusted peers not initialized, returning false');
+            return false;
+        }
+        return this.trustedPeers.has(peerId.toString());
+    }
+    /**
+   * Adds a peer to the trusted peers set.
+   * @param peerId - The peer ID to add to trusted peers.
+   */ addTrustedPeer(peerId) {
+        const peerIdStr = peerId.toString();
+        this.trustedPeers.add(peerIdStr);
+        this.trustedPeersInitialized = true;
+        this.logger.verbose(`Added trusted peer ${peerIdStr}`);
+    }
+    /**
+   * Adds a peer to the private peers set.
+   * @param peerId - The peer ID to add to private peers.
+   */ addPrivatePeer(peerId) {
+        const peerIdStr = peerId.toString();
+        this.trustedPeers.add(peerIdStr);
+        this.privatePeers.add(peerIdStr);
+        this.trustedPeersInitialized = true;
+        this.privatePeersInitialized = true;
+        this.logger.verbose(`Added private peer ${peerIdStr}`);
+    }
+    /**
+   * Checks if a peer is private.
+   * @param peerId - The peer ID.
+   * @returns True if the peer is private, false otherwise.
+   */ isPrivatePeer(peerId) {
+        if (!this.privatePeersInitialized) {
+            this.logger.warn('Private peers not initialized, returning false');
+            return false;
+        }
+        return this.privatePeers.has(peerId.toString());
+    }
+    /**
+   * Adds a peer to the preferred peers set.
+   * @param peerId - The peer ID to add to preferred peers.
+   */ addPreferredPeer(peerId) {
+        const peerIdStr = peerId.toString();
+        this.preferredPeers.add(peerIdStr);
+        this.logger.verbose(`Added preferred peer ${peerIdStr}`);
+    }
+    /**
+   * Checks if a peer is preferred.
+   * @param peerId - The peer ID.
+   * @returns True if the peer is preferred, false otherwise.
+   */ isPreferredPeer(peerId) {
+        return this.preferredPeers.has(peerId.toString());
+    }
+    /**
+   * Checks if a peer is protected (either trusted or private).
+   * @param peerId - The peer ID.
+   * @returns True if the peer is protected, false otherwise.
+   */ isProtectedPeer(peerId) {
+        return this.isTrustedPeer(peerId) || this.isPrivatePeer(peerId) || this.isPreferredPeer(peerId);
+    }
+    /**
+   * Handles a goodbye received from a peer.
+   *
+   * Used as the reqresp handler when a peer sends us goodbye message.
+   * @param peerId - The peer ID.
+   * @param reason - The reason for the goodbye.
+   */ goodbyeReceived(peerId, reason) {
+        this.logger.debug(`Goodbye received from peer ${peerId.toString()} with reason ${prettyGoodbyeReason(reason)}`);
+        this.metrics.recordGoodbyeReceived(reason);
+        this.markPeerForDisconnect(peerId);
+    }
+    penalizePeer(peerId, penalty) {
+        this.peerScoring.penalizePeer(peerId, penalty);
+    }
+    getPeerScore(peerId) {
+        return this.peerScoring.getScore(peerId);
+    }
+    shouldDisableP2PGossip(peerId) {
+        const isAuthenticated = this.isAuthenticatedPeer(peerIdFromString(peerId));
+        return (this.config.p2pAllowOnlyValidators ?? false) && !isAuthenticated;
+    }
+    getPeers(includePending = false) {
+        const connected = this.libP2PNode.getPeers().map((peer)=>({
+                id: peer.toString(),
+                score: this.getPeerScore(peer.toString()),
+                status: 'connected'
+            }));
+        if (!includePending) {
+            return connected;
+        }
+        const dialQueue = this.libP2PNode.getDialQueue().filter((peer)=>!!peer.peerId).map((peer)=>({
+                id: peer.peerId.toString(),
+                status: 'dialing',
+                dialStatus: peer.status,
+                addresses: peer.multiaddrs.map((m)=>m.toString())
+            }));
+        const cachedPeers = Array.from(this.cachedPeers.values()).filter((peer)=>!dialQueue.some((dialPeer)=>dialPeer.id && peer.peerId.toString() === dialPeer.id.toString())).filter((peer)=>!connected.some((connPeer)=>connPeer.id.toString() === peer.peerId.toString())).map((peer)=>({
+                status: 'cached',
+                id: peer.peerId.toString(),
+                addresses: [
+                    peer.multiaddrTcp.toString()
+                ],
+                dialAttempts: peer.dialAttempts,
+                enr: peer.enr.encodeTxt()
+            }));
+        return [
+            ...connected,
+            ...dialQueue,
+            ...cachedPeers
+        ];
+    }
+    isAuthenticatedPeer(peerId) {
+        const peerIdAsString = peerId.toString();
+        return this.privatePeers.has(peerIdAsString) || this.trustedPeers.has(peerIdAsString) || this.preferredPeers.has(peerIdAsString) || this.authenticatedPeerIdToValidatorAddress.has(peerIdAsString);
+    }
+    /*
+   * Checks whether peer is allowed to connect
+   *
+   * @param id: Address of the node or it's peerId
+   *
+   * @returns: True if node is allowed to connect, otherwise false
+   * */ /**
+   * Whether a peer is allowed to connect, given its peer id. Rejects peers serving an active ban and
+   * peers that have exceeded the failed auth-handshake limit. Use this once the peer id is known —
+   * i.e. for the encrypted-inbound gater and when dialing.
+   */ isPeerAllowedToConnect(peerId) {
+        const id = peerId.toString();
+        if (this.peerScoring.getScoreState(id) === PeerScoreState.Banned) {
+            return false;
+        }
+        return this.isWithinFailedAuthLimit(id);
+    }
+    /**
+   * Whether a connection from an address is allowed. Bans are keyed by peer id, which isn't known at
+   * the raw-inbound layer, so only the failed auth-handshake limit (also tracked per address) is
+   * enforced here; the ban is applied once the peer id is known via {@link isPeerAllowedToConnect}.
+   */ isAddressAllowedToConnect(address) {
+        return this.isWithinFailedAuthLimit(address);
+    }
+    /** Whether the failed auth-handshake count for a peer id or address is below the configured limit. */ isWithinFailedAuthLimit(key) {
+        const entry = this.failedAuthHandshakes.get(key);
+        if (!entry) {
+            return true;
+        }
+        // In case entry is too old, remove it and allow connection
+        if (this.dateProvider.now() - entry.lastFailureTimestamp > FAILED_AUTH_HANDSHAKE_EXPIRY_MS) {
+            this.failedAuthHandshakes.delete(key);
+            return true;
+        }
+        return entry.count <= this.config.p2pMaxFailedAuthAttemptsAllowed;
+    }
+    /**
+   * Discovers peers.
+   */ discover() {
+        const connections = this.libP2PNode.getConnections();
+        const healthyConnections = this.prioritizePeers(this.pruneUnhealthyPeers(this.getNonProtectedPeers(connections)));
+        // Calculate how many connections we're looking to make
+        const protectedPeerCount = this.getProtectedPeerCount();
+        const peersToConnect = this.config.maxPeerCount - healthyConnections.length - protectedPeerCount;
+        const logLevel = this.heartbeatCounter % this.displayPeerCountsPeerHeartbeat === 0 ? 'info' : 'debug';
+        this.logger[logLevel](`Connected to ${healthyConnections.length + this.trustedPeers.size} peers`, {
+            discoveredConnections: healthyConnections.length,
+            protectedConnections: protectedPeerCount,
+            maxPeerCount: this.config.maxPeerCount,
+            cachedPeers: this.cachedPeers.size,
+            discv5KadPeers: this.peerDiscoveryService.getKadValues().length,
+            ...this.peerScoring.getStats()
+        });
+        this.metrics.recordPeerCount(connections.length);
+        this.metrics.recordHealthyPeerCount(healthyConnections.length);
+        // Exit if no peers to connect
+        if (peersToConnect <= 0) {
+            return;
+        }
+        const cachedPeersToDial = [];
+        const pendingDials = new Set(this.libP2PNode.getDialQueue().map((pendingDial)=>pendingDial.peerId?.toString()).filter(Boolean));
+        const now = this.dateProvider.now();
+        for (const [id, peerData] of this.cachedPeers.entries()){
+            // if already dialling or connected to, remove from cache
+            if (pendingDials.has(id) || healthyConnections.some((conn)=>conn.remotePeer.equals(peerData.peerId)) || // if peer has been in cache for the max cache age, remove from cache
+            now - peerData.addedUnixMs > MAX_CACHED_PEER_AGE_MS) {
+                this.cachedPeers.delete(id);
+            } else {
+                // cachedPeersToDial.set(id, enr);
+                cachedPeersToDial.push(peerData);
+            }
+        }
+        // reverse to dial older entries first
+        cachedPeersToDial.reverse();
+        for (const peer of cachedPeersToDial){
+            // We remove from the cache before, as dialling will add it back if it fails
+            this.cachedPeers.delete(peer.peerId.toString());
+            void this.dialPeer(peer);
+        }
+        // if we need more peers, start randomNodesQuery
+        if (peersToConnect > 0) {
+            this.logger.trace(`Running random nodes query to connect to ${peersToConnect} peers`);
+            void this.peerDiscoveryService.runRandomNodesQuery();
+        }
+    }
+    getNonProtectedPeers(connections) {
+        return connections.filter((conn)=>!this.isProtectedPeer(conn.remotePeer));
+    }
+    getProtectedPeerCount() {
+        return this.trustedPeers.size + this.privatePeers.size + this.preferredPeers.size;
+    }
+    pruneUnhealthyPeers(connections) {
+        const connectedHealthyPeers = [];
+        for (const peer of connections){
+            const score = this.peerScoring.getScoreState(peer.remotePeer.toString());
+            switch(score){
+                case PeerScoreState.Banned:
+                    this.metrics.recordLowScoreDisconnect('Banned');
+                    void this.goodbyeAndDisconnectPeer(peer.remotePeer, GoodByeReason.BANNED);
+                    break;
+                case PeerScoreState.Disconnect:
+                    this.metrics.recordLowScoreDisconnect('Disconnect');
+                    void this.goodbyeAndDisconnectPeer(peer.remotePeer, GoodByeReason.LOW_SCORE);
+                    break;
+                case PeerScoreState.Healthy:
+                    connectedHealthyPeers.push(peer);
+            }
+        }
+        return connectedHealthyPeers;
+    }
+    /**
+   * If the max peer count is reached, the lowest scoring peers will be pruned to satisfy the max peer count.
+   *
+   * @param connections - The list of connections to prune low scoring peers above the max peer count from.
+   * @returns The pruned list of connections.
+   */ prioritizePeers(connections) {
+        const protectedPeerCount = this.getProtectedPeerCount();
+        if (connections.length > this.config.maxPeerCount - protectedPeerCount) {
+            // Sort the regular peer scores from highest to lowest
+            const prioritizedConnections = connections.sort((connectionA, connectionB)=>{
+                const connectionScoreA = this.peerScoring.getScore(connectionA.remotePeer.toString());
+                const connectionScoreB = this.peerScoring.getScore(connectionB.remotePeer.toString());
+                return connectionScoreB - connectionScoreA;
+            });
+            // Calculate how many regular peers we can keep
+            const peersToKeep = Math.max(0, this.config.maxPeerCount - protectedPeerCount);
+            // Disconnect from the lowest scoring regular connections that exceed our limit
+            for (const conn of prioritizedConnections.slice(peersToKeep)){
+                void this.goodbyeAndDisconnectPeer(conn.remotePeer, GoodByeReason.MAX_PEERS);
+            }
+            // Return trusted connections plus the highest scoring regular connections up to the max peer count
+            return prioritizedConnections.slice(0, peersToKeep);
+        } else {
+            return connections;
+        }
+    }
+    async goodbyeAndDisconnectPeer(peer, reason) {
+        this.logger.debug(`Disconnecting peer ${peer.toString()} with reason ${prettyGoodbyeReason(reason)}`);
+        this.metrics.recordGoodbyeSent(reason);
+        try {
+            const resp = await this.reqresp.sendRequestToPeer(peer, ReqRespSubProtocol.GOODBYE, Buffer.from([
+                reason
+            ]), GOODBYE_DIAL_TIMEOUT_MS);
+            if (resp.status === ReqRespStatus.FAILURE) {
+                this.logger.debug(`Failed to send goodbye to peer ${peer.toString()}`);
+            } else if (resp.status === ReqRespStatus.SUCCESS) {
+                this.logger.verbose(`Sent goodbye to peer ${peer.toString()}`);
+            } else {
+                this.logger.debug(`Unexpected status sending goodbye to peer ${peer.toString()}: ${ReqRespStatus[resp.status]}`);
+            }
+        } catch (error) {
+            this.logger.debug(`Failed to send goodbye to peer ${peer.toString()}: ${error}`);
+        } finally{
+            this.markPeerForDisconnect(peer);
+        }
+    }
+    /*
+   * Marks peer to be disconnected on the next heartbeat
+   * */ markPeerForDisconnect(peer) {
+        const peerIdStr = peer.toString();
+        this.logger.debug(`Scheduling peer ${peerIdStr} for disconnection`);
+        this.peersToBeDisconnected.add(peerIdStr);
+    }
+    /**
+   * Performs the actual disconnection of a peer.
+   * This is called during heartbeat processing to avoid immediate disconnections.
+   *
+   * @returns True if peer was disconnect, otherwise false
+   */ async disconnectPeer(peer) {
+        const peerIdStr = peer.toString();
+        try {
+            await this.libP2PNode.hangUp(peer);
+            this.logger.debug(`Successfully disconnected peer ${peerIdStr}`);
+            return true;
+        } catch (error) {
+            this.logger.warn(`Failed to disconnect peer ${peerIdStr}`, {
+                error
+            });
+            return false;
+        }
+    }
+    /**
+   *  Handles a discovered peer.
+   * @param enr - The discovered peer's ENR.
+   */ async handleDiscoveredPeer(enr) {
+        // Check that the peer has not already been banned
+        const peerId = await enr.peerId();
+        const peerIdString = peerId.toString();
+        // Don't attempt to connect to peers scheduled for disconnection
+        if (this.peersToBeDisconnected.has(peerIdString)) {
+            this.logger.trace(`Skipping peer scheduled for disconnection ${peerId}`);
+            return;
+        }
+        // Check if peer is temporarily timed out
+        const timedOutPeer = this.timedOutPeers.get(peerIdString);
+        if (timedOutPeer) {
+            if (this.dateProvider.now() < timedOutPeer.timeoutUntilMs) {
+                this.logger.trace(`Skipping timed out peer ${peerId}`);
+                return;
+            }
+            // Timeout period expired, remove from timed out peers
+            this.timedOutPeers.delete(peerIdString);
+        }
+        if (this.peerScoring.getScoreState(peerIdString) != PeerScoreState.Healthy) {
+            return;
+        }
+        // Don't dial banned peers or those that have exceeded the auth failure threshold
+        if (!this.isPeerAllowedToConnect(peerId)) {
+            this.logger.trace(`Skipping peer ${peerId} due to ban or failed auth handshake attempts`);
+            return;
+        }
+        const [multiaddrTcp] = await Promise.all([
+            enr.getFullMultiaddr('tcp')
+        ]);
+        this.logger.trace(`Handling discovered peer ${peerId} at ${multiaddrTcp?.toString() ?? 'undefined address'}`);
+        // stop if no tcp addr in multiaddr
+        if (!multiaddrTcp) {
+            this.logger.debug(`No TCP address in discovered node's multiaddr ${enr.encodeTxt()}`);
+            return;
+        }
+        // check if peer is already connected
+        const connections = this.libP2PNode.getConnections();
+        if (connections.some((conn)=>conn.remotePeer.equals(peerId))) {
+            this.logger.trace(`Already connected to peer ${peerId}`);
+            return;
+        }
+        // check if peer is already in cache
+        if (this.cachedPeers.has(peerIdString)) {
+            this.logger.trace(`Peer already in cache ${peerIdString}`);
+            return;
+        }
+        // create cached peer object
+        const cachedPeer = {
+            peerId,
+            enr,
+            multiaddrTcp,
+            dialAttempts: 0,
+            addedUnixMs: this.dateProvider.now()
+        };
+        // Determine if we should dial immediately or not
+        if (this.shouldDialPeer()) {
+            void this.dialPeer(cachedPeer);
+        } else {
+            this.logger.trace(`Caching peer ${peerIdString}`);
+            this.cachedPeers.set(peerIdString, cachedPeer);
+            // Prune set of cached peers
+            this.pruneCachedPeers();
+        }
+    }
+    async dialPeer(peer) {
+        const id = peer.peerId.toString();
+        // Add to the address book before dialing
+        await this.libP2PNode.peerStore.merge(peer.peerId, {
+            multiaddrs: [
+                peer.multiaddrTcp
+            ]
+        });
+        this.logger.trace(`Dialing peer ${id}`);
+        try {
+            await this.libP2PNode.dial(peer.multiaddrTcp);
+        } catch (error) {
+            peer.dialAttempts++;
+            if (peer.dialAttempts < MAX_DIAL_ATTEMPTS) {
+                this.logger.trace(`Failed to dial peer ${id} (attempt ${peer.dialAttempts})`, {
+                    error: inspect(error)
+                });
+                this.cachedPeers.set(id, peer);
+            } else {
+                formatLibp2pDialError(error);
+                this.logger.debug(`Failed to dial peer ${id} (dropping)`, {
+                    error: inspect(error)
+                });
+                this.cachedPeers.delete(id);
+                // Add to timed out peers
+                this.timedOutPeers.set(id, {
+                    peerId: id,
+                    timeoutUntilMs: this.dateProvider.now() + (this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS)
+                });
+            }
+        }
+    }
+    shouldDialPeer() {
+        const connections = this.libP2PNode.getConnections().length;
+        if (connections >= this.config.maxPeerCount) {
+            this.logger.trace(`Not dialing peer due to max peer count of ${this.config.maxPeerCount} reached (${connections} current connections)`);
+            return false;
+        }
+        return true;
+    }
+    pruneCachedPeers() {
+        let peersToDelete = this.cachedPeers.size - MAX_CACHED_PEERS;
+        if (peersToDelete <= 0) {
+            return;
+        }
+        // Remove the oldest peers
+        for (const [key, value] of this.cachedPeers.entries()){
+            if (this.isProtectedPeer(value.peerId)) {
+                this.logger.debug(`Not pruning trusted peer ${key}`);
+                continue;
+            }
+            this.cachedPeers.delete(key);
+            this.logger.trace(`Pruning peer ${key} from cache`);
+            peersToDelete--;
+            if (peersToDelete <= 0) {
+                break;
+            }
+        }
+    }
+    async createStatusMessage() {
+        const syncSummary = (await this.worldStateSynchronizer.status()).syncSummary;
+        return StatusMessage.fromWorldStateSyncStatus(this.protocolVersion, syncSummary);
+    }
+    /**
+   * Performs status Handshake with the Peer
+   * The way the protocol is designed is that each peer will call this method on newly established p2p connection.
+   * Both peers request Status message and both peers perform validation of the received Status message.
+   * If this validation fails on any end that peer will initiate disconnect.
+   *  Note: It's important for both peers to request and perform Status validation,
+   *  Because one of the peers can be _bad peer_ and this peer can simply skip the check.
+   *  If we don't implement validation on both ends the _bad peer_ remains connected.
+   * @param: peerId The Id of the peer to request the Status from.
+   * */ async exchangeStatusHandshake(peerId) {
+        try {
+            const ourStatus = await this.createStatusMessage();
+            //Note: Technically we don't have to send out status to peer as well, but we do.
+            //It will be easier to update protocol in the future this way if need be.
+            this.logger.trace(`Initiating status handshake with peer ${peerId}`);
+            const response = await this.reqresp.sendRequestToPeer(peerId, ReqRespSubProtocol.STATUS, ourStatus.toBuffer());
+            const { status } = response;
+            if (status !== ReqRespStatus.SUCCESS) {
+                //TODO: maybe hard ban these peers in the future.
+                //We could allow this to happen up to N times, and then hard ban?
+                //Hard ban: Disallow connection via e.g. libp2p's Gater
+                this.logger.debug(`Disconnecting peer ${peerId} who failed to respond status handshake`, {
+                    peerId,
+                    status: ReqRespStatus[status]
+                });
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            const { data } = response;
+            const logData = {
+                peerId,
+                status: ReqRespStatus[status],
+                data: data ? bufferToHex(data) : undefined
+            };
+            const peerStatusMessage = StatusMessage.fromBuffer(data);
+            if (!ourStatus.validate(peerStatusMessage)) {
+                this.logger.debug(`Disconnecting peer ${peerId} due to failed status handshake.`, logData);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            this.logger.debug(`Successfully completed status handshake with peer ${peerId}`, logData);
+        } catch (err) {
+            //TODO: maybe hard ban these peers in the future
+            this.logger.debug(`Disconnecting peer ${peerId} due to error during status handshake: ${err.message ?? err}`, {
+                peerId
+            });
+            this.markPeerForDisconnect(peerId);
+        }
+    }
+    /**
+   * Performs auth Handshake with the Peer
+   * A superset of the status handshake. Also includes a challenge that needs to be signed by the peer's validator key.
+   * @param: peerId The Id of the peer to request the Status from.
+   * */ async exchangeAuthHandshake(peerId) {
+        const peerIdString = peerId.toString();
+        try {
+            const ourStatus = await this.createStatusMessage();
+            const authRequest = new AuthRequest(ourStatus, Fr.random());
+            // Note: Technically we don't have to send our status to peer as well, but we do.
+            // It will be easier to update protocol in the future this way if need be.
+            // We also need to send the challenge at least, so that the peer can sign it.
+            this.logger.debug(`Initiating auth handshake with peer ${peerId}`);
+            const response = await this.reqresp.sendRequestToPeer(peerId, ReqRespSubProtocol.AUTH, authRequest.toBuffer());
+            const { status } = response;
+            if (status !== ReqRespStatus.SUCCESS) {
+                this.logger.verbose(`Disconnecting peer ${peerId} who failed to respond auth handshake`, {
+                    peerId,
+                    status: ReqRespStatus[status]
+                });
+                this.markAuthHandshakeFailed(peerId);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            const { data } = response;
+            const logData = {
+                peerId,
+                status: ReqRespStatus[status],
+                data: data ? bufferToHex(data) : undefined
+            };
+            const peerAuthResponse = AuthResponse.fromBuffer(data);
+            const peerStatusMessage = peerAuthResponse.status;
+            if (!ourStatus.validate(peerStatusMessage)) {
+                this.logger.verbose(`Disconnecting peer ${peerId} due to failed status handshake as part of auth.`, logData);
+                this.markAuthHandshakeFailed(peerId);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            const hashToRecover = authRequest.getPayloadToSign();
+            const ethSignedHash = makeEthSignDigest(hashToRecover);
+            const sender = tryRecoverAddress(ethSignedHash, peerAuthResponse.signature);
+            if (!sender) {
+                this.logger.verbose(`Disconnecting peer ${peerId} due to failed auth handshake, invalid signature.`, logData);
+                this.markAuthHandshakeFailed(peerId);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            const registeredValidators = await this.epochCache.getRegisteredValidators();
+            const found = registeredValidators.find((v)=>v.toString() === sender.toString()) !== undefined;
+            if (!found) {
+                this.logger.verbose(`Disconnecting peer ${peerId} due to failed auth handshake, peer is not a registered validator.`, {
+                    ...logData,
+                    address: sender.toString()
+                });
+                this.markAuthHandshakeFailed(peerId);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            // Check to see that this validator address isn't already allocated to a different peer
+            const peerForAddress = this.authenticatedValidatorAddressToPeerId.get(sender.toString());
+            if (peerForAddress !== undefined && peerForAddress.toString() !== peerIdString) {
+                this.logger.verbose(`Received auth for validator ${sender.toString()} from peer ${peerIdString}, but this validator is already authenticated to peer ${peerForAddress.toString()}`, {
+                    ...logData,
+                    address: sender.toString()
+                });
+                this.markAuthHandshakeFailed(peerId);
+                this.markPeerForDisconnect(peerId);
+                return;
+            }
+            this.markAuthHandshakeSuccess(peerId);
+            this.authenticatedPeerIdToValidatorAddress.set(peerIdString, sender);
+            this.authenticatedValidatorAddressToPeerId.set(sender.toString(), peerId);
+            this.logger.info(`Successfully completed auth handshake with peer ${peerId}, validator address ${sender.toString()}`, {
+                ...logData,
+                address: sender.toString()
+            });
+        } catch (err) {
+            //TODO: maybe hard ban these peers in the future
+            this.logger.verbose(`Disconnecting peer ${peerId} due to error during auth handshake: ${err.message}`, {
+                peerId,
+                err
+            });
+            this.markAuthHandshakeFailed(peerId);
+            this.markPeerForDisconnect(peerId);
+        }
+    }
+    /*
+   * Marks when peer fails auth handshake
+   * */ markAuthHandshakeFailed(peerId) {
+        const now = this.dateProvider.now();
+        const peerIdStr = peerId.toString();
+        const existingEntry = this.failedAuthHandshakes.get(peerIdStr);
+        const failureCount = (existingEntry?.count || 0) + 1;
+        this.failedAuthHandshakes.set(peerIdStr, {
+            count: failureCount,
+            lastFailureTimestamp: now
+        });
+        const connections = this.libP2PNode.getConnections(peerId);
+        connections.forEach((conn)=>{
+            const address = conn.remoteAddr.nodeAddress().address;
+            const existingAddressEntry = this.failedAuthHandshakes.get(address);
+            this.failedAuthHandshakes.set(address, {
+                count: (existingAddressEntry?.count || 0) + 1,
+                lastFailureTimestamp: now
+            });
+        });
+        // Ban the peer from being re-dialed for a cooldown period (exponential backoff)
+        const banTimeMs = this.config.peerFailedBanTimeMs ?? DEFAULT_FAILED_PEER_BAN_TIME_MS;
+        const backoffMs = banTimeMs * Math.pow(2, Math.min(failureCount - 1, 5));
+        this.timedOutPeers.set(peerIdStr, {
+            peerId: peerIdStr,
+            timeoutUntilMs: now + backoffMs
+        });
+        this.cachedPeers.delete(peerIdStr);
+    }
+    /*
+   * Marks when peer exchanges auth handshake
+   * Removes any failed previous attempts
+   * */ markAuthHandshakeSuccess(peerId) {
+        this.failedAuthHandshakes.delete(peerId.toString());
+        const connections = this.libP2PNode.getConnections(peerId);
+        connections.forEach((conn)=>{
+            const address = conn.remoteAddr.nodeAddress().address;
+            this.failedAuthHandshakes.delete(address);
+        });
+    }
+    /**
+   * Stops the peer manager.
+   * Removing all event listeners.
+   */ async stop() {
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        this.peerDiscoveryService.off(PeerEvent.DISCOVERED, this.handlers.handleDiscoveredPeer);
+        // Send goodbyes to all peers
+        await Promise.all(this.libP2PNode.getPeers().map((peer)=>this.goodbyeAndDisconnectPeer(peer, GoodByeReason.SHUTDOWN)));
+        this.libP2PNode.removeEventListener(PeerEvent.CONNECTED, this.handlers.handleConnectedPeerEvent);
+        this.libP2PNode.removeEventListener(PeerEvent.DISCONNECTED, this.handlers.handleDisconnectedPeerEvent);
+    }
+    shouldTrustWithIdentity(peerId) {
+        return this.isProtectedPeer(peerId);
+    }
+    /**
+   * Performs auth request verification from peer. An auth request is valid if requested by an authorized peer (a peer we trust).
+   *
+   * @param: _authRequest - Auth request (unused)
+   * @param: peerId - The ID of the peer that requested the auth handshake
+   *
+   * @returns: StatusMessage if peer is trusted
+   *
+   * @throws: If peer is unauthorized
+   * */ async handleAuthRequestFromPeer(_authRequest, peerId) {
+        if (!this.shouldTrustWithIdentity(peerId)) {
+            this.logger.warn(`Received auth request from untrusted peer ${peerId.toString()}`);
+            throw new Error('Unauthorised');
+        }
+        this.logger.debug(`Received auth request from trusted peer ${peerId.toString()}`);
+        return await this.createStatusMessage();
+    }
+    async updateAuthenticatedPeers() {
+        const registeredValidators = await this.epochCache.getRegisteredValidators();
+        const validatorSet = new Set(registeredValidators.map((v)=>v.toString()));
+        const peersToDelete = new Set();
+        const addressesToDelete = new Set();
+        for (const [peer, address] of this.authenticatedPeerIdToValidatorAddress.entries()){
+            const addressString = address.toString();
+            if (!validatorSet.has(addressString)) {
+                peersToDelete.add(peer);
+                addressesToDelete.add(addressString);
+                this.logger.info(`Removing authentication for peer ${peer.toString()} at address ${addressString} due to no longer being a registered validator`);
+            }
+        }
+        for (const peer of peersToDelete){
+            this.authenticatedPeerIdToValidatorAddress.delete(peer);
+        }
+        for (const address of addressesToDelete){
+            this.authenticatedValidatorAddressToPeerId.delete(address);
+        }
+    }
+}
+/**
+ * copied from github.com/ChainSafe/lodestar
+ * libp2p errors with extremely noisy errors here, which are deeply nested taking 30-50 lines.
+ * Some known errors:
+ * ```
+ * Error: The operation was aborted
+ * Error: stream ended before 1 bytes became available
+ * Error: Error occurred during XX handshake: Error occurred while verifying signed payload: Peer ID doesn't match libp2p public key
+ * ```
+ *
+ * Also the error's message is not properly formatted, where the error message is indented and includes the full stack
+ * ```
+ * {
+ *  emessage: '\n' +
+ *    '    Error: stream ended before 1 bytes became available\n' +
+ *    '        at /home/lion/Code/eth2.0/lodestar/node_modules/it-reader/index.js:37:9\n' +
+ *    '        at runMicrotasks (<anonymous>)\n' +
+ *    '        at decoder (/home/lion/Code/eth2.0/lodestar/node_modules/it-length-prefixed/src/decode.js:113:22)\n' +
+ *    '        at first (/home/lion/Code/eth2.0/lodestar/node_modules/it-first/index.js:11:20)\n' +
+ *    '        at Object.exports.read (/home/lion/Code/eth2.0/lodestar/node_modules/multistream-select/src/multistream.js:31:15)\n' +
+ *    '        at module.exports (/home/lion/Code/eth2.0/lodestar/node_modules/multistream-select/src/select.js:21:19)\n' +
+ *    '        at Upgrader._encryptOutbound (/home/lion/Code/eth2.0/lodestar/node_modules/libp2p/src/upgrader.js:397:36)\n' +
+ *    '        at Upgrader.upgradeOutbound (/home/lion/Code/eth2.0/lodestar/node_modules/libp2p/src/upgrader.js:176:11)\n' +
+ *    '        at ClassIsWrapper.dial (/home/lion/Code/eth2.0/lodestar/node_modules/libp2p-tcp/src/index.js:49:18)'
+ * }
+ * ```
+ *
+ * Tracking issue https://github.com/libp2p/js-libp2p/issues/996
+ */ function formatLibp2pDialError(e) {
+    const errorMessage = e.message.trim();
+    const newlineIndex = errorMessage.indexOf('\n');
+    e.message = newlineIndex !== -1 ? errorMessage.slice(0, newlineIndex) : errorMessage;
+    if (e.message.includes('The operation was aborted') || e.message.includes('stream ended before 1 bytes became available') || e.message.includes('The operation was aborted')) {
+        e.stack = undefined;
+    }
+}