libp2p 0.43.4 → 0.44.0-0d46c78f

package/src/connection-manager/auto-dial.ts

@@ -0,0 +1,189 @@
+import { logger } from '@libp2p/logger'
+import type { PeerStore } from '@libp2p/interface-peer-store'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
+import { PeerMap } from '@libp2p/peer-collections'
+import PQueue from 'p-queue'
+import { AUTO_DIAL_CONCURRENCY, AUTO_DIAL_INTERVAL, AUTO_DIAL_PRIORITY, MIN_CONNECTIONS } from './constants.js'
+import type { Startable } from '@libp2p/interfaces/startable'
+import type { EventEmitter } from '@libp2p/interfaces/events'
+import type { Libp2pEvents } from '@libp2p/interface-libp2p'
+
+const log = logger('libp2p:connection-manager:auto-dial')
+
+interface AutoDialInit {
+  minConnections?: number
+  autoDialConcurrency?: number
+  autoDialPriority?: number
+  autoDialInterval?: number
+}
+
+interface AutoDialComponents {
+  connectionManager: ConnectionManager
+  peerStore: PeerStore
+  events: EventEmitter<Libp2pEvents>
+}
+
+const defaultOptions = {
+  minConnections: MIN_CONNECTIONS,
+  autoDialConcurrency: AUTO_DIAL_CONCURRENCY,
+  autoDialPriority: AUTO_DIAL_PRIORITY,
+  autoDialInterval: AUTO_DIAL_INTERVAL
+}
+
+export class AutoDial implements Startable {
+  private readonly connectionManager: ConnectionManager
+  private readonly peerStore: PeerStore
+  private readonly queue: PQueue
+  private readonly minConnections: number
+  private readonly autoDialPriority: number
+  private readonly autoDialIntervalMs: number
+  private autoDialInterval?: ReturnType<typeof setInterval>
+  private started: boolean
+
+  /**
+   * Proactively tries to connect to known peers stored in the PeerStore.
+   * It will keep the number of connections below the upper limit and sort
+   * the peers to connect based on whether we know their keys and protocols.
+   */
+  constructor (components: AutoDialComponents, init: AutoDialInit) {
+    this.connectionManager = components.connectionManager
+    this.peerStore = components.peerStore
+    this.minConnections = init.minConnections ?? defaultOptions.minConnections
+    this.autoDialPriority = init.autoDialPriority ?? defaultOptions.autoDialPriority
+    this.autoDialIntervalMs = init.autoDialInterval ?? defaultOptions.autoDialInterval
+    this.started = false
+    this.queue = new PQueue({
+      concurrency: init.autoDialConcurrency ?? defaultOptions.autoDialConcurrency
+    })
+    this.queue.addListener('error', (err) => {
+      log.error('error during auto-dial', err)
+    })
+
+    // check the min connection limit whenever a peer disconnects
+    components.events.addEventListener('connection:close', () => {
+      this.autoDial()
+        .catch(err => {
+          log.error(err)
+        })
+    })
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  start (): void {
+    this.autoDialInterval = setInterval(() => {
+      this.autoDial()
+        .catch(err => {
+          log.error('error while autodialing', err)
+        })
+    }, this.autoDialIntervalMs)
+    this.started = true
+  }
+
+  afterStart (): void {
+    this.autoDial()
+      .catch(err => {
+        log.error('error while autodialing', err)
+      })
+  }
+
+  stop (): void {
+    // clear the queue
+    this.queue.clear()
+    clearInterval(this.autoDialInterval)
+    this.started = false
+  }
+
+  async autoDial (): Promise<void> {
+    if (!this.started) {
+      return
+    }
+
+    const connections = this.connectionManager.getConnectionsMap()
+    const numConnections = connections.size
+
+    // Already has enough connections
+    if (numConnections >= this.minConnections) {
+      log('have enough connections %d/%d', numConnections, this.minConnections)
+      return
+    }
+
+    log('not enough connections %d/%d - will dial peers to increase the number of connections', numConnections, this.minConnections)
+
+    // Sort peers on whether we know protocols or public keys for them
+    const peers = await this.peerStore.all()
+
+    // Remove some peers
+    const filteredPeers = peers.filter((peer) => {
+      // Remove peers without addresses
+      if (peer.addresses.length === 0) {
+        return false
+      }
+
+      // remove peers we are already connected to
+      if (connections.has(peer.id)) {
+        return false
+      }
+
+      return true
+    })
+
+    // shuffle the peers - this is so peers with the same tag values will be
+    // dialled in a different order each time
+    const shuffledPeers = filteredPeers.sort(() => Math.random() > 0.5 ? 1 : -1)
+
+    // Sort shuffled peers by tag value
+    const peerValues = new PeerMap<number>()
+    for (const peer of shuffledPeers) {
+      if (peerValues.has(peer.id)) {
+        continue
+      }
+
+      // sum all tag values
+      peerValues.set(peer.id, [...peer.tags.values()].reduce((acc, curr) => {
+        return acc + curr.value
+      }, 0))
+    }
+
+    // sort by value, highest to lowest
+    const sortedPeers = shuffledPeers.sort((a, b) => {
+      const peerAValue = peerValues.get(a.id) ?? 0
+      const peerBValue = peerValues.get(b.id) ?? 0
+
+      if (peerAValue > peerBValue) {
+        return -1
+      }
+
+      if (peerAValue < peerBValue) {
+        return 1
+      }
+
+      return 0
+    })
+
+    log('selected %d/%d peers to dial', sortedPeers.length, peers.length)
+
+    for (const peer of sortedPeers) {
+      this.queue.add(async () => {
+        const numConnections = this.connectionManager.getConnectionsMap().size
+
+        // Check to see if we still need to auto dial
+        if (numConnections >= this.minConnections) {
+          log('got enough connections now %d/%d', numConnections, this.minConnections)
+          this.queue.clear()
+          return
+        }
+
+        log('connecting to a peerStore stored peer %p', peer.id)
+        await this.connectionManager.openConnection(peer.id, {
+          // @ts-expect-error needs adding to the ConnectionManager interface
+          priority: this.autoDialPriority
+        })
+      }).catch(err => {
+        log.error('could not connect to peerStore stored peer', err)
+      })
+    }
+  }
+}

package/src/connection-manager/connection-pruner.ts

@@ -0,0 +1,157 @@
+import { logger } from '@libp2p/logger'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
+import type { PeerStore } from '@libp2p/interface-peer-store'
+import { PeerMap } from '@libp2p/peer-collections'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import { MAX_CONNECTIONS } from './constants.js'
+import type { EventEmitter } from '@libp2p/interfaces/events'
+import type { Libp2pEvents } from '@libp2p/interface-libp2p'
+
+const log = logger('libp2p:connection-manager:connection-pruner')
+
+interface ConnectionPrunerInit {
+  maxConnections?: number
+  allow?: Multiaddr[]
+}
+
+interface ConnectionPrunerComponents {
+  connectionManager: ConnectionManager
+  peerStore: PeerStore
+  events: EventEmitter<Libp2pEvents>
+}
+
+const defaultOptions = {
+  maxConnections: MAX_CONNECTIONS,
+  allow: []
+}
+
+/**
+ * If we go over the max connections limit, choose some connections to close
+ */
+export class ConnectionPruner {
+  private readonly maxConnections: number
+  private readonly connectionManager: ConnectionManager
+  private readonly peerStore: PeerStore
+  private readonly allow: Multiaddr[]
+  private readonly events: EventEmitter<Libp2pEvents>
+
+  constructor (components: ConnectionPrunerComponents, init: ConnectionPrunerInit = {}) {
+    this.maxConnections = init.maxConnections ?? defaultOptions.maxConnections
+    this.allow = init.allow ?? defaultOptions.allow
+    this.connectionManager = components.connectionManager
+    this.peerStore = components.peerStore
+    this.events = components.events
+
+    // check the max connection limit whenever a peer connects
+    components.events.addEventListener('connection:open', () => {
+      this.maybePruneConnections()
+        .catch(err => {
+          log.error(err)
+        })
+    })
+  }
+
+  /**
+   * If we have more connections than our maximum, select some excess connections
+   * to prune based on peer value
+   */
+  async maybePruneConnections (): Promise<void> {
+    const connections = this.connectionManager.getConnections()
+    const numConnections = connections.length
+    const toPrune = Math.max(numConnections - this.maxConnections, 0)
+
+    log('checking max connections limit %d/%d', numConnections, this.maxConnections)
+    if (numConnections <= this.maxConnections) {
+      return
+    }
+
+    log('max connections limit exceeded %d/%d, pruning %d connection(s)', numConnections, this.maxConnections, toPrune)
+    const peerValues = new PeerMap<number>()
+
+    // work out peer values
+    for (const connection of connections) {
+      const remotePeer = connection.remotePeer
+
+      if (peerValues.has(remotePeer)) {
+        continue
+      }
+
+      peerValues.set(remotePeer, 0)
+
+      try {
+        const peer = await this.peerStore.get(remotePeer)
+
+        // sum all tag values
+        peerValues.set(remotePeer, [...peer.tags.values()].reduce((acc, curr) => {
+          return acc + curr.value
+        }, 0))
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          log.error('error loading peer tags', err)
+        }
+      }
+    }
+
+    // sort by value, lowest to highest
+    const sortedConnections = connections.sort((a, b) => {
+      const peerAValue = peerValues.get(a.remotePeer) ?? 0
+      const peerBValue = peerValues.get(b.remotePeer) ?? 0
+
+      if (peerAValue > peerBValue) {
+        return 1
+      }
+
+      if (peerAValue < peerBValue) {
+        return -1
+      }
+
+      // if the peers have an equal tag value then we want to close short-lived connections first
+      const connectionALifespan = a.stat.timeline.open
+      const connectionBLifespan = b.stat.timeline.open
+
+      if (connectionALifespan < connectionBLifespan) {
+        return 1
+      }
+
+      if (connectionALifespan > connectionBLifespan) {
+        return -1
+      }
+
+      return 0
+    })
+
+    // close some connections
+    const toClose = []
+
+    for (const connection of sortedConnections) {
+      log('too many connections open - closing a connection to %p', connection.remotePeer)
+      // check allow list
+      const connectionInAllowList = this.allow.some((ma) => {
+        return connection.remoteAddr.toString().startsWith(ma.toString())
+      })
+
+      // Connections in the allow list should be excluded from pruning
+      if (!connectionInAllowList) {
+        toClose.push(connection)
+      }
+
+      if (toClose.length === toPrune) {
+        break
+      }
+    }
+
+    // close connections
+    await Promise.all(
+      toClose.map(async connection => {
+        try {
+          await connection.close()
+        } catch (err) {
+          log.error(err)
+        }
+      })
+    )
+
+    // despatch prune event
+    this.events.safeDispatchEvent('connection:prune', { detail: toClose })
+  }
+}

package/src/connection-manager/constants.ts

@@ -0,0 +1,60 @@
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#dialTimeout
+ */
+export const DIAL_TIMEOUT = 30e3
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#inboundUpgradeTimeout
+ */
+export const INBOUND_UPGRADE_TIMEOUT = 30e3
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxParallelDials
+ */
+export const MAX_PARALLEL_DIALS = 100
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxPeerAddrsToDial
+ */
+export const MAX_PEER_ADDRS_TO_DIAL = 25
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxParallelDialsPerPeer
+ */
+export const MAX_PARALLEL_DIALS_PER_PEER = 10
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#minConnections
+ */
+export const MIN_CONNECTIONS = 50
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxConnections
+ */
+export const MAX_CONNECTIONS = 300
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialInterval
+ */
+export const AUTO_DIAL_INTERVAL = 5000
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialConcurrency
+ */
+export const AUTO_DIAL_CONCURRENCY = 25
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialPriority
+ */
+export const AUTO_DIAL_PRIORITY = 0
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#inboundConnectionThreshold
+ */
+export const INBOUND_CONNECTION_THRESHOLD = 5
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxIncomingPendingConnections
+ */
+export const MAX_INCOMING_PENDING_CONNECTIONS = 10