libp2p 0.43.4 → 0.44.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libp2p",
-  "version": "0.43.4",
+  "version": "0.44.0",
   "description": "JavaScript implementation of libp2p, a modular peer to peer network stack",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/libp2p/js-libp2p#readme",
@@ -102,10 +102,10 @@
     "@achingbrain/nat-port-mapper": "^1.0.3",
     "@libp2p/crypto": "^1.0.4",
     "@libp2p/interface-address-manager": "^2.0.0",
-    "@libp2p/interface-connection": "^3.0.2",
+    "@libp2p/interface-connection": "^4.0.0",
     "@libp2p/interface-connection-encrypter": "^3.0.1",
-    "@libp2p/interface-connection-gater": "^1.0.0",
-    "@libp2p/interface-connection-manager": "^1.1.1",
+    "@libp2p/interface-connection-gater": "^2.0.1",
+    "@libp2p/interface-connection-manager": "^2.1.0",
     "@libp2p/interface-content-routing": "^2.0.0",
     "@libp2p/interface-dht": "^2.0.0",
     "@libp2p/interface-keychain": "^2.0.4",
@@ -136,21 +136,21 @@
     "@multiformats/mafmt": "^12.0.0",
     "@multiformats/multiaddr": "^12.0.0",
     "abortable-iterator": "^4.0.2",
-    "any-signal": "^3.0.0",
+    "any-signal": "^4.1.1",
     "datastore-core": "^9.0.0",
     "interface-datastore": "^8.0.0",
-    "it-all": "^2.0.0",
-    "it-drain": "^2.0.0",
-    "it-filter": "^2.0.0",
-    "it-first": "^2.0.0",
-    "it-parallel": "^3.0.0",
+    "it-all": "^3.0.1",
+    "it-drain": "^3.0.1",
+    "it-filter": "^3.0.1",
+    "it-first": "^3.0.1",
     "it-handshake": "^4.1.2",
-    "it-length-prefixed": "^8.0.2",
-    "it-map": "^2.0.0",
-    "it-merge": "^2.0.0",
+    "it-length-prefixed": "^9.0.0",
+    "it-map": "^3.0.2",
+    "it-merge": "^3.0.0",
     "it-pair": "^2.0.2",
+    "it-parallel": "^3.0.0",
     "it-pb-stream": "^3.2.0",
-    "it-pipe": "^2.0.3",
+    "it-pipe": "^3.0.1",
     "it-stream-types": "^1.0.4",
     "merge-options": "^3.0.4",
     "multiformats": "^11.0.0",
@@ -172,7 +172,7 @@
   "devDependencies": {
     "@chainsafe/libp2p-gossipsub": "^6.2.0",
     "@chainsafe/libp2p-noise": "^11.0.0",
-    "@chainsafe/libp2p-yamux": "^3.0.3",
+    "@chainsafe/libp2p-yamux": "^3.0.8",
     "@libp2p/bootstrap": "^6.0.0",
     "@libp2p/daemon-client": "^5.0.0",
     "@libp2p/daemon-server": "^4.1.0",
@@ -182,9 +182,9 @@
     "@libp2p/interface-connection-encrypter-compliance-tests": "^4.0.0",
     "@libp2p/interface-mocks": "^9.0.0",
     "@libp2p/interop": "^7.0.0",
-    "@libp2p/kad-dht": "^8.0.0",
+    "@libp2p/kad-dht": "^8.0.7",
     "@libp2p/mdns": "^7.0.0",
-    "@libp2p/mplex": "^7.0.0",
+    "@libp2p/mplex": "^7.1.6",
     "@libp2p/pubsub": "^6.0.0",
     "@libp2p/tcp": "^6.0.0",
     "@libp2p/websockets": "^5.0.0",
@@ -197,7 +197,7 @@
     "execa": "^7.0.0",
     "go-libp2p": "^1.0.1",
     "it-pushable": "^3.0.0",
-    "it-to-buffer": "^3.0.1",
+    "it-to-buffer": "^4.0.1",
     "npm-run-all": "^4.1.5",
     "p-event": "^5.0.1",
     "p-times": "^4.0.0",

package/src/autonat/index.ts

@@ -1,6 +1,6 @@
 import type { AddressManager } from '@libp2p/interface-address-manager'
 import type { Connection } from '@libp2p/interface-connection'
-import type { ConnectionManager, Dialer } from '@libp2p/interface-connection-manager'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
 import type { PeerId } from '@libp2p/interface-peer-id'
 import type { PeerInfo } from '@libp2p/interface-peer-info'
 import type { PeerRouting } from '@libp2p/interface-peer-routing'
@@ -74,7 +74,6 @@ export interface DefaultAutonatComponents {
   registrar: Registrar
   addressManager: AddressManager
   transportManager: TransportManager
-  dialer: Dialer
   peerId: PeerId
   connectionManager: ConnectionManager
   peerRouting: PeerRouting
@@ -151,7 +150,7 @@ export class AutonatService implements Startable {
 
       await pipe(
         source,
-        lp.decode(),
+        (source) => lp.decode(source),
         async function * (stream) {
           const buf = await first(stream)
 
@@ -319,9 +318,7 @@ export class AutonatService implements Startable {
             lastMultiaddr = multiaddr
 
             try {
-              // use the dialer so we can dial a specific multiaddr instead of every known
-              // multiaddr for the peer
-              connection = await self.components.dialer.dial(multiaddr, {
+              connection = await self.components.connectionManager.openConnection(multiaddr, {
                 signal: controller.signal
               })
 
@@ -360,7 +357,7 @@ export class AutonatService implements Startable {
             }
           })
         },
-        lp.encode(),
+        (source) => lp.encode(source),
         // pipe to the stream, not the abortable source other wise we
         // can't tell the remote when a dial timed out..
         data.stream
@@ -449,9 +446,9 @@ export class AutonatService implements Startable {
 
           const buf = await pipe(
             [request],
-            lp.encode(),
+            (source) => lp.encode(source),
             source,
-            lp.decode(),
+            (source) => lp.decode(source),
             async (stream) => await first(stream)
           )
           if (buf == null) {

package/src/circuit-relay/transport/reservation-store.ts

@@ -10,7 +10,7 @@ import type { Reservation } from '../pb/index.js'
 import { HopMessage, Status } from '../pb/index.js'
 import { getExpirationMilliseconds } from '../utils.js'
 import type { TransportManager } from '@libp2p/interface-transport'
-import type { Startable } from '@libp2p/interfaces/dist/src/startable.js'
+import type { Startable } from '@libp2p/interfaces/startable'
 import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
 import type { PeerStore } from '@libp2p/interface-peer-store'
 import { multiaddr } from '@multiformats/multiaddr'

package/src/circuit-relay/utils.ts

@@ -7,7 +7,7 @@ import { logger } from '@libp2p/logger'
 import type { Stream } from '@libp2p/interface-connection'
 import { DEFAULT_DATA_LIMIT, DEFAULT_DURATION_LIMIT } from './constants.js'
 import { abortableSource } from 'abortable-iterator'
-import anySignal from 'any-signal'
+import { anySignal } from 'any-signal'
 
 const log = logger('libp2p:circuit-relay:utils')
 
@@ -69,6 +69,7 @@ const doRelay = (src: Stream, dst: Stream, abortSignal: AbortSignal, limit: Requ
         srcDstFinished = true
 
         if (dstSrcFinished) {
+          signal.clear()
           clearTimeout(timeout)
         }
       })
@@ -86,6 +87,7 @@ const doRelay = (src: Stream, dst: Stream, abortSignal: AbortSignal, limit: Requ
         dstSrcFinished = true
 
         if (srcDstFinished) {
+          signal.clear()
           clearTimeout(timeout)
         }
       })

package/src/components.ts

@@ -12,7 +12,7 @@ import type { TransportManager, Upgrader } from '@libp2p/interface-transport'
 import type { Datastore } from 'interface-datastore'
 import type { PubSub } from '@libp2p/interface-pubsub'
 import type { DualDHT } from '@libp2p/interface-dht'
-import type { ConnectionManager, Dialer } from '@libp2p/interface-connection-manager'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
 import type { ConnectionGater } from '@libp2p/interface-connection-gater'
 
 export interface Components {
@@ -28,7 +28,6 @@ export interface Components {
   peerRouting: PeerRouting
   datastore: Datastore
   connectionProtector?: ConnectionProtector
-  dialer: Dialer
   metrics?: Metrics
   dht?: DualDHT
   pubsub?: PubSub
@@ -50,7 +49,6 @@ export interface ComponentsInit {
   connectionProtector?: ConnectionProtector
   dht?: DualDHT
   pubsub?: PubSub
-  dialer?: Dialer
 }
 
 export class DefaultComponents implements Components, Startable {
@@ -69,7 +67,6 @@ export class DefaultComponents implements Components, Startable {
   private _connectionProtector?: ConnectionProtector
   private _dht?: DualDHT
   private _pubsub?: PubSub
-  private _dialer?: Dialer
   private _started = false
 
   constructor (init: ComponentsInit = {}) {
@@ -88,7 +85,6 @@ export class DefaultComponents implements Components, Startable {
     this._connectionProtector = init.connectionProtector
     this._dht = init.dht
     this._pubsub = init.pubsub
-    this._dialer = init.dialer
   }
 
   isStarted (): boolean {
@@ -295,18 +291,6 @@ export class DefaultComponents implements Components, Startable {
     this._connectionProtector = connectionProtector
   }
 
-  get dialer (): Dialer {
-    if (this._dialer == null) {
-      throw new CodeError('dialer not set', 'ERR_SERVICE_MISSING')
-    }
-
-    return this._dialer
-  }
-
-  set dialer (dialer: Dialer) {
-    this._dialer = dialer
-  }
-
   get metrics (): Metrics | undefined {
     return this._metrics
   }

package/src/config.ts

@@ -1,6 +1,5 @@
 import mergeOptions from 'merge-options'
 import { dnsaddrResolver } from '@multiformats/multiaddr/resolvers'
-import * as Constants from './constants.js'
 import { AGENT_VERSION } from './identify/consts.js'
 import { publicAddressesFirst } from '@libp2p/utils/address-sort'
 import { FaultTolerance } from '@libp2p/interface-transport'
@@ -19,19 +18,11 @@ const DefaultConfig: Partial<Libp2pInit> = {
     announceFilter: (multiaddrs: Multiaddr[]) => multiaddrs
   },
   connectionManager: {
-    maxConnections: 300,
-    minConnections: 50,
-    autoDialInterval: 10000,
-    maxParallelDials: Constants.MAX_PARALLEL_DIALS,
-    maxDialsPerPeer: Constants.MAX_PER_PEER_DIALS,
-    dialTimeout: Constants.DIAL_TIMEOUT,
-    inboundUpgradeTimeout: Constants.INBOUND_UPGRADE_TIMEOUT,
     resolvers: {
       dnsaddr: dnsaddrResolver
     },
     addressSorter: publicAddressesFirst
   },
-  connectionGater: {},
   transportManager: {
     faultTolerance: FaultTolerance.FATAL_ALL
   },

package/src/connection-manager/auto-dial.ts

@@ -0,0 +1,171 @@
+import { logger } from '@libp2p/logger'
+import type { PeerStore } from '@libp2p/interface-peer-store'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
+import { PeerMap } from '@libp2p/peer-collections'
+import PQueue from 'p-queue'
+import { AUTO_DIAL_CONCURRENCY, AUTO_DIAL_INTERVAL, AUTO_DIAL_PRIORITY, MIN_CONNECTIONS } from './constants.js'
+import type { Startable } from '@libp2p/interfaces/startable'
+
+const log = logger('libp2p:connection-manager:auto-dial')
+
+interface AutoDialInit {
+  minConnections?: number
+  autoDialConcurrency?: number
+  autoDialPriority?: number
+  autoDialInterval?: number
+}
+
+interface AutoDialComponents {
+  connectionManager: ConnectionManager
+  peerStore: PeerStore
+}
+
+const defaultOptions = {
+  minConnections: MIN_CONNECTIONS,
+  autoDialConcurrency: AUTO_DIAL_CONCURRENCY,
+  autoDialPriority: AUTO_DIAL_PRIORITY,
+  autoDialInterval: AUTO_DIAL_INTERVAL
+}
+
+export class AutoDial implements Startable {
+  private readonly connectionManager: ConnectionManager
+  private readonly peerStore: PeerStore
+  private readonly queue: PQueue
+  private readonly minConnections: number
+  private readonly autoDialPriority: number
+  private readonly autoDialIntervalMs: number
+  private autoDialInterval?: ReturnType<typeof setInterval>
+  private started: boolean
+
+  /**
+   * Proactively tries to connect to known peers stored in the PeerStore.
+   * It will keep the number of connections below the upper limit and sort
+   * the peers to connect based on whether we know their keys and protocols.
+   */
+  constructor (components: AutoDialComponents, init: AutoDialInit) {
+    this.connectionManager = components.connectionManager
+    this.peerStore = components.peerStore
+    this.minConnections = init.minConnections ?? defaultOptions.minConnections
+    this.autoDialPriority = init.autoDialPriority ?? defaultOptions.autoDialPriority
+    this.autoDialIntervalMs = init.autoDialInterval ?? defaultOptions.autoDialInterval
+    this.started = false
+    this.queue = new PQueue({
+      concurrency: init.autoDialConcurrency ?? defaultOptions.autoDialConcurrency
+    })
+    this.queue.addListener('error', (err) => {
+      log.error('error during auto-dial', err)
+    })
+  }
+
+  isStarted (): boolean {
+    return this.started
+  }
+
+  start (): void {
+    this.autoDialInterval = setInterval(() => {
+      this.autoDial()
+        .catch(err => {
+          log.error('error while autodialing', err)
+        })
+    }, this.autoDialIntervalMs)
+    this.started = true
+  }
+
+  afterStart (): void {
+    this.autoDial()
+      .catch(err => {
+        log.error('error while autodialing', err)
+      })
+  }
+
+  stop (): void {
+    // clear the queue
+    this.queue.clear()
+    clearInterval(this.autoDialInterval)
+    this.started = false
+  }
+
+  async autoDial (): Promise<void> {
+    const numConnections = this.connectionManager.getConnections().length
+
+    // Already has enough connections
+    if (numConnections >= this.minConnections) {
+      log('have enough connections %d/%d', numConnections, this.minConnections)
+      return
+    }
+
+    log('not enough connections %d/%d - will dial peers to increase the number of connections', numConnections, this.minConnections)
+
+    // Sort peers on whether we know protocols or public keys for them
+    let peers = await this.peerStore.all()
+
+    log('loaded %d peers from the peer store', peers.length)
+
+    // Remove some peers
+    peers = peers.filter((peer) => {
+      // Remove peers without addresses
+      if (peer.addresses.length === 0) {
+        return false
+      }
+
+      return true
+    })
+
+    // shuffle the peers
+    peers = peers.sort(() => Math.random() > 0.5 ? 1 : -1)
+
+    // Sort shuffled peers by tag value
+    const peerValues = new PeerMap<number>()
+    for (const peer of peers) {
+      if (peerValues.has(peer.id)) {
+        continue
+      }
+
+      const tags = await this.peerStore.getTags(peer.id)
+
+      // sum all tag values
+      peerValues.set(peer.id, (tags ?? []).reduce((acc, curr) => {
+        return acc + curr.value
+      }, 0))
+    }
+
+    // sort by value, highest to lowest
+    const sortedPeers = peers.sort((a, b) => {
+      const peerAValue = peerValues.get(a.id) ?? 0
+      const peerBValue = peerValues.get(b.id) ?? 0
+
+      if (peerAValue > peerBValue) {
+        return -1
+      }
+
+      if (peerAValue < peerBValue) {
+        return 1
+      }
+
+      return 0
+    })
+
+    log('selected %d peers to dial', sortedPeers.length)
+
+    for (const peer of sortedPeers) {
+      this.queue.add(async () => {
+        const numConnections = this.connectionManager.getConnections().length
+
+        // Check to see if we still need to auto dial
+        if (numConnections >= this.minConnections) {
+          log('got enough connections now %d/%d', numConnections, this.minConnections)
+          this.queue.clear()
+          return
+        }
+
+        log('connecting to a peerStore stored peer %p', peer.id)
+        await this.connectionManager.openConnection(peer.id, {
+          // @ts-expect-error needs adding to the ConnectionManager interface
+          priority: this.autoDialPriority
+        })
+      }).catch(err => {
+        log.error('could not connect to peerStore stored peer', err)
+      })
+    }
+  }
+}

package/src/connection-manager/connection-pruner.ts

@@ -0,0 +1,137 @@
+import { logger } from '@libp2p/logger'
+import type { ConnectionManager } from '@libp2p/interface-connection-manager'
+import type { PeerStore } from '@libp2p/interface-peer-store'
+import { PeerMap } from '@libp2p/peer-collections'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import { MAX_CONNECTIONS } from './constants.js'
+import { CustomEvent } from '@libp2p/interfaces/events'
+
+const log = logger('libp2p:connection-manager:connection-pruner')
+
+interface ConnectionPrunerInit {
+  maxConnections?: number
+  allow?: Multiaddr[]
+}
+
+interface ConnectionPrunerComponents {
+  connectionManager: ConnectionManager
+  peerStore: PeerStore
+}
+
+const defaultOptions = {
+  maxConnections: MAX_CONNECTIONS,
+  allow: []
+}
+
+/**
+ * If we go over the max connections limit, choose some connections to close
+ */
+export class ConnectionPruner {
+  private readonly maxConnections: number
+  private readonly connectionManager: ConnectionManager
+  private readonly peerStore: PeerStore
+  private readonly allow: Multiaddr[]
+
+  constructor (components: ConnectionPrunerComponents, init: ConnectionPrunerInit = {}) {
+    this.maxConnections = init.maxConnections ?? defaultOptions.maxConnections
+    this.allow = init.allow ?? defaultOptions.allow
+    this.connectionManager = components.connectionManager
+    this.peerStore = components.peerStore
+  }
+
+  /**
+   * If we have more connections than our maximum, select some excess connections
+   * to prune based on peer value
+   */
+  async maybePruneConnections (): Promise<void> {
+    const connections = this.connectionManager.getConnections()
+    const numConnections = connections.length
+    const toPrune = Math.max(numConnections - this.maxConnections, 0)
+
+    log.trace('checking max connections limit %d/%d', numConnections, this.maxConnections)
+    if (numConnections <= this.maxConnections) {
+      return
+    }
+
+    log('max connections limit exceeded %d/%d, pruning %d connection(s)', numConnections, this.maxConnections, toPrune)
+    const peerValues = new PeerMap<number>()
+
+    // work out peer values
+    for (const connection of connections) {
+      const remotePeer = connection.remotePeer
+
+      if (peerValues.has(remotePeer)) {
+        continue
+      }
+
+      const tags = await this.peerStore.getTags(remotePeer)
+
+      // sum all tag values
+      peerValues.set(remotePeer, tags.reduce((acc, curr) => {
+        return acc + curr.value
+      }, 0))
+    }
+
+    // sort by value, lowest to highest
+    const sortedConnections = connections.sort((a, b) => {
+      const peerAValue = peerValues.get(a.remotePeer) ?? 0
+      const peerBValue = peerValues.get(b.remotePeer) ?? 0
+
+      if (peerAValue > peerBValue) {
+        return 1
+      }
+
+      if (peerAValue < peerBValue) {
+        return -1
+      }
+
+      // if the peers have an equal tag value then we want to close short-lived connections first
+      const connectionALifespan = a.stat.timeline.open
+      const connectionBLifespan = b.stat.timeline.open
+
+      if (connectionALifespan < connectionBLifespan) {
+        return 1
+      }
+
+      if (connectionALifespan > connectionBLifespan) {
+        return -1
+      }
+
+      return 0
+    })
+
+    // close some connections
+    const toClose = []
+
+    for (const connection of sortedConnections) {
+      log('too many connections open - closing a connection to %p', connection.remotePeer)
+      // check allow list
+      const connectionInAllowList = this.allow.some((ma) => {
+        return connection.remoteAddr.toString().startsWith(ma.toString())
+      })
+
+      // Connections in the allow list should be excluded from pruning
+      if (!connectionInAllowList) {
+        toClose.push(connection)
+      }
+
+      if (toClose.length === toPrune) {
+        break
+      }
+    }
+
+    // close connections
+    await Promise.all(
+      toClose.map(async connection => {
+        try {
+          await connection.close()
+        } catch (err) {
+          log.error(err)
+        }
+      })
+    )
+
+    // despatch prune event
+    this.connectionManager.dispatchEvent(new CustomEvent('peer:prune', { detail: toClose.map(conn => conn.remotePeer) }))
+  }
+}

package/src/connection-manager/constants.ts

@@ -0,0 +1,60 @@
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#dialTimeout
+ */
+export const DIAL_TIMEOUT = 30e3
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#inboundUpgradeTimeout
+ */
+export const INBOUND_UPGRADE_TIMEOUT = 30e3
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxParallelDials
+ */
+export const MAX_PARALLEL_DIALS = 100
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxPeerAddrsToDial
+ */
+export const MAX_PEER_ADDRS_TO_DIAL = 25
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxParallelDialsPerPeer
+ */
+export const MAX_PARALLEL_DIALS_PER_PEER = 10
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#minConnections
+ */
+export const MIN_CONNECTIONS = 50
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxConnections
+ */
+export const MAX_CONNECTIONS = 300
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialInterval
+ */
+export const AUTO_DIAL_INTERVAL = 5000
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialConcurrency
+ */
+export const AUTO_DIAL_CONCURRENCY = 25
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#autoDialPriority
+ */
+export const AUTO_DIAL_PRIORITY = 0
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#inboundConnectionThreshold
+ */
+export const INBOUND_CONNECTION_THRESHOLD = 5
+
+/**
+ * @see https://libp2p.github.io/js-libp2p/interfaces/index._internal_.ConnectionManagerConfig.html#maxIncomingPendingConnections
+ */
+export const MAX_INCOMING_PENDING_CONNECTIONS = 10