libp2p 0.35.5 → 0.35.8-63aa480

package/src/circuit/auto-relay.js

@@ -8,7 +8,7 @@ const log = Object.assign(debug('libp2p:auto-relay'), {
 const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
 const { Multiaddr } = require('multiaddr')
-const PeerId = require('peer-id')
+const all = require('it-all')
 
 const { relay: multicodec } = require('./multicodec')
 const { canHop } = require('./circuit/hop')
@@ -22,7 +22,8 @@ const {
 
 /**
  * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection
- * @typedef {import('../peer-store/address-book').Address} Address
+ * @typedef {import('../peer-store/types').Address} Address
+ * @typedef {import('peer-id')} PeerId
  */
 
 /**
@@ -91,7 +92,7 @@ class AutoRelay {
 
     // If no protocol, check if we were keeping the peer before as a listenRelay
     if (!hasProtocol && this._listenRelays.has(id)) {
-      this._removeListenRelay(id)
+      await this._removeListenRelay(id)
       return
     } else if (!hasProtocol || this._listenRelays.has(id)) {
       return
@@ -113,7 +114,7 @@ class AutoRelay {
       const supportsHop = await canHop({ connection })
 
       if (supportsHop) {
-        this._peerStore.metadataBook.set(peerId, HOP_METADATA_KEY, uint8ArrayFromString(HOP_METADATA_VALUE))
+        await this._peerStore.metadataBook.setValue(peerId, HOP_METADATA_KEY, uint8ArrayFromString(HOP_METADATA_VALUE))
         await this._addListenRelay(connection, id)
       }
     } catch (/** @type {any} */ err) {
@@ -125,7 +126,6 @@ class AutoRelay {
    * Peer disconnects.
    *
    * @param {Connection} connection - connection to the peer
-   * @returns {void}
    */
   _onPeerDisconnected (connection) {
     const peerId = connection.remotePeer
@@ -136,7 +136,9 @@ class AutoRelay {
       return
     }
 
-    this._removeListenRelay(id)
+    this._removeListenRelay(id).catch(err => {
+      log.error(err)
+    })
   }
 
   /**
@@ -148,27 +150,35 @@ class AutoRelay {
    * @returns {Promise<void>}
    */
   async _addListenRelay (connection, id) {
-    // Check if already listening on enough relays
-    if (this._listenRelays.size >= this.maxListeners) {
-      return
-    }
-
-    // Get peer known addresses and sort them per public addresses first
-    const remoteAddrs = this._peerStore.addressBook.getMultiaddrsForPeer(
-      connection.remotePeer, this._addressSorter
-    )
-
-    if (!remoteAddrs || !remoteAddrs.length) {
-      return
-    }
-
-    const listenAddr = `${remoteAddrs[0].toString()}/p2p-circuit`
-    this._listenRelays.add(id)
-
-    // Attempt to listen on relay
     try {
-      await this._transportManager.listen([new Multiaddr(listenAddr)])
-      // Announce multiaddrs will update on listen success by TransportManager event being triggered
+      // Check if already listening on enough relays
+      if (this._listenRelays.size >= this.maxListeners) {
+        return
+      }
+
+      // Get peer known addresses and sort them per public addresses first
+      const remoteAddrs = await this._peerStore.addressBook.getMultiaddrsForPeer(
+        connection.remotePeer, this._addressSorter
+      )
+
+      // Attempt to listen on relay
+      const result = await Promise.all(
+        remoteAddrs.map(async addr => {
+          try {
+            // Announce multiaddrs will update on listen success by TransportManager event being triggered
+            await this._transportManager.listen([new Multiaddr(`${addr.toString()}/p2p-circuit`)])
+            return true
+          } catch (/** @type {any} */ err) {
+            this._onError(err)
+          }
+
+          return false
+        })
+      )
+
+      if (result.includes(true)) {
+        this._listenRelays.add(id)
+      }
     } catch (/** @type {any} */ err) {
       this._onError(err)
       this._listenRelays.delete(id)
@@ -180,12 +190,11 @@ class AutoRelay {
    *
    * @private
    * @param {string} id - peer identifier string.
-   * @returns {void}
    */
-  _removeListenRelay (id) {
+  async _removeListenRelay (id) {
     if (this._listenRelays.delete(id)) {
       // TODO: this should be responsibility of the connMgr
-      this._listenOnAvailableHopRelays([id])
+      await this._listenOnAvailableHopRelays([id])
     }
   }
 
@@ -197,7 +206,6 @@ class AutoRelay {
    * 3. Search the network.
    *
    * @param {string[]} [peersToIgnore]
-   * @returns {Promise<void>}
    */
   async _listenOnAvailableHopRelays (peersToIgnore = []) {
     // TODO: The peer redial issue on disconnect should be handled by connection gating
@@ -207,31 +215,37 @@ class AutoRelay {
     }
 
     const knownHopsToDial = []
+    const peers = await all(this._peerStore.getPeers())
 
     // Check if we have known hop peers to use and attempt to listen on the already connected
-    for (const [id, metadataMap] of this._peerStore.metadataBook.data.entries()) {
+    for await (const { id, metadata } of peers) {
+      const idStr = id.toB58String()
+
       // Continue to next if listening on this or peer to ignore
-      if (this._listenRelays.has(id) || peersToIgnore.includes(id)) {
+      if (this._listenRelays.has(idStr)) {
         continue
       }
 
-      const supportsHop = metadataMap.get(HOP_METADATA_KEY)
+      if (peersToIgnore.includes(idStr)) {
+        continue
+      }
+
+      const supportsHop = metadata.get(HOP_METADATA_KEY)
 
       // Continue to next if it does not support Hop
       if (!supportsHop || uint8ArrayToString(supportsHop) !== HOP_METADATA_VALUE) {
         continue
       }
 
-      const peerId = PeerId.createFromB58String(id)
-      const connection = this._connectionManager.get(peerId)
+      const connection = this._connectionManager.get(id)
 
       // If not connected, store for possible later use.
       if (!connection) {
-        knownHopsToDial.push(peerId)
+        knownHopsToDial.push(id)
         continue
       }
 
-      await this._addListenRelay(connection, id)
+      await this._addListenRelay(connection, idStr)
 
       // Check if already listening on enough relays
       if (this._listenRelays.size >= this.maxListeners) {
@@ -258,7 +272,7 @@ class AutoRelay {
         }
 
         const peerId = provider.id
-        this._peerStore.addressBook.add(peerId, provider.multiaddrs)
+        await this._peerStore.addressBook.add(peerId, provider.multiaddrs)
 
         await this._tryToListenOnRelay(peerId)
 

package/src/connection-manager/auto-dialler.js

@@ -4,6 +4,7 @@ const debug = require('debug')
 const mergeOptions = require('merge-options')
 // @ts-ignore retimer does not have types
 const retimer = require('retimer')
+const all = require('it-all')
 
 const log = Object.assign(debug('libp2p:connection-manager:auto-dialler'), {
   error: debug('libp2p:connection-manager:auto-dialler:err')
@@ -50,14 +51,16 @@ class AutoDialler {
   /**
    * Starts the auto dialer
    */
-  start () {
+  async start () {
     if (!this._options.enabled) {
       log('not enabled')
       return
     }
 
     this._running = true
-    this._autoDial()
+    this._autoDial().catch(err => {
+      log.error('could start autodial', err)
+    })
     log('started')
   }
 
@@ -84,8 +87,10 @@ class AutoDialler {
       return
     }
 
-    // Sort peers on wether we know protocols of public keys for them
-    const peers = Array.from(this._libp2p.peerStore.peers.values())
+    // Sort peers on whether we know protocols of public keys for them
+    // TODO: assuming the `peerStore.getPeers()` order is stable this will mean
+    // we keep trying to connect to the same peers?
+    const peers = (await all(this._libp2p.peerStore.getPeers()))
       .sort((a, b) => {
         if (b.protocols && b.protocols.length && (!a.protocols || !a.protocols.length)) {
           return 1

package/src/connection-manager/index.js

@@ -12,7 +12,7 @@ const LatencyMonitor = require('./latency-monitor')
 const retimer = require('retimer')
 
 const { EventEmitter } = require('events')
-
+const trackedMap = require('../metrics/tracked-map')
 const PeerId = require('peer-id')
 
 const {
@@ -34,7 +34,7 @@ const defaultOptions = {
 
 const METRICS_COMPONENT = 'connection-manager'
 const METRICS_PEER_CONNECTIONS = 'peer-connections'
-const METRICS_ALL_CONNECTIONS = 'all-connections'
+const METRICS_PEER_VALUES = 'peer-values'
 
 /**
  * @typedef {import('../')} Libp2p
@@ -87,14 +87,22 @@ class ConnectionManager extends EventEmitter {
      *
      * @type {Map<string, number>}
      */
-    this._peerValues = new Map()
+    this._peerValues = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PEER_VALUES,
+      metrics: this._libp2p.metrics
+    })
 
     /**
      * Map of connections per peer
      *
      * @type {Map<string, Connection[]>}
      */
-    this.connections = new Map()
+    this.connections = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PEER_CONNECTIONS,
+      metrics: this._libp2p.metrics
+    })
 
     this._started = false
     this._timer = null
@@ -104,6 +112,9 @@ class ConnectionManager extends EventEmitter {
       latencyCheckIntervalMs: this._options.pollInterval,
       dataEmitIntervalMs: this._options.pollInterval
     })
+
+    // This emitter gets listened to a lot
+    this.setMaxListeners(Infinity)
   }
 
   /**
@@ -164,8 +175,6 @@ class ConnectionManager extends EventEmitter {
 
     await Promise.all(tasks)
     this.connections.clear()
-    this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PEER_CONNECTIONS, 0)
-    this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_ALL_CONNECTIONS, 0)
   }
 
   /**
@@ -189,19 +198,22 @@ class ConnectionManager extends EventEmitter {
    *
    * @private
    */
-  _checkMetrics () {
+  async _checkMetrics () {
     if (this._libp2p.metrics) {
-      const movingAverages = this._libp2p.metrics.global.movingAverages
-      // @ts-ignore moving averages object types
-      const received = movingAverages.dataReceived[this._options.movingAverageInterval].movingAverage()
-      this._checkMaxLimit('maxReceivedData', received)
-      // @ts-ignore moving averages object types
-      const sent = movingAverages.dataSent[this._options.movingAverageInterval].movingAverage()
-      this._checkMaxLimit('maxSentData', sent)
-      const total = received + sent
-      this._checkMaxLimit('maxData', total)
-      log('metrics update', total)
-      this._timer = retimer(this._checkMetrics, this._options.pollInterval)
+      try {
+        const movingAverages = this._libp2p.metrics.global.movingAverages
+        // @ts-ignore moving averages object types
+        const received = movingAverages.dataReceived[this._options.movingAverageInterval].movingAverage()
+        await this._checkMaxLimit('maxReceivedData', received)
+        // @ts-ignore moving averages object types
+        const sent = movingAverages.dataSent[this._options.movingAverageInterval].movingAverage()
+        await this._checkMaxLimit('maxSentData', sent)
+        const total = received + sent
+        await this._checkMaxLimit('maxData', total)
+        log('metrics update', total)
+      } finally {
+        this._timer = retimer(this._checkMetrics, this._options.pollInterval)
+      }
     }
   }
 
@@ -209,9 +221,8 @@ class ConnectionManager extends EventEmitter {
    * Tracks the incoming connection and check the connection limit
    *
    * @param {Connection} connection
-   * @returns {void}
    */
-  onConnect (connection) {
+  async onConnect (connection) {
     const peerId = connection.remotePeer
     const peerIdStr = peerId.toB58String()
     const storedConn = this.connections.get(peerIdStr)
@@ -222,17 +233,15 @@ class ConnectionManager extends EventEmitter {
       storedConn.push(connection)
     } else {
       this.connections.set(peerIdStr, [connection])
-      this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PEER_CONNECTIONS, this.connections.size)
-      this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_ALL_CONNECTIONS, this.size)
     }
 
-    this._libp2p.peerStore.keyBook.set(peerId, peerId.pubKey)
+    await this._libp2p.peerStore.keyBook.set(peerId, peerId.pubKey)
 
     if (!this._peerValues.has(peerIdStr)) {
       this._peerValues.set(peerIdStr, this._options.defaultPeerValue)
     }
 
-    this._checkMaxLimit('maxConnections', this.size)
+    await this._checkMaxLimit('maxConnections', this.size)
   }
 
   /**
@@ -255,9 +264,6 @@ class ConnectionManager extends EventEmitter {
 
       this._libp2p.metrics && this._libp2p.metrics.onPeerDisconnected(connection.remotePeer)
     }
-
-    this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PEER_CONNECTIONS, this.connections.size)
-    this._libp2p.metrics && this._libp2p.metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_ALL_CONNECTIONS, this.size)
   }
 
   /**
@@ -303,6 +309,9 @@ class ConnectionManager extends EventEmitter {
    */
   _onLatencyMeasure (summary) {
     this._checkMaxLimit('maxEventLoopDelay', summary.avgMs)
+      .catch(err => {
+        log.error(err)
+      })
   }
 
   /**
@@ -312,12 +321,12 @@ class ConnectionManager extends EventEmitter {
    * @param {string} name - The name of the field to check limits for
    * @param {number} value - The current value of the field
    */
-  _checkMaxLimit (name, value) {
+  async _checkMaxLimit (name, value) {
     const limit = this._options[name]
     log('checking limit of %s. current value: %d of %d', name, value, limit)
     if (value > limit) {
       log('%s: limit exceeded: %s, %d', this._peerId, name, value)
-      this._maybeDisconnectOne()
+      await this._maybeDisconnectOne()
     }
   }
 
@@ -327,7 +336,7 @@ class ConnectionManager extends EventEmitter {
    *
    * @private
    */
-  _maybeDisconnectOne () {
+  async _maybeDisconnectOne () {
     if (this._options.minConnections < this.connections.size) {
       const peerValues = Array.from(new Map([...this._peerValues.entries()].sort((a, b) => a[1] - b[1])))
       log('%s: sorted peer values: %j', this._peerId, peerValues)
@@ -338,7 +347,11 @@ class ConnectionManager extends EventEmitter {
         log('%s: closing a connection to %j', this._peerId, peerId)
         for (const connections of this.connections.values()) {
           if (connections[0].remotePeer.toB58String() === peerId) {
-            connections[0].close()
+            connections[0].close().catch(err => {
+              log.error(err)
+            })
+            // TODO: should not need to invoke this manually
+            this.onDisconnect(connections[0])
             break
           }
         }

package/src/content-routing/utils.js

@@ -14,12 +14,12 @@ const take = require('it-take')
  * Store the multiaddrs from every peer in the passed peer store
  *
  * @param {AsyncIterable<{ id: PeerId, multiaddrs: Multiaddr[] }>} source
- * @param {import('../peer-store')} peerStore
+ * @param {import('../peer-store/types').PeerStore} peerStore
  */
-function storeAddresses (source, peerStore) {
-  return map(source, (peer) => {
+async function * storeAddresses (source, peerStore) {
+  yield * map(source, async (peer) => {
     // ensure we have the addresses for a given peer
-    peerStore.addressBook.add(peer.id, peer.multiaddrs)
+    await peerStore.addressBook.add(peer.id, peer.multiaddrs)
 
     return peer
   })

package/src/dht/dht-peer-routing.js

@@ -27,12 +27,8 @@ class DHTPeerRouting {
    */
   async findPeer (peerId, options = {}) {
     for await (const event of this._dht.findPeer(peerId, options)) {
-      if (event.name === 'PEER_RESPONSE') {
-        const peer = event.closer.find(peerData => peerData.id.equals(peerId))
-
-        if (peer) {
-          return peer
-        }
+      if (event.name === 'FINAL_PEER') {
+        return event.peer
       }
     }
 

package/src/dialer/dial-request.js

@@ -63,7 +63,10 @@ class DialRequest {
     tokens.forEach(token => tokenHolder.push(token))
     const dialAbortControllers = this.addrs.map(() => {
       const controller = new AbortController()
-      setMaxListeners && setMaxListeners(Infinity, controller.signal)
+      try {
+        // fails on node < 15.4
+        setMaxListeners && setMaxListeners(Infinity, controller.signal)
+      } catch {}
 
       return controller
     })

package/src/dialer/index.js

@@ -9,11 +9,12 @@ const { Multiaddr } = require('multiaddr')
 const { TimeoutController } = require('timeout-abort-controller')
 const { AbortError } = require('abortable-iterator')
 const { anySignal } = require('any-signal')
-
+// @ts-expect-error setMaxListeners is missing from the types
+const { setMaxListeners } = require('events')
 const DialRequest = require('./dial-request')
 const { publicAddressesFirst } = require('libp2p-utils/src/address-sort')
 const getPeer = require('../get-peer')
-
+const trackedMap = require('../metrics/tracked-map')
 const { codes } = require('../errors')
 const {
   DIAL_TIMEOUT,
@@ -29,8 +30,8 @@ const METRICS_PENDING_DIAL_TARGETS = 'pending-dial-targets'
 /**
  * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection
  * @typedef {import('peer-id')} PeerId
- * @typedef {import('../peer-store')} PeerStore
- * @typedef {import('../peer-store/address-book').Address} Address
+ * @typedef {import('../peer-store/types').PeerStore} PeerStore
+ * @typedef {import('../peer-store/types').Address} Address
  * @typedef {import('../transport-manager')} TransportManager
  */
 
@@ -55,8 +56,8 @@ const METRICS_PENDING_DIAL_TARGETS = 'pending-dial-targets'
  * @property {Multiaddr[]} addrs
  *
  * @typedef PendingDial
- * @property {DialRequest} dialRequest
- * @property {TimeoutController} controller
+ * @property {import('./dial-request')} dialRequest
+ * @property {import('timeout-abort-controller').TimeoutController} controller
  * @property {Promise<Connection>} promise
  * @property {function():void} destroy
  */
@@ -85,9 +86,20 @@ class Dialer {
     this.timeout = dialTimeout
     this.maxDialsPerPeer = maxDialsPerPeer
     this.tokens = [...new Array(maxParallelDials)].map((_, index) => index)
-    this._pendingDials = new Map()
-    this._pendingDialTargets = new Map()
-    this._metrics = metrics
+
+    /** @type {Map<string, PendingDial>} */
+    this._pendingDials = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PENDING_DIALS,
+      metrics
+    })
+
+    /** @type {Map<string, { resolve: (value: any) => void, reject: (err: Error) => void}>} */
+    this._pendingDialTargets = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PENDING_DIAL_TARGETS,
+      metrics
+    })
 
     for (const [key, value] of Object.entries(resolvers)) {
       Multiaddr.resolvers.set(key, value)
@@ -111,9 +123,6 @@ class Dialer {
       pendingTarget.reject(new AbortError('Dialer was destroyed'))
     }
     this._pendingDialTargets.clear()
-
-    this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIALS, 0)
-    this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIAL_TARGETS, 0)
   }
 
   /**
@@ -163,7 +172,6 @@ class Dialer {
     const id = `${(parseInt(String(Math.random() * 1e9), 10)).toString() + Date.now()}`
     const cancellablePromise = new Promise((resolve, reject) => {
       this._pendingDialTargets.set(id, { resolve, reject })
-      this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIAL_TARGETS, this._pendingDialTargets.size)
     })
 
     try {
@@ -175,7 +183,6 @@ class Dialer {
       return dialTarget
     } finally {
       this._pendingDialTargets.delete(id)
-      this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIAL_TARGETS, this._pendingDialTargets.size)
     }
   }
 
@@ -193,10 +200,10 @@ class Dialer {
     const { id, multiaddrs } = getPeer(peer)
 
     if (multiaddrs) {
-      this.peerStore.addressBook.add(id, multiaddrs)
+      await this.peerStore.addressBook.add(id, multiaddrs)
     }
 
-    let knownAddrs = this.peerStore.addressBook.getMultiaddrsForPeer(id, this.addressSorter) || []
+    let knownAddrs = await this.peerStore.addressBook.getMultiaddrsForPeer(id, this.addressSorter) || []
 
     // If received a multiaddr to dial, it should be the first to use
     // But, if we know other multiaddrs for the peer, we should try them too.
@@ -216,7 +223,7 @@ class Dialer {
     const supportedAddrs = addrs.filter(a => this.transportManager.transportForMultiaddr(a))
 
     if (supportedAddrs.length > this.maxAddrsToDial) {
-      this.peerStore.delete(id)
+      await this.peerStore.delete(id)
       throw errCode(new Error('dial with more addresses than allowed'), codes.ERR_TOO_MANY_ADDRESSES)
     }
 
@@ -253,10 +260,18 @@ class Dialer {
 
     // Combine the timeout signal and options.signal, if provided
     const timeoutController = new TimeoutController(this.timeout)
+
     const signals = [timeoutController.signal]
     options.signal && signals.push(options.signal)
     const signal = anySignal(signals)
 
+    // this signal will potentially be used while dialing lots of
+    // peers so prevent MaxListenersExceededWarning appearing in the console
+    try {
+      // fails on node < 15.4
+      setMaxListeners && setMaxListeners(Infinity, signal)
+    } catch {}
+
     const pendingDial = {
       dialRequest,
       controller: timeoutController,
@@ -264,13 +279,10 @@ class Dialer {
       destroy: () => {
         timeoutController.clear()
         this._pendingDials.delete(dialTarget.id)
-        this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIALS, this._pendingDials.size)
       }
     }
     this._pendingDials.set(dialTarget.id, pendingDial)
 
-    this._metrics && this._metrics.updateComponentMetric(METRICS_COMPONENT, METRICS_PENDING_DIALS, this._pendingDials.size)
-
     return pendingDial
   }