libp2p 0.35.8 → 0.36.0

package/src/circuit/auto-relay.js

@@ -8,7 +8,7 @@ const log = Object.assign(debug('libp2p:auto-relay'), {
 const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
 const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
 const { Multiaddr } = require('multiaddr')
-const PeerId = require('peer-id')
+const all = require('it-all')
 
 const { relay: multicodec } = require('./multicodec')
 const { canHop } = require('./circuit/hop')
@@ -22,7 +22,8 @@ const {
 
 /**
  * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection
- * @typedef {import('../peer-store/address-book').Address} Address
+ * @typedef {import('../peer-store/types').Address} Address
+ * @typedef {import('peer-id')} PeerId
  */
 
 /**
@@ -91,7 +92,7 @@ class AutoRelay {
 
     // If no protocol, check if we were keeping the peer before as a listenRelay
     if (!hasProtocol && this._listenRelays.has(id)) {
-      this._removeListenRelay(id)
+      await this._removeListenRelay(id)
       return
     } else if (!hasProtocol || this._listenRelays.has(id)) {
       return
@@ -113,7 +114,7 @@ class AutoRelay {
       const supportsHop = await canHop({ connection })
 
       if (supportsHop) {
-        this._peerStore.metadataBook.set(peerId, HOP_METADATA_KEY, uint8ArrayFromString(HOP_METADATA_VALUE))
+        await this._peerStore.metadataBook.setValue(peerId, HOP_METADATA_KEY, uint8ArrayFromString(HOP_METADATA_VALUE))
         await this._addListenRelay(connection, id)
       }
     } catch (/** @type {any} */ err) {
@@ -125,7 +126,6 @@ class AutoRelay {
    * Peer disconnects.
    *
    * @param {Connection} connection - connection to the peer
-   * @returns {void}
    */
   _onPeerDisconnected (connection) {
     const peerId = connection.remotePeer
@@ -136,7 +136,9 @@ class AutoRelay {
       return
     }
 
-    this._removeListenRelay(id)
+    this._removeListenRelay(id).catch(err => {
+      log.error(err)
+    })
   }
 
   /**
@@ -148,27 +150,35 @@ class AutoRelay {
    * @returns {Promise<void>}
    */
   async _addListenRelay (connection, id) {
-    // Check if already listening on enough relays
-    if (this._listenRelays.size >= this.maxListeners) {
-      return
-    }
-
-    // Get peer known addresses and sort them per public addresses first
-    const remoteAddrs = this._peerStore.addressBook.getMultiaddrsForPeer(
-      connection.remotePeer, this._addressSorter
-    )
-
-    if (!remoteAddrs || !remoteAddrs.length) {
-      return
-    }
-
-    const listenAddr = `${remoteAddrs[0].toString()}/p2p-circuit`
-    this._listenRelays.add(id)
-
-    // Attempt to listen on relay
     try {
-      await this._transportManager.listen([new Multiaddr(listenAddr)])
-      // Announce multiaddrs will update on listen success by TransportManager event being triggered
+      // Check if already listening on enough relays
+      if (this._listenRelays.size >= this.maxListeners) {
+        return
+      }
+
+      // Get peer known addresses and sort them per public addresses first
+      const remoteAddrs = await this._peerStore.addressBook.getMultiaddrsForPeer(
+        connection.remotePeer, this._addressSorter
+      )
+
+      // Attempt to listen on relay
+      const result = await Promise.all(
+        remoteAddrs.map(async addr => {
+          try {
+            // Announce multiaddrs will update on listen success by TransportManager event being triggered
+            await this._transportManager.listen([new Multiaddr(`${addr.toString()}/p2p-circuit`)])
+            return true
+          } catch (/** @type {any} */ err) {
+            this._onError(err)
+          }
+
+          return false
+        })
+      )
+
+      if (result.includes(true)) {
+        this._listenRelays.add(id)
+      }
     } catch (/** @type {any} */ err) {
       this._onError(err)
       this._listenRelays.delete(id)
@@ -180,12 +190,11 @@ class AutoRelay {
    *
    * @private
    * @param {string} id - peer identifier string.
-   * @returns {void}
    */
-  _removeListenRelay (id) {
+  async _removeListenRelay (id) {
     if (this._listenRelays.delete(id)) {
       // TODO: this should be responsibility of the connMgr
-      this._listenOnAvailableHopRelays([id])
+      await this._listenOnAvailableHopRelays([id])
     }
   }
 
@@ -197,7 +206,6 @@ class AutoRelay {
    * 3. Search the network.
    *
    * @param {string[]} [peersToIgnore]
-   * @returns {Promise<void>}
    */
   async _listenOnAvailableHopRelays (peersToIgnore = []) {
     // TODO: The peer redial issue on disconnect should be handled by connection gating
@@ -207,31 +215,37 @@ class AutoRelay {
     }
 
     const knownHopsToDial = []
+    const peers = await all(this._peerStore.getPeers())
 
     // Check if we have known hop peers to use and attempt to listen on the already connected
-    for (const [id, metadataMap] of this._peerStore.metadataBook.data.entries()) {
+    for await (const { id, metadata } of peers) {
+      const idStr = id.toB58String()
+
       // Continue to next if listening on this or peer to ignore
-      if (this._listenRelays.has(id) || peersToIgnore.includes(id)) {
+      if (this._listenRelays.has(idStr)) {
         continue
       }
 
-      const supportsHop = metadataMap.get(HOP_METADATA_KEY)
+      if (peersToIgnore.includes(idStr)) {
+        continue
+      }
+
+      const supportsHop = metadata.get(HOP_METADATA_KEY)
 
       // Continue to next if it does not support Hop
       if (!supportsHop || uint8ArrayToString(supportsHop) !== HOP_METADATA_VALUE) {
         continue
       }
 
-      const peerId = PeerId.createFromB58String(id)
-      const connection = this._connectionManager.get(peerId)
+      const connection = this._connectionManager.get(id)
 
       // If not connected, store for possible later use.
       if (!connection) {
-        knownHopsToDial.push(peerId)
+        knownHopsToDial.push(id)
         continue
       }
 
-      await this._addListenRelay(connection, id)
+      await this._addListenRelay(connection, idStr)
 
       // Check if already listening on enough relays
       if (this._listenRelays.size >= this.maxListeners) {
@@ -258,7 +272,7 @@ class AutoRelay {
         }
 
         const peerId = provider.id
-        this._peerStore.addressBook.add(peerId, provider.multiaddrs)
+        await this._peerStore.addressBook.add(peerId, provider.multiaddrs)
 
         await this._tryToListenOnRelay(peerId)
 

package/src/config.js

@@ -13,6 +13,7 @@ const { FaultTolerance } = require('./transport-manager')
 
 /**
  * @typedef {import('multiaddr').Multiaddr} Multiaddr
+ * @typedef {import('./types').ConnectionGater} ConnectionGater
  * @typedef {import('.').Libp2pOptions} Libp2pOptions
  * @typedef {import('.').constructorOptions} constructorOptions
  */
@@ -27,6 +28,7 @@ const DefaultConfig = {
   connectionManager: {
     minConnections: 25
   },
+  connectionGater: /** @type {ConnectionGater} */ {},
   transportManager: {
     faultTolerance: FaultTolerance.FATAL_ALL
   },

package/src/connection-manager/auto-dialler.js

@@ -4,6 +4,10 @@ const debug = require('debug')
 const mergeOptions = require('merge-options')
 // @ts-ignore retimer does not have types
 const retimer = require('retimer')
+const all = require('it-all')
+const { pipe } = require('it-pipe')
+const filter = require('it-filter')
+const sort = require('it-sort')
 
 const log = Object.assign(debug('libp2p:connection-manager:auto-dialler'), {
   error: debug('libp2p:connection-manager:auto-dialler:err')
@@ -50,7 +54,7 @@ class AutoDialler {
   /**
    * Starts the auto dialer
    */
-  start () {
+  async start () {
     if (!this._options.enabled) {
       log('not enabled')
       return
@@ -86,22 +90,30 @@ class AutoDialler {
       return
     }
 
-    // Sort peers on wether we know protocols of public keys for them
-    const peers = Array.from(this._libp2p.peerStore.peers.values())
-      .sort((a, b) => {
+    // Sort peers on whether we know protocols of public keys for them
+    // TODO: assuming the `peerStore.getPeers()` order is stable this will mean
+    // we keep trying to connect to the same peers?
+    const peers = await pipe(
+      this._libp2p.peerStore.getPeers(),
+      (source) => filter(source, (peer) => !peer.id.equals(this._libp2p.peerId)),
+      (source) => sort(source, (a, b) => {
         if (b.protocols && b.protocols.length && (!a.protocols || !a.protocols.length)) {
           return 1
         } else if (b.id.pubKey && !a.id.pubKey) {
           return 1
         }
         return -1
-      })
+      }),
+      (source) => all(source)
+    )
 
     for (let i = 0; this._running && i < peers.length && this._libp2p.connections.size < minConnections; i++) {
-      if (!this._libp2p.connectionManager.get(peers[i].id)) {
-        log('connecting to a peerStore stored peer %s', peers[i].id.toB58String())
+      const peer = peers[i]
+
+      if (!this._libp2p.connectionManager.get(peer.id)) {
+        log('connecting to a peerStore stored peer %s', peer.id.toB58String())
         try {
-          await this._libp2p.dialer.connectToPeer(peers[i].id)
+          await this._libp2p.dialer.connectToPeer(peer.id)
         } catch (/** @type {any} */ err) {
           log.error('could not connect to peerStore stored peer', err)
         }

package/src/connection-manager/index.js

@@ -87,14 +87,22 @@ class ConnectionManager extends EventEmitter {
      *
      * @type {Map<string, number>}
      */
-    this._peerValues = trackedMap(METRICS_COMPONENT, METRICS_PEER_VALUES, this._libp2p.metrics)
+    this._peerValues = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PEER_VALUES,
+      metrics: this._libp2p.metrics
+    })
 
     /**
      * Map of connections per peer
      *
      * @type {Map<string, Connection[]>}
      */
-    this.connections = trackedMap(METRICS_COMPONENT, METRICS_PEER_CONNECTIONS, this._libp2p.metrics)
+    this.connections = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PEER_CONNECTIONS,
+      metrics: this._libp2p.metrics
+    })
 
     this._started = false
     this._timer = null
@@ -104,6 +112,9 @@ class ConnectionManager extends EventEmitter {
       latencyCheckIntervalMs: this._options.pollInterval,
       dataEmitIntervalMs: this._options.pollInterval
     })
+
+    // This emitter gets listened to a lot
+    this.setMaxListeners(Infinity)
   }
 
   /**
@@ -187,19 +198,22 @@ class ConnectionManager extends EventEmitter {
    *
    * @private
    */
-  _checkMetrics () {
+  async _checkMetrics () {
     if (this._libp2p.metrics) {
-      const movingAverages = this._libp2p.metrics.global.movingAverages
-      // @ts-ignore moving averages object types
-      const received = movingAverages.dataReceived[this._options.movingAverageInterval].movingAverage()
-      this._checkMaxLimit('maxReceivedData', received)
-      // @ts-ignore moving averages object types
-      const sent = movingAverages.dataSent[this._options.movingAverageInterval].movingAverage()
-      this._checkMaxLimit('maxSentData', sent)
-      const total = received + sent
-      this._checkMaxLimit('maxData', total)
-      log('metrics update', total)
-      this._timer = retimer(this._checkMetrics, this._options.pollInterval)
+      try {
+        const movingAverages = this._libp2p.metrics.global.movingAverages
+        // @ts-ignore moving averages object types
+        const received = movingAverages.dataReceived[this._options.movingAverageInterval].movingAverage()
+        await this._checkMaxLimit('maxReceivedData', received)
+        // @ts-ignore moving averages object types
+        const sent = movingAverages.dataSent[this._options.movingAverageInterval].movingAverage()
+        await this._checkMaxLimit('maxSentData', sent)
+        const total = received + sent
+        await this._checkMaxLimit('maxData', total)
+        log('metrics update', total)
+      } finally {
+        this._timer = retimer(this._checkMetrics, this._options.pollInterval)
+      }
     }
   }
 
@@ -207,9 +221,8 @@ class ConnectionManager extends EventEmitter {
    * Tracks the incoming connection and check the connection limit
    *
    * @param {Connection} connection
-   * @returns {void}
    */
-  onConnect (connection) {
+  async onConnect (connection) {
     const peerId = connection.remotePeer
     const peerIdStr = peerId.toB58String()
     const storedConn = this.connections.get(peerIdStr)
@@ -222,13 +235,13 @@ class ConnectionManager extends EventEmitter {
       this.connections.set(peerIdStr, [connection])
     }
 
-    this._libp2p.peerStore.keyBook.set(peerId, peerId.pubKey)
+    await this._libp2p.peerStore.keyBook.set(peerId, peerId.pubKey)
 
     if (!this._peerValues.has(peerIdStr)) {
       this._peerValues.set(peerIdStr, this._options.defaultPeerValue)
     }
 
-    this._checkMaxLimit('maxConnections', this.size)
+    await this._checkMaxLimit('maxConnections', this.size)
   }
 
   /**
@@ -296,6 +309,9 @@ class ConnectionManager extends EventEmitter {
    */
   _onLatencyMeasure (summary) {
     this._checkMaxLimit('maxEventLoopDelay', summary.avgMs)
+      .catch(err => {
+        log.error(err)
+      })
   }
 
   /**
@@ -305,12 +321,12 @@ class ConnectionManager extends EventEmitter {
    * @param {string} name - The name of the field to check limits for
    * @param {number} value - The current value of the field
    */
-  _checkMaxLimit (name, value) {
+  async _checkMaxLimit (name, value) {
     const limit = this._options[name]
     log('checking limit of %s. current value: %d of %d', name, value, limit)
     if (value > limit) {
       log('%s: limit exceeded: %s, %d', this._peerId, name, value)
-      this._maybeDisconnectOne()
+      await this._maybeDisconnectOne()
     }
   }
 
@@ -320,7 +336,7 @@ class ConnectionManager extends EventEmitter {
    *
    * @private
    */
-  _maybeDisconnectOne () {
+  async _maybeDisconnectOne () {
     if (this._options.minConnections < this.connections.size) {
       const peerValues = Array.from(new Map([...this._peerValues.entries()].sort((a, b) => a[1] - b[1])))
       log('%s: sorted peer values: %j', this._peerId, peerValues)
@@ -331,7 +347,11 @@ class ConnectionManager extends EventEmitter {
         log('%s: closing a connection to %j', this._peerId, peerId)
         for (const connections of this.connections.values()) {
           if (connections[0].remotePeer.toB58String() === peerId) {
-            connections[0].close()
+            connections[0].close().catch(err => {
+              log.error(err)
+            })
+            // TODO: should not need to invoke this manually
+            this.onDisconnect(connections[0])
             break
           }
         }

package/src/content-routing/utils.js

@@ -14,12 +14,12 @@ const take = require('it-take')
  * Store the multiaddrs from every peer in the passed peer store
  *
  * @param {AsyncIterable<{ id: PeerId, multiaddrs: Multiaddr[] }>} source
- * @param {import('../peer-store')} peerStore
+ * @param {import('../peer-store/types').PeerStore} peerStore
  */
-function storeAddresses (source, peerStore) {
-  return map(source, (peer) => {
+async function * storeAddresses (source, peerStore) {
+  yield * map(source, async (peer) => {
     // ensure we have the addresses for a given peer
-    peerStore.addressBook.add(peer.id, peer.multiaddrs)
+    await peerStore.addressBook.add(peer.id, peer.multiaddrs)
 
     return peer
   })

package/src/dialer/dial-request.js

@@ -63,7 +63,10 @@ class DialRequest {
     tokens.forEach(token => tokenHolder.push(token))
     const dialAbortControllers = this.addrs.map(() => {
       const controller = new AbortController()
-      setMaxListeners && setMaxListeners(Infinity, controller.signal)
+      try {
+        // fails on node < 15.4
+        setMaxListeners && setMaxListeners(Infinity, controller.signal)
+      } catch {}
 
       return controller
     })

package/src/dialer/index.js

@@ -1,6 +1,9 @@
 'use strict'
 
 const debug = require('debug')
+const all = require('it-all')
+const filter = require('it-filter')
+const { pipe } = require('it-pipe')
 const log = Object.assign(debug('libp2p:dialer'), {
   error: debug('libp2p:dialer:err')
 })
@@ -30,15 +33,17 @@ const METRICS_PENDING_DIAL_TARGETS = 'pending-dial-targets'
 /**
  * @typedef {import('libp2p-interfaces/src/connection').Connection} Connection
  * @typedef {import('peer-id')} PeerId
- * @typedef {import('../peer-store')} PeerStore
- * @typedef {import('../peer-store/address-book').Address} Address
+ * @typedef {import('../peer-store/types').PeerStore} PeerStore
+ * @typedef {import('../peer-store/types').Address} Address
  * @typedef {import('../transport-manager')} TransportManager
+ * @typedef {import('../types').ConnectionGater} ConnectionGater
  */
 
 /**
  * @typedef {Object} DialerProperties
  * @property {PeerStore} peerStore
  * @property {TransportManager} transportManager
+ * @property {ConnectionGater} connectionGater
  *
  * @typedef {(addr:Multiaddr) => Promise<string[]>} Resolver
  *
@@ -70,6 +75,7 @@ class Dialer {
   constructor ({
     transportManager,
     peerStore,
+    connectionGater,
     addressSorter = publicAddressesFirst,
     maxParallelDials = MAX_PARALLEL_DIALS,
     maxAddrsToDial = MAX_ADDRS_TO_DIAL,
@@ -78,6 +84,7 @@ class Dialer {
     resolvers = {},
     metrics
   }) {
+    this.connectionGater = connectionGater
     this.transportManager = transportManager
     this.peerStore = peerStore
     this.addressSorter = addressSorter
@@ -88,10 +95,18 @@ class Dialer {
     this.tokens = [...new Array(maxParallelDials)].map((_, index) => index)
 
     /** @type {Map<string, PendingDial>} */
-    this._pendingDials = trackedMap(METRICS_COMPONENT, METRICS_PENDING_DIALS, metrics)
+    this._pendingDials = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PENDING_DIALS,
+      metrics
+    })
 
     /** @type {Map<string, { resolve: (value: any) => void, reject: (err: Error) => void}>} */
-    this._pendingDialTargets = trackedMap(METRICS_COMPONENT, METRICS_PENDING_DIAL_TARGETS, metrics)
+    this._pendingDialTargets = trackedMap({
+      component: METRICS_COMPONENT,
+      metric: METRICS_PENDING_DIAL_TARGETS,
+      metrics
+    })
 
     for (const [key, value] of Object.entries(resolvers)) {
       Multiaddr.resolvers.set(key, value)
@@ -128,6 +143,12 @@ class Dialer {
    * @returns {Promise<Connection>}
    */
   async connectToPeer (peer, options = {}) {
+    const { id } = getPeer(peer)
+
+    if (await this.connectionGater.denyDialPeer(id)) {
+      throw errCode(new Error('The dial request is blocked by gater.allowDialPeer'), codes.ERR_PEER_DIAL_INTERCEPTED)
+    }
+
     const dialTarget = await this._createCancellableDialTarget(peer)
 
     if (!dialTarget.addrs.length) {
@@ -192,10 +213,16 @@ class Dialer {
     const { id, multiaddrs } = getPeer(peer)
 
     if (multiaddrs) {
-      this.peerStore.addressBook.add(id, multiaddrs)
+      await this.peerStore.addressBook.add(id, multiaddrs)
     }
 
-    let knownAddrs = this.peerStore.addressBook.getMultiaddrsForPeer(id, this.addressSorter) || []
+    let knownAddrs = await pipe(
+      await this.peerStore.addressBook.getMultiaddrsForPeer(id, this.addressSorter),
+      (source) => filter(source, async (multiaddr) => {
+        return !(await this.connectionGater.denyDialMultiaddr(id, multiaddr))
+      }),
+      (source) => all(source)
+    )
 
     // If received a multiaddr to dial, it should be the first to use
     // But, if we know other multiaddrs for the peer, we should try them too.
@@ -215,7 +242,7 @@ class Dialer {
     const supportedAddrs = addrs.filter(a => this.transportManager.transportForMultiaddr(a))
 
     if (supportedAddrs.length > this.maxAddrsToDial) {
-      this.peerStore.delete(id)
+      await this.peerStore.delete(id)
       throw errCode(new Error('dial with more addresses than allowed'), codes.ERR_TOO_MANY_ADDRESSES)
     }
 
@@ -259,7 +286,10 @@ class Dialer {
 
     // this signal will potentially be used while dialing lots of
     // peers so prevent MaxListenersExceededWarning appearing in the console
-    setMaxListeners && setMaxListeners(Infinity, signal)
+    try {
+      // fails on node < 15.4
+      setMaxListeners && setMaxListeners(Infinity, signal)
+    } catch {}
 
     const pendingDial = {
       dialRequest,

package/src/errors.js

@@ -12,6 +12,8 @@ exports.codes = {
   PUBSUB_NOT_STARTED: 'ERR_PUBSUB_NOT_STARTED',
   DHT_NOT_STARTED: 'ERR_DHT_NOT_STARTED',
   CONN_ENCRYPTION_REQUIRED: 'ERR_CONN_ENCRYPTION_REQUIRED',
+  ERR_PEER_DIAL_INTERCEPTED: 'ERR_PEER_DIAL_INTERCEPTED',
+  ERR_CONNECTION_INTERCEPTED: 'ERR_CONNECTION_INTERCEPTED',
   ERR_INVALID_PROTOCOLS_FOR_STREAM: 'ERR_INVALID_PROTOCOLS_FOR_STREAM',
   ERR_CONNECTION_ENDED: 'ERR_CONNECTION_ENDED',
   ERR_CONNECTION_FAILED: 'ERR_CONNECTION_FAILED',

package/src/fetch/README.md

@@ -0,0 +1,36 @@
+libp2p-fetch JavaScript Implementation
+=====================================
+
+> Libp2p fetch protocol JavaScript implementation
+
+## Overview
+
+An implementation of the Fetch protocol as described here: https://github.com/libp2p/specs/tree/master/fetch
+
+The fetch protocol is a simple protocol for requesting a value corresponding to a key from a peer.
+
+## Usage
+
+```javascript
+const Libp2p = require('libp2p')
+
+/**
+ * Given a key (as a string) returns a value (as a Uint8Array), or null if the key isn't found.
+ * All keys must be prefixed my the same prefix, which will be used to find the appropriate key
+ * lookup function.
+ * @param key - a string
+ * @returns value - a Uint8Array value that corresponds to the given key, or null if the key doesn't
+ *   have a corresponding value.
+ */
+async function my_subsystem_key_lookup(key) {
+    // app specific callback to lookup key-value pairs.
+}
+
+// Enable this peer to respond to fetch requests for keys that begin with '/my_subsystem_key_prefix/'
+const libp2p = Libp2p.create(...)
+libp2p.fetchService.registerLookupFunction('/my_subsystem_key_prefix/', my_subsystem_key_lookup)
+
+const key = '/my_subsystem_key_prefix/{...}'
+const peerDst = PeerId.parse('Qmfoo...') // or Multiaddr instance
+const value = await libp2p.fetch(peerDst, key)
+```

package/src/fetch/constants.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = {
+  // https://github.com/libp2p/specs/tree/master/fetch#wire-protocol
+  PROTOCOL: '/libp2p/fetch/0.0.1'
+}