libp2p 0.35.5 → 0.35.8-63aa480

package/src/identify/index.js

@@ -77,8 +77,6 @@ class IdentifyService {
       ...libp2p._options.host
     }
 
-    this.peerStore.metadataBook.set(this.peerId, 'AgentVersion', uint8ArrayFromString(this._host.agentVersion))
-    this.peerStore.metadataBook.set(this.peerId, 'ProtocolVersion', uint8ArrayFromString(this._host.protocolVersion))
     // When a new connection happens, trigger identify
     this.connectionManager.on('peer:connect', (connection) => {
       this.identify(connection).catch(log.error)
@@ -87,18 +85,27 @@ class IdentifyService {
     // When self multiaddrs change, trigger identify-push
     this.peerStore.on('change:multiaddrs', ({ peerId }) => {
       if (peerId.toString() === this.peerId.toString()) {
-        this.pushToPeerStore()
+        this.pushToPeerStore().catch(err => log.error(err))
       }
     })
 
     // When self protocols change, trigger identify-push
     this.peerStore.on('change:protocols', ({ peerId }) => {
       if (peerId.toString() === this.peerId.toString()) {
-        this.pushToPeerStore()
+        this.pushToPeerStore().catch(err => log.error(err))
       }
     })
   }
 
+  async start () {
+    await this.peerStore.metadataBook.setValue(this.peerId, 'AgentVersion', uint8ArrayFromString(this._host.agentVersion))
+    await this.peerStore.metadataBook.setValue(this.peerId, 'ProtocolVersion', uint8ArrayFromString(this._host.protocolVersion))
+  }
+
+  async stop () {
+
+  }
+
   /**
    * Send an Identify Push update to the list of connections
    *
@@ -108,7 +115,7 @@ class IdentifyService {
   async push (connections) {
     const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId)
     const listenAddrs = this._libp2p.multiaddrs.map((ma) => ma.bytes)
-    const protocols = this.peerStore.protoBook.get(this.peerId) || []
+    const protocols = await this.peerStore.protoBook.get(this.peerId)
 
     const pushes = connections.map(async connection => {
       try {
@@ -135,10 +142,8 @@ class IdentifyService {
 
   /**
    * Calls `push` for all peers in the `peerStore` that are connected
-   *
-   * @returns {void}
    */
-  pushToPeerStore () {
+  async pushToPeerStore () {
     // Do not try to push if libp2p node is not running
     if (!this._libp2p.isStarted()) {
       return
@@ -146,13 +151,13 @@ class IdentifyService {
 
     const connections = []
     let connection
-    for (const peer of this.peerStore.peers.values()) {
+    for await (const peer of this.peerStore.getPeers()) {
       if (peer.protocols.includes(this.identifyPushProtocolStr) && (connection = this.connectionManager.get(peer.id))) {
         connections.push(connection)
       }
     }
 
-    this.push(connections)
+    await this.push(connections)
   }
 
   /**
@@ -205,10 +210,10 @@ class IdentifyService {
 
     try {
       const envelope = await Envelope.openAndCertify(signedPeerRecord, PeerRecord.DOMAIN)
-      if (this.peerStore.addressBook.consumePeerRecord(envelope)) {
-        this.peerStore.protoBook.set(id, protocols)
-        this.peerStore.metadataBook.set(id, 'AgentVersion', uint8ArrayFromString(message.agentVersion))
-        this.peerStore.metadataBook.set(id, 'ProtocolVersion', uint8ArrayFromString(message.protocolVersion))
+      if (await this.peerStore.addressBook.consumePeerRecord(envelope)) {
+        await this.peerStore.protoBook.set(id, protocols)
+        await this.peerStore.metadataBook.setValue(id, 'AgentVersion', uint8ArrayFromString(message.agentVersion))
+        await this.peerStore.metadataBook.setValue(id, 'ProtocolVersion', uint8ArrayFromString(message.protocolVersion))
         return
       }
     } catch (/** @type {any} */ err) {
@@ -217,14 +222,14 @@ class IdentifyService {
 
     // LEGACY: Update peers data in PeerStore
     try {
-      this.peerStore.addressBook.set(id, listenAddrs.map((addr) => new Multiaddr(addr)))
+      await this.peerStore.addressBook.set(id, listenAddrs.map((addr) => new Multiaddr(addr)))
     } catch (/** @type {any} */ err) {
       log.error('received invalid addrs', err)
     }
 
-    this.peerStore.protoBook.set(id, protocols)
-    this.peerStore.metadataBook.set(id, 'AgentVersion', uint8ArrayFromString(message.agentVersion))
-    this.peerStore.metadataBook.set(id, 'ProtocolVersion', uint8ArrayFromString(message.protocolVersion))
+    await this.peerStore.protoBook.set(id, protocols)
+    await this.peerStore.metadataBook.setValue(id, 'AgentVersion', uint8ArrayFromString(message.agentVersion))
+    await this.peerStore.metadataBook.setValue(id, 'ProtocolVersion', uint8ArrayFromString(message.protocolVersion))
 
     // TODO: Add and score our observed addr
     log('received observed address of %s', cleanObservedAddr)
@@ -262,25 +267,25 @@ class IdentifyService {
    * @returns {Promise<void>}
    */
   async _handleIdentify ({ connection, stream }) {
-    let publicKey = new Uint8Array(0)
-    if (this.peerId.pubKey) {
-      publicKey = this.peerId.pubKey.bytes
-    }
+    try {
+      let publicKey = new Uint8Array(0)
+      if (this.peerId.pubKey) {
+        publicKey = this.peerId.pubKey.bytes
+      }
 
-    const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId)
-    const protocols = this.peerStore.protoBook.get(this.peerId) || []
+      const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId)
+      const protocols = await this.peerStore.protoBook.get(this.peerId)
 
-    const message = Message.Identify.encode({
-      protocolVersion: this._host.protocolVersion,
-      agentVersion: this._host.agentVersion,
-      publicKey,
-      listenAddrs: this._libp2p.multiaddrs.map((ma) => ma.bytes),
-      signedPeerRecord,
-      observedAddr: connection.remoteAddr.bytes,
-      protocols
-    }).finish()
+      const message = Message.Identify.encode({
+        protocolVersion: this._host.protocolVersion,
+        agentVersion: this._host.agentVersion,
+        publicKey,
+        listenAddrs: this._libp2p.multiaddrs.map((ma) => ma.bytes),
+        signedPeerRecord,
+        observedAddr: connection.remoteAddr.bytes,
+        protocols
+      }).finish()
 
-    try {
       await pipe(
         [message],
         lp.encode(),
@@ -321,8 +326,8 @@ class IdentifyService {
 
     try {
       const envelope = await Envelope.openAndCertify(message.signedPeerRecord, PeerRecord.DOMAIN)
-      if (this.peerStore.addressBook.consumePeerRecord(envelope)) {
-        this.peerStore.protoBook.set(id, message.protocols)
+      if (await this.peerStore.addressBook.consumePeerRecord(envelope)) {
+        await this.peerStore.protoBook.set(id, message.protocols)
         return
       }
     } catch (/** @type {any} */ err) {
@@ -331,14 +336,18 @@ class IdentifyService {
 
     // LEGACY: Update peers data in PeerStore
     try {
-      this.peerStore.addressBook.set(id,
+      await this.peerStore.addressBook.set(id,
         message.listenAddrs.map((addr) => new Multiaddr(addr)))
     } catch (/** @type {any} */ err) {
       log.error('received invalid addrs', err)
     }
 
     // Update the protocols
-    this.peerStore.protoBook.set(id, message.protocols)
+    try {
+      await this.peerStore.protoBook.set(id, message.protocols)
+    } catch (/** @type {any} */ err) {
+      log.error('received invalid protocols', err)
+    }
   }
 
   /**

package/src/index.js

@@ -9,7 +9,7 @@ const { EventEmitter } = require('events')
 const errCode = require('err-code')
 const PeerId = require('peer-id')
 const { Multiaddr } = require('multiaddr')
-
+const { MemoryDatastore } = require('datastore-core/memory')
 const PeerRouting = require('./peer-routing')
 const ContentRouting = require('./content-routing')
 const getPeer = require('./get-peer')
@@ -28,7 +28,6 @@ const TransportManager = require('./transport-manager')
 const Upgrader = require('./upgrader')
 const PeerStore = require('./peer-store')
 const PubsubAdapter = require('./pubsub-adapter')
-const PersistentPeerStore = require('./peer-store/persistent')
 const Registrar = require('./registrar')
 const ping = require('./ping')
 const IdentifyService = require('./identify')
@@ -48,6 +47,8 @@ const { updateSelfPeerRecord } = require('./record/utils')
  * @typedef {import('libp2p-interfaces/src/pubsub').PubsubOptions} PubsubOptions
  * @typedef {import('interface-datastore').Datastore} Datastore
  * @typedef {import('./pnet')} Protector
+ * @typedef {Object} PersistentPeerStoreOptions
+ * @property {number} [threshold]
  */
 
 /**
@@ -110,7 +111,7 @@ const { updateSelfPeerRecord } = require('./record/utils')
  * @property {KeychainOptions & import('./keychain/index').KeychainOptions} [keychain]
  * @property {MetricsOptions & import('./metrics').MetricsOptions} [metrics]
  * @property {import('./peer-routing').PeerRoutingOptions} [peerRouting]
- * @property {PeerStoreOptions & import('./peer-store/persistent').PersistentPeerStoreOptions} [peerStore]
+ * @property {PeerStoreOptions} [peerStore]
  * @property {import('./transport-manager').TransportManagerOptions} [transportManager]
  * @property {Libp2pConfig} [config]
  *
@@ -161,13 +162,20 @@ class Libp2p extends EventEmitter {
     this.peerId = this._options.peerId
     this.datastore = this._options.datastore
 
-    this.peerStore = (this.datastore && this._options.peerStore.persistence)
-      ? new PersistentPeerStore({
-        peerId: this.peerId,
-        datastore: this.datastore,
-        ...this._options.peerStore
+    // Create Metrics
+    if (this._options.metrics.enabled) {
+      const metrics = new Metrics({
+        ...this._options.metrics
       })
-      : new PeerStore({ peerId: this.peerId })
+
+      this.metrics = metrics
+    }
+
+    /** @type {import('./peer-store/types').PeerStore} */
+    this.peerStore = new PeerStore({
+      peerId: this.peerId,
+      datastore: (this.datastore && this._options.peerStore.persistence) ? this.datastore : new MemoryDatastore()
+    })
 
     // Addresses {listen, announce, noAnnounce}
     this.addresses = this._options.addresses
@@ -195,15 +203,6 @@ class Libp2p extends EventEmitter {
       autoDialInterval: this._options.connectionManager.autoDialInterval
     })
 
-    // Create Metrics
-    if (this._options.metrics.enabled) {
-      const metrics = new Metrics({
-        ...this._options.metrics
-      })
-
-      this.metrics = metrics
-    }
-
     // Create keychain
     if (this._options.keychain && this._options.keychain.datastore) {
       log('creating keychain')
@@ -288,7 +287,6 @@ class Libp2p extends EventEmitter {
 
       // Add the identify service since we can multiplex
       this.identifyService = new IdentifyService({ libp2p: this })
-      this.handle(Object.values(IdentifyService.getProtocolStr(this)), this.identifyService.handleMessage)
     }
 
     // Attach private network protector
@@ -354,6 +352,10 @@ class Libp2p extends EventEmitter {
   async start () {
     log('libp2p is starting')
 
+    if (this.identifyService) {
+      await this.handle(Object.values(IdentifyService.getProtocolStr(this)), this.identifyService.handleMessage)
+    }
+
     try {
       await this._onStarting()
       await this._onDidStart()
@@ -378,9 +380,13 @@ class Libp2p extends EventEmitter {
     try {
       this._isStarted = false
 
+      if (this.identifyService) {
+        await this.identifyService.stop()
+      }
+
       this.relay && this.relay.stop()
       this.peerRouting.stop()
-      this._autodialler.stop()
+      await this._autodialler.stop()
       await (this._dht && this._dht.stop())
 
       for (const service of this._discovery.values()) {
@@ -391,7 +397,6 @@ class Libp2p extends EventEmitter {
 
       this._discovery = new Map()
 
-      await this.peerStore.stop()
       await this.connectionManager.stop()
 
       await Promise.all([
@@ -497,7 +502,7 @@ class Libp2p extends EventEmitter {
     if (!connection) {
       connection = await this.dialer.connectToPeer(peer, options)
     } else if (multiaddrs) {
-      this.peerStore.addressBook.add(id, multiaddrs)
+      await this.peerStore.addressBook.add(id, multiaddrs)
     }
 
     return connection
@@ -577,14 +582,14 @@ class Libp2p extends EventEmitter {
    * @param {string[]|string} protocols
    * @param {(props: HandlerProps) => void} handler
    */
-  handle (protocols, handler) {
+  async handle (protocols, handler) {
     protocols = Array.isArray(protocols) ? protocols : [protocols]
     protocols.forEach(protocol => {
       this.upgrader.protocols.set(protocol, handler)
     })
 
     // Add new protocols to self protocols in the Protobook
-    this.peerStore.protoBook.add(this.peerId, protocols)
+    await this.peerStore.protoBook.add(this.peerId, protocols)
   }
 
   /**
@@ -593,14 +598,14 @@ class Libp2p extends EventEmitter {
    *
    * @param {string[]|string} protocols
    */
-  unhandle (protocols) {
+  async unhandle (protocols) {
     protocols = Array.isArray(protocols) ? protocols : [protocols]
     protocols.forEach(protocol => {
       this.upgrader.protocols.delete(protocol)
     })
 
     // Remove protocols from self protocols in the Protobook
-    this.peerStore.protoBook.remove(this.peerId, protocols)
+    await this.peerStore.protoBook.remove(this.peerId, protocols)
   }
 
   async _onStarting () {
@@ -611,11 +616,8 @@ class Libp2p extends EventEmitter {
     // Manage your NATs
     this.natManager.start()
 
-    // Start PeerStore
-    await this.peerStore.start()
-
     if (this._config.pubsub.enabled) {
-      this.pubsub && this.pubsub.start()
+      this.pubsub && await this.pubsub.start()
     }
 
     // DHT subsystem
@@ -629,6 +631,10 @@ class Libp2p extends EventEmitter {
 
     // Start metrics if present
     this.metrics && this.metrics.start()
+
+    if (this.identifyService) {
+      await this.identifyService.start()
+    }
   }
 
   /**
@@ -641,17 +647,19 @@ class Libp2p extends EventEmitter {
 
     this.peerStore.on('peer', peerId => {
       this.emit('peer:discovery', peerId)
-      this._maybeConnect(peerId)
+      this._maybeConnect(peerId).catch(err => {
+        log.error(err)
+      })
     })
 
     // Once we start, emit any peers we may have already discovered
     // TODO: this should be removed, as we already discovered these peers in the past
-    for (const peer of this.peerStore.peers.values()) {
+    for await (const peer of this.peerStore.getPeers()) {
       this.emit('peer:discovery', peer.id)
     }
 
     this.connectionManager.start()
-    this._autodialler.start()
+    await this._autodialler.start()
 
     // Peer discovery
     await this._setupPeerDiscovery()
@@ -675,8 +683,8 @@ class Libp2p extends EventEmitter {
       return
     }
 
-    peer.multiaddrs && this.peerStore.addressBook.add(peer.id, peer.multiaddrs)
-    peer.protocols && this.peerStore.protoBook.set(peer.id, peer.protocols)
+    peer.multiaddrs && this.peerStore.addressBook.add(peer.id, peer.multiaddrs).catch(err => log.error(err))
+    peer.protocols && this.peerStore.protoBook.set(peer.id, peer.protocols).catch(err => log.error(err))
   }
 
   /**

package/src/metrics/index.js

@@ -44,7 +44,7 @@ class Metrics {
     this._oldPeers = oldPeerLRU(this._options.maxOldPeersRetention)
     this._running = false
     this._onMessage = this._onMessage.bind(this)
-    this._componentMetrics = new Map()
+    this._systems = new Map()
   }
 
   /**
@@ -89,19 +89,26 @@ class Metrics {
   }
 
   /**
-   * @returns {Map}
+   * @returns {Map<string, Map<string, Map<string, any>>>}
    */
   getComponentMetrics () {
-    return this._componentMetrics
+    return this._systems
   }
 
-  updateComponentMetric (component, metric, value) {
-    if (!this._componentMetrics.has(component)) {
-      this._componentMetrics.set(component, new Map())
+  updateComponentMetric ({ system = 'libp2p', component, metric, value }) {
+    if (!this._systems.has(system)) {
+      this._systems.set(system, new Map())
     }
 
-    const map = this._componentMetrics.get(component)
-    map.set(metric, value)
+    const systemMetrics = this._systems.get(system)
+
+    if (!systemMetrics.has(component)) {
+      systemMetrics.set(component, new Map())
+    }
+
+    const componentMetrics = systemMetrics.get(component)
+
+    componentMetrics.set(metric, value)
   }
 
   /**

package/src/metrics/stats.js

@@ -8,6 +8,7 @@ const retimer = require('retimer')
 
 /**
  * @typedef {import('@vascosantos/moving-average').IMovingAverage} IMovingAverage
+ * @typedef {import('bignumber.js').BigNumber} Big
  */
 
 class Stats extends EventEmitter {

package/src/metrics/tracked-map.js

@@ -0,0 +1,94 @@
+'use strict'
+
+/**
+ * @template K
+ * @template V
+ */
+class TrackedMap extends Map {
+  /**
+   * @param {object} options
+   * @param {string} options.system
+   * @param {string} options.component
+   * @param {string} options.metric
+   * @param {import('.')} options.metrics
+   */
+  constructor (options) {
+    super()
+
+    const { system, component, metric, metrics } = options
+    this._system = system
+    this._component = component
+    this._metric = metric
+    this._metrics = metrics
+
+    this._metrics.updateComponentMetric({
+      system: this._system,
+      component: this._component,
+      metric: this._metric,
+      value: this.size
+    })
+  }
+
+  /**
+   * @param {K} key
+   * @param {V} value
+   */
+  set (key, value) {
+    super.set(key, value)
+    this._metrics.updateComponentMetric({
+      system: this._system,
+      component: this._component,
+      metric: this._metric,
+      value: this.size
+    })
+    return this
+  }
+
+  /**
+   * @param {K} key
+   */
+  delete (key) {
+    const deleted = super.delete(key)
+    this._metrics.updateComponentMetric({
+      system: this._system,
+      component: this._component,
+      metric: this._metric,
+      value: this.size
+    })
+    return deleted
+  }
+
+  clear () {
+    super.clear()
+
+    this._metrics.updateComponentMetric({
+      system: this._system,
+      component: this._component,
+      metric: this._metric,
+      value: this.size
+    })
+  }
+}
+
+/**
+ * @template K
+ * @template V
+ * @param {object} options
+ * @param {string} [options.system]
+ * @param {string} options.component
+ * @param {string} options.metric
+ * @param {import('.')} [options.metrics]
+ * @returns {Map<K, V>}
+ */
+module.exports = ({ system = 'libp2p', component, metric, metrics }) => {
+  /** @type {Map<K, V>} */
+  let map
+
+  if (metrics) {
+    map = new TrackedMap({ system, component, metric, metrics })
+  } else {
+    map = new Map()
+  }
+
+  return map
+}

package/src/peer-routing.js

@@ -24,6 +24,8 @@ const {
 // @ts-ignore module with no types
 } = require('set-delayed-interval')
 const { DHTPeerRouting } = require('./dht/dht-peer-routing')
+// @ts-expect-error setMaxListeners is missing from the types
+const { setMaxListeners } = require('events')
 
 /**
  * @typedef {import('peer-id')} PeerId
@@ -149,7 +151,15 @@ class PeerRouting {
     }
 
     if (options.timeout) {
-      options.signal = new TimeoutController(options.timeout).signal
+      const controller = new TimeoutController(options.timeout)
+      // this controller will potentially be used while dialing lots of
+      // peers so prevent MaxListenersExceededWarning appearing in the console
+      try {
+        // fails on node < 15.4
+        setMaxListeners && setMaxListeners(Infinity, controller.signal)
+      } catch {}
+
+      options.signal = controller.signal
     }
 
     yield * pipe(