libp2p 0.36.2-fab4f13 → 0.37.0-d5386df

package/src/connection-manager/index.ts

@@ -4,16 +4,21 @@ import mergeOptions from 'merge-options'
 import { LatencyMonitor, SummaryObject } from './latency-monitor.js'
 // @ts-expect-error retimer does not have types
 import retimer from 'retimer'
-import { CustomEvent, EventEmitter, Startable } from '@libp2p/interfaces'
+import type { AbortOptions } from '@libp2p/interfaces'
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
+import type { Startable } from '@libp2p/interfaces/startable'
 import { trackedMap } from '@libp2p/tracked-map'
 import { codes } from '../errors.js'
 import { isPeerId, PeerId } from '@libp2p/interfaces/peer-id'
 // @ts-expect-error setMaxListeners is missing from the node 16 types
 import { setMaxListeners } from 'events'
 import type { Connection } from '@libp2p/interfaces/connection'
-import type { ConnectionManager } from '@libp2p/interfaces/registrar'
-import type { Components } from '@libp2p/interfaces/components'
+import type { ConnectionManager } from '@libp2p/interfaces/connection-manager'
+import { Components, Initializable } from '@libp2p/interfaces/components'
 import * as STATUS from '@libp2p/interfaces/connection/status'
+import { Dialer } from './dialer/index.js'
+import type { AddressSorter } from '@libp2p/interfaces/peer-store'
+import type { Resolver } from '@multiformats/multiaddr'
 
 const log = logger('libp2p:connection-manager')
 
@@ -34,21 +39,16 @@ const METRICS_COMPONENT = 'connection-manager'
 const METRICS_PEER_CONNECTIONS = 'peer-connections'
 const METRICS_PEER_VALUES = 'peer-values'
 
-export interface ConnectionManagerEvents {
-  'peer:connect': CustomEvent<PeerId>
-  'peer:disconnect': CustomEvent<PeerId>
-}
-
 export interface ConnectionManagerInit {
   /**
-   * The maximum number of connections allowed
+   * The maximum number of connections to keep open
    */
-  maxConnections?: number
+  maxConnections: number
 
   /**
-   * The minimum number of connections to avoid pruning
+   * The minimum number of connections to keep open
    */
-  minConnections?: number
+  minConnections: number
 
   /**
    * The max data (in and out), per average interval to allow
@@ -86,39 +86,75 @@ export interface ConnectionManagerInit {
   defaultPeerValue?: number
 
   /**
-   * Should preemptively guarantee connections are above the low watermark
+   * If true, try to connect to all discovered peers up to the connection manager limit
    */
   autoDial?: boolean
 
   /**
-   * How often, in milliseconds, it should preemptively guarantee connections are above the low watermark
+   * How long to wait between attempting to keep our number of concurrent connections
+   * above minConnections
+   */
+  autoDialInterval: number
+
+  /**
+   * Sort the known addresses of a peer before trying to dial
+   */
+  addressSorter?: AddressSorter
+
+  /**
+   * Number of max concurrent dials
+   */
+  maxParallelDials?: number
+
+  /**
+   * Number of max addresses to dial for a given peer
+   */
+  maxAddrsToDial?: number
+
+  /**
+   * How long a dial attempt is allowed to take
    */
-  autoDialInterval?: number
+  dialTimeout?: number
+
+  /**
+   * Number of max concurrent dials per peer
+   */
+  maxDialsPerPeer?: number
+
+  /**
+   * Multiaddr resolvers to use when dialing
+   */
+  resolvers?: Record<string, Resolver>
+}
+
+export interface ConnectionManagerEvents {
+  'peer:connect': CustomEvent<PeerId>
+  'peer:disconnect': CustomEvent<PeerId>
 }
 
 /**
  * Responsible for managing known connections.
  */
-export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEvents> implements ConnectionManager, Startable {
-  private readonly components: Components
-  private readonly init: Required<ConnectionManagerInit>
+export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEvents> implements ConnectionManager, Startable, Initializable {
+  public readonly dialer: Dialer
+  private components = new Components()
+  private readonly opts: Required<ConnectionManagerInit>
   private readonly peerValues: Map<string, number>
   private readonly connections: Map<string, Connection[]>
   private started: boolean
   private timer?: ReturnType<retimer>
   private readonly latencyMonitor: LatencyMonitor
 
-  constructor (components: Components, init: ConnectionManagerInit = {}) {
+  constructor (init: ConnectionManagerInit) {
     super()
 
-    this.components = components
-    this.init = mergeOptions.call({ ignoreUndefined: true }, defaultOptions, init)
+    this.opts = mergeOptions.call({ ignoreUndefined: true }, defaultOptions, init)
 
-    if (this.init.maxConnections < this.init.minConnections) {
+    if (this.opts.maxConnections < this.opts.minConnections) {
       throw errCode(new Error('Connection Manager maxConnections must be greater than minConnections'), codes.ERR_INVALID_PARAMETERS)
     }
 
-    log('options: %o', this.init)
+    log('options: %o', this.opts)
 
     /**
      * Map of peer identifiers to their peer value for pruning connections.
@@ -153,12 +189,16 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
       setMaxListeners?.(Infinity, this)
     } catch {}
 
-    this.components.getUpgrader().addEventListener('connection', (evt) => {
-      void this.onConnect(evt).catch(err => {
-        log.error(err)
-      })
-    })
-    this.components.getUpgrader().addEventListener('connectionEnd', this.onDisconnect.bind(this))
+    this.dialer = new Dialer(this.opts)
+
+    this.onConnect = this.onConnect.bind(this)
+    this.onDisconnect = this.onDisconnect.bind(this)
+  }
+
+  init (components: Components): void {
+    this.components = components
+
+    this.dialer.init(components)
   }
 
   isStarted () {
@@ -171,18 +211,29 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    */
   async start () {
     if (this.components.getMetrics() != null) {
-      this.timer = this.timer ?? retimer(this._checkMetrics, this.init.pollInterval)
+      this.timer = this.timer ?? retimer(this._checkMetrics, this.opts.pollInterval)
     }
 
     // latency monitor
     this.latencyMonitor.start()
     this._onLatencyMeasure = this._onLatencyMeasure.bind(this)
     this.latencyMonitor.addEventListener('data', this._onLatencyMeasure)
+    await this.dialer.start()
 
     this.started = true
     log('started')
   }
 
+  async afterStart () {
+    this.components.getUpgrader().addEventListener('connection', this.onConnect)
+    this.components.getUpgrader().addEventListener('connectionEnd', this.onDisconnect)
+  }
+
+  async beforeStop () {
+    this.components.getUpgrader().removeEventListener('connection', this.onConnect)
+    this.components.getUpgrader().removeEventListener('connectionEnd', this.onDisconnect)
+  }
+
   /**
    * Stops the Connection Manager
    */
@@ -191,6 +242,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
 
     this.latencyMonitor.removeEventListener('data', this._onLatencyMeasure)
     this.latencyMonitor.stop()
+    await this.dialer.stop()
 
     this.started = false
     await this._close()
@@ -238,23 +290,29 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     if (metrics != null) {
       try {
         const movingAverages = metrics.getGlobal().getMovingAverages()
-        const received = movingAverages.dataReceived[this.init.movingAverageInterval].movingAverage
+        const received = movingAverages.dataReceived[this.opts.movingAverageInterval].movingAverage
         await this._checkMaxLimit('maxReceivedData', received)
-        const sent = movingAverages.dataSent[this.init.movingAverageInterval].movingAverage
+        const sent = movingAverages.dataSent[this.opts.movingAverageInterval].movingAverage
         await this._checkMaxLimit('maxSentData', sent)
         const total = received + sent
         await this._checkMaxLimit('maxData', total)
-        log('metrics update', total)
+        log.trace('metrics update', total)
       } finally {
-        this.timer = retimer(this._checkMetrics, this.init.pollInterval)
+        this.timer = retimer(this._checkMetrics, this.opts.pollInterval)
       }
     }
   }
 
+  onConnect (evt: CustomEvent<Connection>) {
+    void this._onConnect(evt).catch(err => {
+      log.error(err)
+    })
+  }
+
   /**
    * Tracks the incoming connection and check the connection limit
    */
-  async onConnect (evt: CustomEvent<Connection>) {
+  async _onConnect (evt: CustomEvent<Connection>) {
     const { detail: connection } = evt
 
     if (!this.started) {
@@ -267,8 +325,6 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     const peerIdStr = peerId.toString()
     const storedConns = this.connections.get(peerIdStr)
 
-    this.dispatchEvent(new CustomEvent<Connection>('peer:connect', { detail: connection }))
-
     if (storedConns != null) {
       storedConns.push(connection)
     } else {
@@ -280,10 +336,11 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     }
 
     if (!this.peerValues.has(peerIdStr)) {
-      this.peerValues.set(peerIdStr, this.init.defaultPeerValue)
+      this.peerValues.set(peerIdStr, this.opts.defaultPeerValue)
     }
 
-    await this._checkMaxLimit('maxConnections', this.getConnectionList().length)
+    await this._checkMaxLimit('maxConnections', this.getConnections().length)
+    this.dispatchEvent(new CustomEvent<Connection>('peer:connect', { detail: connection }))
   }
 
   /**
@@ -312,35 +369,64 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     }
   }
 
-  getConnectionMap (): Map<string, Connection[]> {
-    return this.connections
-  }
+  getConnections (peerId?: PeerId): Connection[] {
+    if (peerId != null) {
+      return this.connections.get(peerId.toString()) ?? []
+    }
 
-  getConnectionList (): Connection[] {
-    let output: Connection[] = []
+    let conns: Connection[] = []
 
-    for (const connections of this.connections.values()) {
-      output = output.concat(connections)
+    for (const c of this.connections.values()) {
+      conns = conns.concat(c)
     }
 
-    return output
+    return conns
   }
 
-  getConnections (peerId: PeerId): Connection[] {
-    return this.connections.get(peerId.toString()) ?? []
-  }
+  async openConnection (peerId: PeerId, options?: AbortOptions): Promise<Connection> {
+    log('dial to %p', peerId)
+    const existingConnections = this.getConnections(peerId)
 
-  /**
-   * Get a connection with a peer
-   */
-  getConnection (peerId: PeerId): Connection | undefined {
-    const connections = this.getAll(peerId)
+    if (existingConnections.length > 0) {
+      log('had an existing connection to %p', peerId)
+
+      return existingConnections[0]
+    }
+
+    const connection = await this.dialer.dial(peerId, options)
+    let peerConnections = this.connections.get(peerId.toString())
+
+    if (peerConnections == null) {
+      peerConnections = []
+      this.connections.set(peerId.toString(), peerConnections)
+    }
 
-    if (connections.length > 0) {
-      return connections[0]
+    // we get notified of connections via the Upgrader emitting "connection"
+    // events, double check we aren't already tracking this connection before
+    // storing it
+    let trackedConnection = false
+
+    for (const conn of peerConnections) {
+      if (conn.id === connection.id) {
+        trackedConnection = true
+      }
+    }
+
+    if (!trackedConnection) {
+      peerConnections.push(connection)
     }
 
-    return undefined
+    return connection
+  }
+
+  async closeConnections (peerId: PeerId): Promise<void> {
+    const connections = this.connections.get(peerId.toString()) ?? []
+
+    await Promise.all(
+      connections.map(async connection => {
+        return await connection.close()
+      })
+    )
   }
 
   /**
@@ -378,7 +464,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    * If the `value` of `name` has exceeded its limit, maybe close a connection
    */
   async _checkMaxLimit (name: keyof ConnectionManagerInit, value: number) {
-    const limit = this.init[name]
+    const limit = this.opts[name]
     log.trace('checking limit of %s. current value: %d of %d', name, value, limit)
     if (value > limit) {
       log('%s: limit exceeded: %p, %d', this.components.getPeerId(), name, value)
@@ -391,7 +477,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    * to the lowest valued peer.
    */
   async _maybeDisconnectOne () {
-    if (this.init.minConnections < this.connections.size) {
+    if (this.opts.minConnections < this.connections.size) {
       const peerValues = Array.from(new Map([...this.peerValues.entries()].sort((a, b) => a[1] - b[1])))
 
       log('%p: sorted peer values: %j', this.components.getPeerId(), peerValues)

package/src/connection-manager/latency-monitor.ts

@@ -2,7 +2,7 @@
  * This code is based on `latency-monitor` (https://github.com/mlucool/latency-monitor) by `mlucool` (https://github.com/mlucool), available under Apache License 2.0 (https://github.com/mlucool/latency-monitor/blob/master/LICENSE)
  */
 
-import { CustomEvent, EventEmitter } from '@libp2p/interfaces'
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
 import { VisibilityChangeEmitter } from './visibility-change-emitter.js'
 import { logger } from '@libp2p/logger'
 

package/src/connection-manager/visibility-change-emitter.ts

@@ -2,7 +2,7 @@
  * This code is based on `latency-monitor` (https://github.com/mlucool/latency-monitor) by `mlucool` (https://github.com/mlucool), available under Apache License 2.0 (https://github.com/mlucool/latency-monitor/blob/master/LICENSE)
  */
 
-import { CustomEvent, EventEmitter } from '@libp2p/interfaces'
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
 import { logger } from '@libp2p/logger'
 
 const log = logger('libp2p:connection-manager:latency-monitor:visibility-change-emitter')

package/src/content-routing/index.ts

@@ -9,7 +9,8 @@ import drain from 'it-drain'
 import merge from 'it-merge'
 import { pipe } from 'it-pipe'
 import type { ContentRouting } from '@libp2p/interfaces/content-routing'
-import type { AbortOptions, Startable } from '@libp2p/interfaces'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Startable } from '@libp2p/interfaces/startable'
 import type { CID } from 'multiformats/cid'
 import type { Components } from '@libp2p/interfaces/components'
 

package/src/dht/dummy-dht.ts

@@ -2,9 +2,18 @@ import type { DualDHT, QueryEvent, SingleDHT } from '@libp2p/interfaces/dht'
 import type { PeerDiscoveryEvents } from '@libp2p/interfaces/peer-discovery'
 import errCode from 'err-code'
 import { messages, codes } from '../errors.js'
-import { EventEmitter } from '@libp2p/interfaces'
+import { EventEmitter } from '@libp2p/interfaces/events'
+import { symbol } from '@libp2p/interfaces/peer-discovery'
 
 export class DummyDHT extends EventEmitter<PeerDiscoveryEvents> implements DualDHT {
+  get [symbol] (): true {
+    return true
+  }
+
+  get [Symbol.toStringTag] () {
+    return '@libp2p/dummy-dht'
+  }
+
   get wan (): SingleDHT {
     throw errCode(new Error(messages.DHT_DISABLED), codes.DHT_DISABLED)
   }

package/src/fetch/index.ts

@@ -6,7 +6,7 @@ import { FetchRequest, FetchResponse } from './pb/proto.js'
 import { handshake } from 'it-handshake'
 import { PROTOCOL } from './constants.js'
 import type { PeerId } from '@libp2p/interfaces/peer-id'
-import type { Startable } from '@libp2p/interfaces'
+import type { Startable } from '@libp2p/interfaces/startable'
 import type { Stream } from '@libp2p/interfaces/connection'
 import type { IncomingStreamData } from '@libp2p/interfaces/registrar'
 import type { Components } from '@libp2p/interfaces/components'
@@ -70,7 +70,7 @@ export class FetchService implements Startable {
   async fetch (peer: PeerId, key: string): Promise<Uint8Array | null> {
     log('dialing %s to %p', this.protocol, peer)
 
-    const connection = await this.components.getDialer().dial(peer)
+    const connection = await this.components.getConnectionManager().openConnection(peer)
     const { stream } = await connection.newStream([this.protocol])
     const shake = handshake(stream)
 

package/src/fetch/pb/proto.ts

@@ -2,13 +2,14 @@
 /* eslint-disable @typescript-eslint/no-namespace */
 
 import { encodeMessage, decodeMessage, message, string, enumeration, bytes } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
 
 export interface FetchRequest {
   identifier: string
 }
 
 export namespace FetchRequest {
-  export const codec = () => {
+  export const codec = (): Codec<FetchRequest> => {
     return message<FetchRequest>({
       1: { name: 'identifier', codec: string }
     })
@@ -35,13 +36,19 @@ export namespace FetchResponse {
     ERROR = 'ERROR'
   }
 
+  enum __StatusCodeValues {
+    OK = 0,
+    NOT_FOUND = 1,
+    ERROR = 2
+  }
+
   export namespace StatusCode {
     export const codec = () => {
-      return enumeration<typeof StatusCode>(StatusCode)
+      return enumeration<typeof StatusCode>(__StatusCodeValues)
     }
   }
 
-  export const codec = () => {
+  export const codec = (): Codec<FetchResponse> => {
     return message<FetchResponse>({
       1: { name: 'status', codec: FetchResponse.StatusCode.codec() },
       2: { name: 'data', codec: bytes }

package/src/identify/index.ts

@@ -22,8 +22,8 @@ import {
 import { codes } from '../errors.js'
 import type { IncomingStreamData } from '@libp2p/interfaces/registrar'
 import type { Connection } from '@libp2p/interfaces/connection'
-import type { Startable } from '@libp2p/interfaces'
-import { peerIdFromKeys, peerIdFromString } from '@libp2p/peer-id'
+import type { Startable } from '@libp2p/interfaces/startable'
+import { peerIdFromKeys } from '@libp2p/peer-id'
 import type { Components } from '@libp2p/interfaces/components'
 
 const log = logger('libp2p:identify')
@@ -161,15 +161,15 @@ export class IdentifyService implements Startable {
 
     const connections: Connection[] = []
 
-    for (const [peerIdStr, conns] of this.components.getConnectionManager().getConnectionMap().entries()) {
-      const peerId = peerIdFromString(peerIdStr)
+    for (const conn of this.components.getConnectionManager().getConnections()) {
+      const peerId = conn.remotePeer
       const peer = await this.components.getPeerStore().get(peerId)
 
       if (!peer.protocols.includes(this.identifyPushProtocolStr)) {
         continue
       }
 
-      connections.push(...conns)
+      connections.push(conn)
     }
 
     await this.push(connections)

package/src/identify/pb/message.ts

@@ -2,6 +2,7 @@
 /* eslint-disable @typescript-eslint/no-namespace */
 
 import { encodeMessage, decodeMessage, message, string, bytes } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
 
 export interface Identify {
   protocolVersion?: string
@@ -14,7 +15,7 @@ export interface Identify {
 }
 
 export namespace Identify {
-  export const codec = () => {
+  export const codec = (): Codec<Identify> => {
     return message<Identify>({
       5: { name: 'protocolVersion', codec: string, optional: true },
       6: { name: 'agentVersion', codec: string, optional: true },

package/src/index.ts

@@ -1,5 +1,7 @@
 import { createLibp2pNode } from './libp2p.js'
-import type { AbortOptions, EventEmitter, RecursivePartial, Startable } from '@libp2p/interfaces'
+import type { AbortOptions, RecursivePartial } from '@libp2p/interfaces'
+import type { EventEmitter } from '@libp2p/interfaces/events'
+import type { Startable } from '@libp2p/interfaces/startable'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { FaultTolerance } from './transport-manager.js'
 import type { HostProperties } from './identify/index.js'
@@ -16,11 +18,12 @@ import type { ConnectionEncrypter } from '@libp2p/interfaces/connection-encrypte
 import type { PeerRouting } from '@libp2p/interfaces/peer-routing'
 import type { ContentRouting } from '@libp2p/interfaces/content-routing'
 import type { PubSub } from '@libp2p/interfaces/pubsub'
-import type { ConnectionManager, Registrar, StreamHandler } from '@libp2p/interfaces/registrar'
+import type { Registrar, StreamHandler } from '@libp2p/interfaces/registrar'
+import type { ConnectionManager } from '@libp2p/interfaces/connection-manager'
 import type { Metrics, MetricsInit } from '@libp2p/interfaces/metrics'
 import type { PeerInfo } from '@libp2p/interfaces/peer-info'
-import type { DialerInit } from '@libp2p/interfaces/dialer'
 import type { KeyChain } from './keychain/index.js'
+import type { ConnectionManagerInit } from './connection-manager/index.js'
 
 export interface PersistentPeerStoreOptions {
   threshold?: number
@@ -71,29 +74,6 @@ export interface AddressesConfig {
   announceFilter: (multiaddrs: Multiaddr[]) => Multiaddr[]
 }
 
-export interface ConnectionManagerConfig {
-  /**
-   * If true, try to connect to all discovered peers up to the connection manager limit
-   */
-  autoDial?: boolean
-
-  /**
-   * The maximum number of connections to keep open
-   */
-  maxConnections: number
-
-  /**
-   * The minimum number of connections to keep open
-   */
-  minConnections: number
-
-  /**
-   * How long to wait between attempting to keep our number of concurrent connections
-   * above minConnections
-   */
-  autoDialInterval: number
-}
-
 export interface TransportManagerConfig {
   faultTolerance?: FaultTolerance
 }
@@ -117,11 +97,10 @@ export interface Libp2pInit {
   peerId: PeerId
   host: HostProperties
   addresses: AddressesConfig
-  connectionManager: ConnectionManagerConfig
+  connectionManager: ConnectionManagerInit
   connectionGater: Partial<ConnectionGater>
   transportManager: TransportManagerConfig
   datastore: Datastore
-  dialer: DialerInit
   metrics: MetricsInit
   peerStore: PeerStoreInit
   peerRouting: PeerRoutingConfig

package/src/insecure/pb/proto.ts

@@ -2,6 +2,7 @@
 /* eslint-disable @typescript-eslint/no-namespace */
 
 import { encodeMessage, decodeMessage, message, bytes, enumeration } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
 
 export interface Exchange {
   id?: Uint8Array
@@ -9,7 +10,7 @@ export interface Exchange {
 }
 
 export namespace Exchange {
-  export const codec = () => {
+  export const codec = (): Codec<Exchange> => {
     return message<Exchange>({
       1: { name: 'id', codec: bytes, optional: true },
       2: { name: 'pubkey', codec: PublicKey.codec(), optional: true }
@@ -32,19 +33,25 @@ export enum KeyType {
   ECDSA = 'ECDSA'
 }
 
+enum __KeyTypeValues {
+  RSA = 0,
+  Ed25519 = 1,
+  Secp256k1 = 2,
+  ECDSA = 3
+}
+
 export namespace KeyType {
   export const codec = () => {
-    return enumeration<typeof KeyType>(KeyType)
+    return enumeration<typeof KeyType>(__KeyTypeValues)
   }
 }
-
 export interface PublicKey {
   Type: KeyType
   Data: Uint8Array
 }
 
 export namespace PublicKey {
-  export const codec = () => {
+  export const codec = (): Codec<PublicKey> => {
     return message<PublicKey>({
       1: { name: 'Type', codec: KeyType.codec() },
       2: { name: 'Data', codec: bytes }

package/src/keychain/index.ts

@@ -9,7 +9,6 @@ import errCode from 'err-code'
 import { codes } from '../errors.js'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
-import 'node-forge/lib/sha512.js'
 import { generateKeyPair, importKey, unmarshalPrivateKey } from '@libp2p/crypto/keys'
 import type { PeerId } from '@libp2p/interfaces/peer-id'
 import type { Components } from '@libp2p/interfaces/components'