libp2p 0.40.0 → 0.41.0-19e96cc

package/src/connection-manager/dialer/index.ts

@@ -1,12 +1,7 @@
 import { logger } from '@libp2p/logger'
-import all from 'it-all'
-import filter from 'it-filter'
-import { pipe } from 'it-pipe'
 import errCode from 'err-code'
-import type { Multiaddr, Resolver } from '@multiformats/multiaddr'
-import { multiaddr, resolvers } from '@multiformats/multiaddr'
+import { isMultiaddr, Multiaddr, Resolver, multiaddr, resolvers } from '@multiformats/multiaddr'
 import { TimeoutController } from 'timeout-abort-controller'
-import { AbortError } from '@libp2p/interfaces/errors'
 import { anySignal } from 'any-signal'
 import { setMaxListeners } from 'events'
 import { DialAction, DialRequest } from './dial-request.js'
@@ -22,21 +17,15 @@ import {
 import type { Connection, ConnectionGater } from '@libp2p/interface-connection'
 import type { AbortOptions } from '@libp2p/interfaces'
 import type { Startable } from '@libp2p/interfaces/startable'
-import type { PeerId } from '@libp2p/interface-peer-id'
+import { isPeerId, PeerId } from '@libp2p/interface-peer-id'
 import { getPeer } from '../../get-peer.js'
-import sort from 'it-sort'
-import map from 'it-map'
 import type { AddressSorter, PeerStore } from '@libp2p/interface-peer-store'
-import type { ComponentMetricsTracker, Metrics } from '@libp2p/interface-metrics'
+import type { Metrics } from '@libp2p/interface-metrics'
 import type { Dialer } from '@libp2p/interface-connection-manager'
 import type { TransportManager } from '@libp2p/interface-transport'
 
 const log = logger('libp2p:dialer')
 
-const METRICS_COMPONENT = 'dialler'
-const METRICS_PENDING_DIALS = 'pending-dials'
-const METRICS_PENDING_DIAL_TARGETS = 'pending-dial-targets'
-
 export interface DialTarget {
   id: string
   addrs: Multiaddr[]
@@ -84,7 +73,6 @@ export interface DialerInit {
    * Multiaddr resolvers to use when dialing
    */
   resolvers?: Record<string, Resolver>
-  metrics?: ComponentMetricsTracker
 }
 
 export interface DefaultDialerComponents {
@@ -103,7 +91,7 @@ export class DefaultDialer implements Startable, Dialer {
   private readonly maxDialsPerPeer: number
   public tokens: number[]
   public pendingDials: Map<string, PendingDial>
-  public pendingDialTargets: Map<string, PendingDialTarget>
+  public pendingDialTargets: Map<string, AbortController>
   private started: boolean
 
   constructor (components: DefaultDialerComponents, init: DialerInit = {}) {
@@ -115,13 +103,11 @@ export class DefaultDialer implements Startable, Dialer {
     this.tokens = [...new Array(init.maxParallelDials ?? MAX_PARALLEL_DIALS)].map((_, index) => index)
     this.components = components
     this.pendingDials = trackedMap({
-      component: METRICS_COMPONENT,
-      metric: METRICS_PENDING_DIALS,
-      metrics: init.metrics
+      name: 'libp2p_dialler_pending_dials',
+      metrics: components.metrics
     })
     this.pendingDialTargets = trackedMap({
-      component: METRICS_COMPONENT,
-      metric: METRICS_PENDING_DIAL_TARGETS,
+      name: 'libp2p_dialler_pending_dial_targets',
       metrics: components.metrics
     })
 
@@ -154,7 +140,7 @@ export class DefaultDialer implements Startable, Dialer {
     this.pendingDials.clear()
 
     for (const pendingTarget of this.pendingDialTargets.values()) {
-      pendingTarget.reject(new AbortError('Dialer was destroyed'))
+      pendingTarget.abort()
     }
     this.pendingDialTargets.clear()
   }
@@ -164,8 +150,8 @@ export class DefaultDialer implements Startable, Dialer {
    * The dial to the first address that is successfully able to upgrade a connection
    * will be used.
    */
-  async dial (peer: PeerId | Multiaddr, options: AbortOptions = {}): Promise<Connection> {
-    const { id, multiaddrs } = getPeer(peer)
+  async dial (peerIdOrMultiaddr: PeerId | Multiaddr, options: AbortOptions = {}): Promise<Connection> {
+    const { id, multiaddrs } = getPeer(peerIdOrMultiaddr)
 
     if (this.components.peerId.equals(id)) {
       throw errCode(new Error('Tried to dial self'), codes.ERR_DIALED_SELF)
@@ -184,13 +170,35 @@ export class DefaultDialer implements Startable, Dialer {
 
     log('creating dial target for %p', id)
 
-    const dialTarget = await this._createCancellableDialTarget(id, options)
+    // resolving multiaddrs can involve dns lookups so allow them to be aborted
+    const controller = new AbortController()
+    const controllerId = randomId()
+    this.pendingDialTargets.set(controllerId, controller)
+    let signal = controller.signal
+
+    // merge with the passed signal, if any
+    if (options.signal != null) {
+      signal = anySignal([signal, options.signal])
+    }
+
+    let dialTarget: DialTarget
+
+    try {
+      dialTarget = await this._createDialTarget(peerIdOrMultiaddr, {
+        ...options,
+        signal
+      })
+    } finally {
+      // done resolving the multiaddrs so remove the abort controller
+      this.pendingDialTargets.delete(controllerId)
+    }
 
     if (dialTarget.addrs.length === 0) {
       throw errCode(new Error('The dial request has no valid addresses'), codes.ERR_NO_VALID_ADDRESSES)
     }
 
-    const pendingDial = this.pendingDials.get(dialTarget.id) ?? this._createPendingDial(dialTarget, options)
+    // try to join an in-flight dial for this peer if one is available
+    const pendingDial = this.pendingDials.get(dialTarget.id.toString()) ?? this._createPendingDial(dialTarget, options)
 
     try {
       const connection = await pendingDial.promise
@@ -209,74 +217,77 @@ export class DefaultDialer implements Startable, Dialer {
     }
   }
 
-  /**
-   * Connects to a given `peer` by dialing all of its known addresses.
-   * The dial to the first address that is successfully able to upgrade a connection
-   * will be used.
-   */
-  async _createCancellableDialTarget (peer: PeerId, options: AbortOptions): Promise<DialTarget> {
-    // Make dial target promise cancellable
-    const id = `${(parseInt(String(Math.random() * 1e9), 10)).toString()}${Date.now()}`
-    const cancellablePromise = new Promise<DialTarget>((resolve, reject) => {
-      this.pendingDialTargets.set(id, { resolve, reject })
-    })
-
-    try {
-      const dialTarget = await Promise.race([
-        this._createDialTarget(peer, options),
-        cancellablePromise
-      ])
-
-      return dialTarget
-    } finally {
-      this.pendingDialTargets.delete(id)
-    }
-  }
-
   /**
    * Creates a DialTarget. The DialTarget is used to create and track
    * the DialRequest to a given peer.
-   * If a multiaddr is received it should be the first address attempted.
+   *
+   * If a multiaddr is received it should be the only address attempted.
+   *
    * Multiaddrs not supported by the available transports will be filtered out.
    */
-  async _createDialTarget (peer: PeerId, options: AbortOptions): Promise<DialTarget> {
+  async _createDialTarget (peerIdOrMultiaddr: PeerId | Multiaddr, options: AbortOptions): Promise<DialTarget> {
     const _resolve = this._resolve.bind(this)
 
-    const addrs = await pipe(
-      await this.components.peerStore.addressBook.get(peer),
-      (source) => filter(source, async (address) => {
-        return !(await this.components.connectionGater.denyDialMultiaddr(peer, address.multiaddr))
-      }),
-      // Sort addresses so, for example, we try certified public address first
-      (source) => sort(source, this.addressSorter),
-      async function * resolve (source) {
-        for await (const a of source) {
-          yield * await _resolve(a.multiaddr, options)
-        }
-      },
+    let addrs = isMultiaddr(peerIdOrMultiaddr) ? [peerIdOrMultiaddr] : await this._loadAddresses(peerIdOrMultiaddr)
+
+    addrs = (await Promise.all(
+      addrs.map(async (ma) => await _resolve(ma, options))
+    ))
+      .flat()
       // Multiaddrs not supported by the available transports will be filtered out.
-      (source) => filter(source, (ma) => Boolean(this.components.transportManager.transportForMultiaddr(ma))),
-      (source) => map(source, (ma) => {
-        if (peer.toString() === ma.getPeerId()) {
-          return ma
-        }
+      .filter(ma => Boolean(this.components.transportManager.transportForMultiaddr(ma)))
 
-        return ma.encapsulate(`/p2p/${peer.toString()}`)
-      }),
-      async (source) => await all(source)
-    )
+    // deduplicate addresses
+    addrs = [...new Set(addrs)]
 
     if (addrs.length > this.maxAddrsToDial) {
-      await this.components.peerStore.delete(peer)
       throw errCode(new Error('dial with more addresses than allowed'), codes.ERR_TOO_MANY_ADDRESSES)
     }
 
+    const peerId = isPeerId(peerIdOrMultiaddr) ? peerIdOrMultiaddr : undefined
+
+    if (peerId != null) {
+      const peerIdMultiaddr = `/p2p/${peerId.toString()}`
+      addrs = addrs.map(addr => {
+        const addressPeerId = addr.getPeerId()
+
+        if (addressPeerId == null || !peerId.equals(addressPeerId)) {
+          return addr.encapsulate(peerIdMultiaddr)
+        }
+
+        return addr
+      })
+    }
+
     return {
-      id: peer.toString(),
+      id: peerId == null ? randomId() : peerId.toString(),
       addrs
     }
   }
 
+  /**
+   * Loads a list of addresses from the peer store for the passed peer id
+   */
+  async _loadAddresses (peer: PeerId): Promise<Multiaddr[]> {
+    const addresses = await this.components.peerStore.addressBook.get(peer)
+
+    return (await Promise.all(
+      addresses.map(async address => {
+        const deny = await this.components.connectionGater.denyDialMultiaddr(peer, address.multiaddr)
+
+        if (deny) {
+          return false
+        }
+
+        return address
+      })
+    ))
+      .filter(isTruthy)
+      // Sort addresses so, for example, we try certified public address first
+      .sort(this.addressSorter)
+      .map(address => address.multiaddr)
+  }
+
   /**
    * Creates a PendingDial that wraps the underlying DialRequest
    */
@@ -388,3 +399,17 @@ export class DefaultDialer implements Startable, Dialer {
     }
   }
 }
+
+/**
+ * Type safe version of `list.filter(Boolean)`
+ */
+function isTruthy <T> (e: T | false | null | undefined): e is T {
+  return Boolean(e)
+}
+
+/**
+ * Returns a random string
+ */
+function randomId (): string {
+  return `${(parseInt(String(Math.random() * 1e9), 10)).toString()}${Date.now()}`
+}

package/src/connection-manager/index.ts

@@ -2,8 +2,6 @@ import { logger } from '@libp2p/logger'
 import errCode from 'err-code'
 import mergeOptions from 'merge-options'
 import { LatencyMonitor, SummaryObject } from './latency-monitor.js'
-// @ts-expect-error retimer does not have types
-import retimer from 'retimer'
 import type { AbortOptions } from '@libp2p/interfaces'
 import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
 import type { Startable } from '@libp2p/interfaces/startable'
@@ -11,162 +9,33 @@ import { codes } from '../errors.js'
 import { isPeerId, PeerId } from '@libp2p/interface-peer-id'
 import { setMaxListeners } from 'events'
 import type { Connection, MultiaddrConnection } from '@libp2p/interface-connection'
-import type { ConnectionManager, Dialer } from '@libp2p/interface-connection-manager'
+import type { ConnectionManager, ConnectionManagerEvents, Dialer } from '@libp2p/interface-connection-manager'
 import * as STATUS from '@libp2p/interface-connection/status'
-import type { AddressSorter, PeerStore } from '@libp2p/interface-peer-store'
-import { multiaddr, Multiaddr, Resolver } from '@multiformats/multiaddr'
+import type { PeerStore } from '@libp2p/interface-peer-store'
+import { isMultiaddr, multiaddr, Multiaddr } from '@multiformats/multiaddr'
 import { PeerMap } from '@libp2p/peer-collections'
 import { TimeoutController } from 'timeout-abort-controller'
 import { KEEP_ALIVE } from '@libp2p/interface-peer-store/tags'
 import { RateLimiterMemory } from 'rate-limiter-flexible'
 import type { Metrics } from '@libp2p/interface-metrics'
 import type { Upgrader } from '@libp2p/interface-transport'
+import { getPeer } from '../get-peer.js'
+import type { ConnectionManagerConfig } from '../index.js'
 
 const log = logger('libp2p:connection-manager')
 
-const defaultOptions: Partial<ConnectionManagerInit> = {
+const defaultOptions: Partial<ConnectionManagerConfig> = {
   maxConnections: Infinity,
   minConnections: 0,
-  maxData: Infinity,
-  maxSentData: Infinity,
-  maxReceivedData: Infinity,
   maxEventLoopDelay: Infinity,
   pollInterval: 2000,
   autoDialInterval: 10000,
-  movingAverageInterval: 60000,
   inboundConnectionThreshold: 5,
   maxIncomingPendingConnections: 10
 }
 
-const METRICS_SYSTEM = 'libp2p'
-const METRICS_COMPONENT = 'connection-manager'
 const STARTUP_RECONNECT_TIMEOUT = 60000
 
-export interface ConnectionManagerInit {
-  /**
-   * The maximum number of connections to keep open
-   */
-  maxConnections: number
-
-  /**
-   * The minimum number of connections to keep open
-   */
-  minConnections: number
-
-  /**
-   * The max data (in and out), per average interval to allow
-   */
-  maxData?: number
-
-  /**
-   * The max outgoing data, per average interval to allow
-   */
-  maxSentData?: number
-
-  /**
-   * The max incoming data, per average interval to allow
-   */
-  maxReceivedData?: number
-
-  /**
-   * The upper limit the event loop can take to run
-   */
-  maxEventLoopDelay?: number
-
-  /**
-   * How often, in milliseconds, metrics and latency should be checked
-   */
-  pollInterval?: number
-
-  /**
-   * How often, in milliseconds, to compute averages
-   */
-  movingAverageInterval?: number
-
-  /**
-   * If true, try to connect to all discovered peers up to the connection manager limit
-   */
-  autoDial?: boolean
-
-  /**
-   * How long to wait between attempting to keep our number of concurrent connections
-   * above minConnections
-   */
-  autoDialInterval: number
-
-  /**
-   * Sort the known addresses of a peer before trying to dial
-   */
-  addressSorter?: AddressSorter
-
-  /**
-   * Number of max concurrent dials
-   */
-  maxParallelDials?: number
-
-  /**
-   * Number of max addresses to dial for a given peer
-   */
-  maxAddrsToDial?: number
-
-  /**
-   * How long a dial attempt is allowed to take, including DNS resolution
-   * of the multiaddr, opening a socket and upgrading it to a Connection.
-   */
-  dialTimeout?: number
-
-  /**
-   * When a new inbound connection is opened, the upgrade process (e.g. protect,
-   * encrypt, multiplex etc) must complete within this number of ms.
-   */
-  inboundUpgradeTimeout: number
-
-  /**
-   * Number of max concurrent dials per peer
-   */
-  maxDialsPerPeer?: number
-
-  /**
-   * Multiaddr resolvers to use when dialing
-   */
-  resolvers?: Record<string, Resolver>
-
-  /**
-   * On startup we try to dial any peer that has previously been
-   * tagged with KEEP_ALIVE up to this timeout in ms. (default: 60000)
-   */
-  startupReconnectTimeout?: number
-
-  /**
-   * A list of multiaddrs that will always be allowed (except if they are in the
-   * deny list) to open connections to this node even if we've reached maxConnections
-   */
-  allow?: string[]
-
-  /**
-   * A list of multiaddrs that will never be allowed to open connections to
-   * this node under any circumstances
-   */
-  deny?: string[]
-
-  /**
-   * If more than this many connections are opened per second by a single
-   * host, reject subsequent connections
-   */
-  inboundConnectionThreshold?: number
-
-  /**
-   * The maximum number of parallel incoming connections allowed that have yet to
-   * complete the connection upgrade - e.g. choosing connection encryption, muxer, etc
-   */
-  maxIncomingPendingConnections?: number
-}
-
-export interface ConnectionManagerEvents {
-  'peer:connect': CustomEvent<PeerId>
-  'peer:disconnect': CustomEvent<PeerId>
-}
-
 export interface DefaultConnectionManagerComponents {
   peerId: PeerId
   metrics?: Metrics
@@ -175,15 +44,16 @@ export interface DefaultConnectionManagerComponents {
   dialer: Dialer
 }
 
+export type ConnectionManagerInit = Required<ConnectionManagerConfig>
+
 /**
  * Responsible for managing known connections.
  */
 export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEvents> implements ConnectionManager, Startable {
   private readonly components: DefaultConnectionManagerComponents
-  private readonly opts: Required<ConnectionManagerInit>
+  private readonly opts: ConnectionManagerInit
   private readonly connections: Map<string, Connection[]>
   private started: boolean
-  private timer?: ReturnType<retimer>
   private readonly latencyMonitor: LatencyMonitor
   private readonly startupReconnectTimeout: number
   private connectOnStartupController?: TimeoutController
@@ -193,7 +63,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
   private readonly inboundConnectionRateLimiter: RateLimiterMemory
   private incomingPendingConnections: number
 
-  constructor (components: DefaultConnectionManagerComponents, init: ConnectionManagerInit) {
+  constructor (components: DefaultConnectionManagerComponents, init: ConnectionManagerConfig) {
     super()
 
     this.opts = mergeOptions.call({ ignoreUndefined: true }, defaultOptions, init)
@@ -212,7 +82,6 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     this.connections = new Map()
 
     this.started = false
-    this._checkMetrics = this._checkMetrics.bind(this)
 
     this.latencyMonitor = new LatencyMonitor({
       latencyCheckIntervalMs: init.pollInterval,
@@ -250,17 +119,9 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    * only event loop and connection limits will be monitored.
    */
   async start () {
-    if (this.components.metrics != null) {
-      this.timer = this.timer ?? retimer(this._checkMetrics, this.opts.pollInterval)
-    }
-
     // track inbound/outbound connections
-    this.components.metrics?.updateComponentMetric({
-      system: METRICS_SYSTEM,
-      component: METRICS_COMPONENT,
-      metric: 'connections',
-      label: 'direction',
-      value: () => {
+    this.components.metrics?.registerMetricGroup('libp2p_connection_manager_connections', {
+      calculate: () => {
         const metric = {
           inbound: 0,
           outbound: 0
@@ -281,12 +142,9 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     })
 
     // track total number of streams per protocol
-    this.components.metrics?.updateComponentMetric({
-      system: METRICS_SYSTEM,
-      component: METRICS_COMPONENT,
-      metric: 'protocol-streams-total',
+    this.components.metrics?.registerMetricGroup('libp2p_protocol_streams_total', {
       label: 'protocol',
-      value: () => {
+      calculate: () => {
         const metric: Record<string, number> = {}
 
         for (const conns of this.connections.values()) {
@@ -304,12 +162,9 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     })
 
     // track 90th percentile of streams per protocol
-    this.components.metrics?.updateComponentMetric({
-      system: METRICS_SYSTEM,
-      component: METRICS_COMPONENT,
-      metric: 'protocol-streams-per-connection-90th-percentile',
+    this.components.metrics?.registerMetricGroup('libp2p_connection_manager_protocol_streams_per_connection_90th_percentile', {
       label: 'protocol',
-      value: () => {
+      calculate: () => {
         const allStreams: Record<string, number[]> = {}
 
         for (const conns of this.connections.values()) {
@@ -407,8 +262,6 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    * Stops the Connection Manager
    */
   async stop () {
-    this.timer?.clear()
-
     this.latencyMonitor.removeEventListener('data', this._onLatencyMeasure)
     this.latencyMonitor.stop()
 
@@ -440,31 +293,6 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     this.connections.clear()
   }
 
-  /**
-   * Checks the libp2p metrics to determine if any values have exceeded
-   * the configured maximums.
-   *
-   * @private
-   */
-  async _checkMetrics () {
-    const metrics = this.components.metrics
-
-    if (metrics != null) {
-      try {
-        const movingAverages = metrics.getGlobal().getMovingAverages()
-        const received = movingAverages.dataReceived[this.opts.movingAverageInterval].movingAverage
-        await this._checkMaxLimit('maxReceivedData', received)
-        const sent = movingAverages.dataSent[this.opts.movingAverageInterval].movingAverage
-        await this._checkMaxLimit('maxSentData', sent)
-        const total = received + sent
-        await this._checkMaxLimit('maxData', total)
-        log.trace('metrics update', total)
-      } finally {
-        this.timer = retimer(this._checkMetrics, this.opts.pollInterval)
-      }
-    }
-  }
-
   onConnect (evt: CustomEvent<Connection>) {
     void this._onConnect(evt).catch(err => {
       log.error(err)
@@ -524,8 +352,6 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     } else if (storedConn != null) {
       this.connections.delete(peerId)
       this.dispatchEvent(new CustomEvent<Connection>('peer:disconnect', { detail: connection }))
-
-      this.components.metrics?.onPeerDisconnected(connection.remotePeer)
     }
   }
 
@@ -543,7 +369,18 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     return conns
   }
 
-  async openConnection (peerId: PeerId, options: AbortOptions = {}): Promise<Connection> {
+  async openConnection (peerIdOrMultiaddr: PeerId | Multiaddr, options: AbortOptions = {}): Promise<Connection> {
+    let peerId: PeerId
+
+    if (isPeerId(peerIdOrMultiaddr)) {
+      peerId = peerIdOrMultiaddr
+    } else if (isMultiaddr(peerIdOrMultiaddr)) {
+      const info = getPeer(peerIdOrMultiaddr)
+      peerId = info.id
+    } else {
+      throw errCode(new TypeError('Can only open connections to PeerIds or Multiaddrs'), codes.ERR_INVALID_PARAMETERS)
+    }
+
     log('dial to %p', peerId)
     const existingConnections = this.getConnections(peerId)
 
@@ -566,7 +403,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     }
 
     try {
-      const connection = await this.components.dialer.dial(peerId, options)
+      const connection = await this.components.dialer.dial(peerIdOrMultiaddr, options)
       let peerConnections = this.connections.get(peerId.toString())
 
       if (peerConnections == null) {

package/src/fetch/README.md

@@ -1,8 +1,13 @@
-libp2p-fetch JavaScript Implementation
+libp2p-fetch JavaScript Implementation  <!-- omit in toc -->
 =====================================
 
 > Libp2p fetch protocol JavaScript implementation
 
+## Table of contents <!-- omit in toc -->
+
+- [Overview](#overview)
+- [Usage](#usage)
+
 ## Overview
 
 An implementation of the Fetch protocol as described here: https://github.com/libp2p/specs/tree/master/fetch
@@ -27,7 +32,7 @@ async function my_subsystem_key_lookup(key) {
 }
 
 // Enable this peer to respond to fetch requests for keys that begin with '/my_subsystem_key_prefix/'
-const libp2p = Libp2p.create(...)
+const libp2p = createLibp2p()
 libp2p.fetchService.registerLookupFunction('/my_subsystem_key_prefix/', my_subsystem_key_lookup)
 
 const key = '/my_subsystem_key_prefix/{...}'