libp2p 0.40.0 → 0.41.0-19e96cc

package/src/pnet/README.md

@@ -1,34 +1,41 @@
-js-libp2p-pnet
+js-libp2p-pnet <!-- omit in toc -->
 ==================
 
 > Connection protection management for libp2p leveraging PSK encryption via XSalsa20.
 
 **Note**: git history prior to merging into js-libp2p can be found in the original repository, https://github.com/libp2p/js-libp2p-pnet.
 
-## Table of Contents
+## Table of Contents <!-- omit in toc -->
 
-- [js-libp2p-pnet](#js-libp2p-pnet)
-  - [Table of Contents](#table-of-contents)
-  - [Usage](#usage)
-    - [Examples](#examples)
-    - [Private Shared Keys](#private-shared-keys)
-    - [PSK Generation](#psk-generation)
-      - [From libp2p-pnet](#from-libp2p-pnet)
-      - [From a module using libp2p](#from-a-module-using-libp2p)
-      - [Programmatically](#programmatically)
+- [Usage](#usage)
+- [Examples](#examples)
+- [Private Shared Keys](#private-shared-keys)
+- [PSK Generation](#psk-generation)
+    - [From a module using libp2p](#from-a-module-using-libp2p)
+    - [Programmatically](#programmatically)
 
 ## Usage
 
 ```js
-const Protector from 'libp2p-pnet')
-const protector = new Protector(swarmKeyBuffer)
-const privateConnection = protector.protect(myPublicConnection, (err) => { })
+import { createLibp2p } from 'libp2p'
+import { preSharedKey, generateKey } from 'libp2p/pnet'
+
+// Create a Uint8Array and write the swarm key to it
+const swarmKey = new Uint8Array(95)
+generateKey(swarmKey)
+
+const node = await createLibp2p({
+  // ...other options
+  connectionProtector: preSharedKey({
+    psk: swarmKey
+  })
+})
 ```
 
-### Examples
+## Examples
 [Private Networks with IPFS](../../examples/pnet-ipfs)
 
-### Private Shared Keys
+## Private Shared Keys
 
 Private Shared Keys are expected to be in the following format:
 
@@ -38,33 +45,28 @@ Private Shared Keys are expected to be in the following format:
 dffb7e3135399a8b1612b2aaca1c36a3a8ac2cd0cca51ceeb2ced87d308cac6d
 ```
 
-### PSK Generation
+## PSK Generation
 
 A utility method has been created to generate a key for your private network. You can
 use one of the methods below to generate your key.
 
-#### From libp2p-pnet
-
-If you have libp2p locally, you can run the following from the projects root.
-
-```sh
-node ./src/pnet/key-generator.js > swarm.key
-```
-
 #### From a module using libp2p
 
 If you have a module locally that depends on libp2p, you can run the following from
 that project, assuming the node_modules are installed.
 
-```sh
-node -e "require('libp2p/src/pnet').generate(process.stdout)" > swarm.key
+```console
+node -e "import('libp2p/pnet').then(({ generateKey }) => generateKey(process.stdout))" > swarm.key
 ```
 
 #### Programmatically
 
 ```js
-const writeKey from 'libp2p/src/pnet').generate
+import fs from 'fs'
+import { generateKey } from 'libp2p/pnet'
+
 const swarmKey = new Uint8Array(95)
-writeKey(swarmKey)
+generateKey(swarmKey)
+
 fs.writeFileSync('swarm.key', swarmKey)
 ```

package/src/pnet/index.ts

@@ -1,3 +1,27 @@
+/**
+ * @packageDocumentation
+ *
+ * Connection protection management for libp2p leveraging PSK encryption via XSalsa20.
+ *
+ * @example
+ *
+ * ```typescript
+ * import { createLibp2p } from 'libp2p'
+ * import { preSharedKey, generateKey } from 'libp2p/pnet'
+ *
+ * // Create a Uint8Array and write the swarm key to it
+ * const swarmKey = new Uint8Array(95)
+ * generateKey(swarmKey)
+ *
+ * const node = await createLibp2p({
+ *   // ...other options
+ *   connectionProtector: preSharedKey({
+ *     psk: swarmKey
+ *   })
+ * })
+ * ```
+ */
+
 import { logger } from '@libp2p/logger'
 import { pipe } from 'it-pipe'
 import errCode from 'err-code'

package/src/pnet/key-generator.ts

@@ -5,24 +5,19 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 /**
  * Generates a PSK that can be used in a libp2p-pnet private network
  *
- * @param {Uint8Array} bytes - An object to write the psk into
+ * @param {Uint8Array | NodeJS.WriteStream} bytes - An object to write the psk into
  * @returns {void}
  */
-export function generateKey (bytes: Uint8Array) {
+export function generateKey (bytes: Uint8Array | NodeJS.WriteStream) {
   const psk = uint8ArrayToString(randomBytes(KEY_LENGTH), 'base16')
   const key = uint8ArrayFromString('/key/swarm/psk/1.0.0/\n/base16/\n' + psk)
 
-  bytes.set(key)
+  if (bytes instanceof Uint8Array) {
+    bytes.set(key)
+  } else {
+    bytes.write(key)
+  }
 }
 
 export const NONCE_LENGTH = 24
 export const KEY_LENGTH = 32
-
-try {
-  if (require.main === module) {
-    // @ts-expect-error
-    generate(process.stdout)
-  }
-} catch (error: any) {
-
-}

package/src/registrar.ts

@@ -35,8 +35,10 @@ export class DefaultRegistrar implements Registrar {
 
     this._onDisconnect = this._onDisconnect.bind(this)
     this._onProtocolChange = this._onProtocolChange.bind(this)
+    this._onConnect = this._onConnect.bind(this)
 
     this.components.connectionManager.addEventListener('peer:disconnect', this._onDisconnect)
+    this.components.connectionManager.addEventListener('peer:connect', this._onConnect)
 
     // happens after identify
     this.components.peerStore.addEventListener('change:protocols', this._onProtocolChange)
@@ -176,12 +178,38 @@ export class DefaultRegistrar implements Registrar {
       })
   }
 
+  /**
+   * On peer connected if we already have their protocols. Usually used for reconnects
+   * as change:protocols event won't be emitted due to identical protocols.
+   */
+  _onConnect (evt: CustomEvent<Connection>) {
+    const connection = evt.detail
+
+    void this.components.peerStore.protoBook.get(connection.remotePeer)
+      .then(peerProtocols => {
+        for (const protocol of peerProtocols) {
+          const topologies = this.topologies.get(protocol)
+
+          if (topologies == null) {
+            // no topologies are interested in this protocol
+            continue
+          }
+
+          for (const topology of topologies.values()) {
+            topology.onConnect(connection.remotePeer, connection)
+          }
+        }
+      })
+      .catch(err => {
+        log.error(err)
+      })
+  }
+
   /**
    * Check if a new peer support the multicodecs for this topology
    */
   _onProtocolChange (evt: CustomEvent<PeerProtocolsChangeData>) {
     const { peerId, protocols, oldProtocols } = evt.detail
-
     const removed = oldProtocols.filter(protocol => !protocols.includes(protocol))
     const added = protocols.filter(protocol => !oldProtocols.includes(protocol))
 
@@ -212,7 +240,6 @@ export class DefaultRegistrar implements Registrar {
         if (connection == null) {
           continue
         }
-
         topology.onConnect(peerId, connection)
       }
     }

package/src/transport-manager.ts

@@ -38,8 +38,7 @@ export class DefaultTransportManager extends EventEmitter<TransportManagerEvents
     this.started = false
     this.transports = new Map<string, Transport>()
     this.listeners = trackedMap({
-      component: 'transport-manager',
-      metric: 'listeners',
+      name: 'libp2p_transport_manager_listeners',
       metrics: this.components.metrics
     })
     this.faultTolerance = init.faultTolerance ?? FaultTolerance.FATAL_ALL

package/src/upgrader.ts

@@ -1,11 +1,8 @@
 import { logger } from '@libp2p/logger'
 import errCode from 'err-code'
 import * as mss from '@libp2p/multistream-select'
-import { pipe } from 'it-pipe'
-// @ts-expect-error mutable-proxy does not export types
-import mutableProxy from 'mutable-proxy'
 import { codes } from './errors.js'
-import { createConnection } from '@libp2p/connection'
+import { createConnection } from './connection/index.js'
 import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
 import { peerIdFromString } from '@libp2p/peer-id'
 import type { MultiaddrConnection, Connection, Stream, ConnectionGater, ConnectionProtector } from '@libp2p/interface-connection'
@@ -134,7 +131,7 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
   /**
    * Upgrades an inbound connection
    */
-  async upgradeInbound (maConn: MultiaddrConnection): Promise<Connection> {
+  async upgradeInbound (maConn: MultiaddrConnection, opts?: UpgraderOptions): Promise<Connection> {
     const accept = await this.components.connectionManager.acceptIncomingConnection(maConn)
 
     if (!accept) {
@@ -146,9 +143,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
     let upgradedConn: Duplex<Uint8Array>
     let muxerFactory: StreamMuxerFactory | undefined
     let cryptoProtocol
-    let setPeer
-    let proxyPeer
-    const metrics = this.components.metrics
 
     const timeoutController = new TimeoutController(this.inboundUpgradeTimeout)
 
@@ -166,49 +160,62 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
         throw errCode(new Error('The multiaddr connection is blocked by gater.acceptConnection'), codes.ERR_CONNECTION_INTERCEPTED)
       }
 
-      if (metrics != null) {
-        ({ setTarget: setPeer, proxy: proxyPeer } = mutableProxy())
-        const idString = `${(Math.random() * 1e9).toString(36)}${Date.now()}`
-        setPeer({ toString: () => idString })
-        metrics.trackStream({ stream: maConn, remotePeer: proxyPeer })
-      }
+      this.components.metrics?.trackMultiaddrConnection(maConn)
 
       log('starting the inbound connection upgrade')
 
       // Protect
       let protectedConn = maConn
-      const protector = this.components.connectionProtector
 
-      if (protector != null) {
-        log('protecting the inbound connection')
-        protectedConn = await protector.protect(maConn)
+      if (opts?.skipProtection !== true) {
+        const protector = this.components.connectionProtector
+
+        if (protector != null) {
+          log('protecting the inbound connection')
+          protectedConn = await protector.protect(maConn)
+        }
       }
 
       try {
         // Encrypt the connection
-        ({
-          conn: encryptedConn,
-          remotePeer,
-          protocol: cryptoProtocol
-        } = await this._encryptInbound(protectedConn))
+        encryptedConn = protectedConn
+        if (opts?.skipEncryption !== true) {
+          ({
+            conn: encryptedConn,
+            remotePeer,
+            protocol: cryptoProtocol
+          } = await this._encryptInbound(protectedConn))
+
+          if (await this.components.connectionGater.denyInboundEncryptedConnection(remotePeer, {
+            ...protectedConn,
+            ...encryptedConn
+          })) {
+            throw errCode(new Error('The multiaddr connection is blocked by gater.acceptEncryptedConnection'), codes.ERR_CONNECTION_INTERCEPTED)
+          }
+        } else {
+          const idStr = maConn.remoteAddr.getPeerId()
 
-        if (await this.components.connectionGater.denyInboundEncryptedConnection(remotePeer, {
-          ...protectedConn,
-          ...encryptedConn
-        })) {
-          throw errCode(new Error('The multiaddr connection is blocked by gater.acceptEncryptedConnection'), codes.ERR_CONNECTION_INTERCEPTED)
+          if (idStr == null) {
+            throw errCode(new Error('inbound connection that skipped encryption must have a peer id'), codes.ERR_INVALID_MULTIADDR)
+          }
+
+          const remotePeerId = peerIdFromString(idStr)
+
+          cryptoProtocol = 'native'
+          remotePeer = remotePeerId
         }
 
-        // Multiplex the connection
-        if (this.muxers.size > 0) {
+        upgradedConn = encryptedConn
+        if (opts?.muxerFactory != null) {
+          muxerFactory = opts.muxerFactory
+        } else if (this.muxers.size > 0) {
+          // Multiplex the connection
           const multiplexed = await this._multiplexInbound({
             ...protectedConn,
             ...encryptedConn
           }, this.muxers)
           muxerFactory = multiplexed.muxerFactory
           upgradedConn = multiplexed.stream
-        } else {
-          upgradedConn = encryptedConn
         }
       } catch (err: any) {
         log.error('Failed to upgrade inbound connection', err)
@@ -222,11 +229,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
         throw errCode(new Error('The multiaddr connection is blocked by gater.acceptEncryptedConnection'), codes.ERR_CONNECTION_INTERCEPTED)
       }
 
-      if (metrics != null) {
-        metrics.updatePlaceholder(proxyPeer, remotePeer)
-        setPeer(remotePeer)
-      }
-
       log('Successfully upgraded inbound connection')
 
       return this._createConnection({
@@ -263,16 +265,8 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
     let upgradedConn
     let cryptoProtocol
     let muxerFactory
-    let setPeer
-    let proxyPeer
-    const metrics = this.components.metrics
-
-    if (metrics != null) {
-      ({ setTarget: setPeer, proxy: proxyPeer } = mutableProxy())
-      const idString = `${(Math.random() * 1e9).toString(36)}${Date.now()}`
-      setPeer({ toB58String: () => idString })
-      metrics.trackStream({ stream: maConn, remotePeer: proxyPeer })
-    }
+
+    this.components.metrics?.trackMultiaddrConnection(maConn)
 
     log('Starting the outbound connection upgrade')
 
@@ -335,11 +329,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
       throw errCode(new Error('The multiaddr connection is blocked by gater.acceptEncryptedConnection'), codes.ERR_CONNECTION_INTERCEPTED)
     }
 
-    if (metrics != null) {
-      metrics.updatePlaceholder(proxyPeer, remotePeer)
-      setPeer(remotePeer)
-    }
-
     log('Successfully upgraded outbound connection')
 
     return this._createConnection({
@@ -385,12 +374,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
               const { stream, protocol } = await mss.handle(muxedStream, protocols)
               log('%s: incoming stream opened on %s', direction, protocol)
 
-              const metrics = this.components.metrics
-
-              if (metrics != null) {
-                metrics.trackStream({ stream, remotePeer, protocol })
-              }
-
               if (connection == null) {
                 return
               }
@@ -415,6 +398,8 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
               this.components.peerStore.protoBook.add(remotePeer, [protocol]).catch(err => log.error(err))
 
               connection.addStream(muxedStream)
+              this.components.metrics?.trackProtocolStream(muxedStream, connection)
+
               this._onStream({ connection, stream: muxedStream, protocol })
             })
             .catch(err => {
@@ -438,7 +423,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
 
         log('%s: starting new stream on %s', direction, protocols)
         const muxedStream = await muxer.newStream()
-        const metrics = this.components.metrics
         let controller: TimeoutController | undefined
 
         try {
@@ -456,10 +440,6 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
 
           const { stream, protocol } = await mss.select(muxedStream, protocols, options)
 
-          if (metrics != null) {
-            metrics.trackStream({ stream, remotePeer, protocol })
-          }
-
           const outgoingLimit = findOutgoingStreamLimit(protocol, this.components.registrar)
           const streamCount = countStreams(protocol, 'outbound', connection)
 
@@ -480,6 +460,8 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
           muxedStream.sink = stream.sink
           muxedStream.stat.protocol = protocol
 
+          this.components.metrics?.trackProtocolStream(muxedStream, connection)
+
           return muxedStream
         } catch (err: any) {
           log.error('could not create new stream', err)
@@ -501,7 +483,12 @@ export class DefaultUpgrader extends EventEmitter<UpgraderEvents> implements Upg
       }
 
       // Pipe all data through the muxer
-      pipe(upgradedConn, muxer, upgradedConn).catch(log.error)
+      void Promise.all([
+        muxer.sink(upgradedConn.source),
+        upgradedConn.sink(muxer.source)
+      ]).catch(err => {
+        log.error(err)
+      })
     }
 
     const _timeline = maConn.timeline

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.40.0'
+export const version = '0.41.0-19e96cc'
 export const name = 'libp2p'

package/dist/src/metrics/index.d.ts

@@ -1,93 +0,0 @@
-import { DefaultStats } from './stats.js';
-import type { ComponentMetricsUpdate, Metrics, Stats, TrackedMetric, TrackStreamOptions } from '@libp2p/interface-metrics';
-import type { PeerId } from '@libp2p/interface-peer-id';
-import type { Startable } from '@libp2p/interfaces/startable';
-export interface OnMessageOptions {
-    remotePeer: PeerId;
-    protocol?: string;
-    direction: 'in' | 'out';
-    dataLength: number;
-}
-export interface MetricsInit {
-    enabled: boolean;
-    computeThrottleMaxQueueSize: number;
-    computeThrottleTimeout: number;
-    movingAverageIntervals: number[];
-    maxOldPeersRetention: number;
-}
-export declare class DefaultMetrics implements Metrics, Startable {
-    globalStats: DefaultStats;
-    private readonly enabled;
-    private readonly peerStats;
-    private readonly protocolStats;
-    private readonly oldPeers;
-    private running;
-    private readonly systems;
-    private readonly statsInit;
-    constructor(init: MetricsInit);
-    isStarted(): boolean;
-    /**
-     * Must be called for stats to saved. Any data pushed for tracking
-     * will be ignored.
-     */
-    start(): Promise<void>;
-    /**
-     * Stops all averages timers and prevents new data from being tracked.
-     * Once `stop` is called, `start` must be called to resume stats tracking.
-     */
-    stop(): Promise<void>;
-    /**
-     * Gets the global `Stats` object
-     */
-    getGlobal(): DefaultStats;
-    /**
-     * Returns a list of `PeerId` strings currently being tracked
-     */
-    getPeers(): string[];
-    getComponentMetrics(): Map<string, Map<string, Map<string, TrackedMetric>>>;
-    updateComponentMetric(update: ComponentMetricsUpdate): void;
-    /**
-     * Returns the `Stats` object for the given `PeerId` whether it
-     * is a live peer, or in the disconnected peer LRU cache.
-     */
-    forPeer(peerId: PeerId): Stats | undefined;
-    /**
-     * Returns a list of all protocol strings currently being tracked
-     */
-    getProtocols(): string[];
-    /**
-     * Returns the `Stats` object for the given `protocol`
-     */
-    forProtocol(protocol: string): Stats | undefined;
-    /**
-     * Should be called when all connections to a given peer
-     * have closed. The `Stats` collection for the peer will
-     * be stopped and moved to an LRU for temporary retention.
-     */
-    onPeerDisconnected(peerId: PeerId): void;
-    /**
-     * Takes the metadata for a message and tracks it in the
-     * appropriate categories. If the protocol is present, protocol
-     * stats will also be tracked.
-     */
-    _onMessage(opts: OnMessageOptions): void;
-    /**
-     * Replaces the `PeerId` string with the given `peerId`.
-     * If stats are already being tracked for the given `peerId`, the
-     * placeholder stats will be merged with the existing stats.
-     *
-     * @param {PeerId} placeholder - A peerId string
-     * @param {PeerId} peerId
-     * @returns {void}
-     */
-    updatePlaceholder(placeholder: PeerId, peerId: PeerId): void;
-    /**
-     * Tracks data running through a given Duplex Iterable `stream`. If
-     * the `peerId` is not provided, a placeholder string will be created and
-     * returned. This allows lazy tracking of a peer when the peer is not yet known.
-     * When the `PeerId` is known, `Metrics.updatePlaceholder` should be called
-     * with the placeholder string returned from here, and the known `PeerId`.
-     */
-    trackStream(opts: TrackStreamOptions): void;
-}
-//# sourceMappingURL=index.d.ts.map

package/dist/src/metrics/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/metrics/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAa,MAAM,YAAY,CAAA;AACpD,OAAO,KAAK,EAAE,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC1H,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAA;AAY7D,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,IAAI,GAAG,KAAK,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,2BAA2B,EAAE,MAAM,CAAA;IACnC,sBAAsB,EAAE,MAAM,CAAA;IAC9B,sBAAsB,EAAE,MAAM,EAAE,CAAA;IAChC,oBAAoB,EAAE,MAAM,CAAA;CAC7B;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,SAAS;IAChD,WAAW,EAAE,YAAY,CAAA;IAEhC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2B;IACrD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsD;IAC9E,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;gBAExB,IAAI,EAAE,WAAW;IAe9B,SAAS;IAIT;;;OAGG;IACG,KAAK;IAQX;;;OAGG;IACG,IAAI;IAiBV;;OAEG;IACH,SAAS;IAIT;;OAEG;IACH,QAAQ;IAIR,mBAAmB;IAInB,qBAAqB,CAAE,MAAM,EAAE,sBAAsB;IA8BrD;;;OAGG;IACH,OAAO,CAAE,MAAM,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;IAK3C;;OAEG;IACH,YAAY,IAAK,MAAM,EAAE;IAIzB;;OAEG;IACH,WAAW,CAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;IAIjD;;;;OAIG;IACH,kBAAkB,CAAE,MAAM,EAAE,MAAM;IAYlC;;;;OAIG;IACH,UAAU,CAAE,IAAI,EAAE,gBAAgB;IAkClC;;;;;;;;OAQG;IACH,iBAAiB,CAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAwBtD;;;;;;OAMG;IACH,WAAW,CAAE,IAAI,EAAE,kBAAkB,GAAG,IAAI;CA+B7C"}