libp2p 0.37.3 → 0.38.0

package/src/fetch/index.ts

@@ -3,21 +3,30 @@ import errCode from 'err-code'
 import { codes } from '../errors.js'
 import * as lp from 'it-length-prefixed'
 import { FetchRequest, FetchResponse } from './pb/proto.js'
-import { handshake } from 'it-handshake'
 import { PROTOCOL_NAME, PROTOCOL_VERSION } from './constants.js'
-import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { PeerId } from '@libp2p/interface-peer-id'
 import type { Startable } from '@libp2p/interfaces/startable'
-import type { Stream } from '@libp2p/interfaces/connection'
-import type { IncomingStreamData } from '@libp2p/interfaces/registrar'
-import type { Components } from '@libp2p/interfaces/components'
+import type { Stream } from '@libp2p/interface-connection'
+import type { IncomingStreamData } from '@libp2p/interface-registrar'
+import type { Components } from '@libp2p/components'
 import type { AbortOptions } from '@libp2p/interfaces'
-import type { Duplex } from 'it-stream-types'
 import { abortableDuplex } from 'abortable-iterator'
+import { pipe } from 'it-pipe'
+import first from 'it-first'
+import { TimeoutController } from 'timeout-abort-controller'
+import { setMaxListeners } from 'events'
 
 const log = logger('libp2p:fetch')
 
 export interface FetchServiceInit {
   protocolPrefix: string
+  maxInboundStreams: number
+  maxOutboundStreams: number
+
+  /**
+   * How long we should wait for a remote peer to send any data
+   */
+  timeout: number
 }
 
 export interface HandleMessageOptions {
@@ -40,6 +49,7 @@ export class FetchService implements Startable {
   private readonly components: Components
   private readonly lookupFunctions: Map<string, LookupFunction>
   private started: boolean
+  private readonly init: FetchServiceInit
 
   constructor (components: Components, init: FetchServiceInit) {
     this.started = false
@@ -47,13 +57,21 @@ export class FetchService implements Startable {
     this.protocol = `/${init.protocolPrefix ?? 'libp2p'}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`
     this.lookupFunctions = new Map() // Maps key prefix to value lookup function
     this.handleMessage = this.handleMessage.bind(this)
+    this.init = init
   }
 
   async start () {
     await this.components.getRegistrar().handle(this.protocol, (data) => {
-      void this.handleMessage(data).catch(err => {
-        log.error(err)
-      })
+      void this.handleMessage(data)
+        .catch(err => {
+          log.error(err)
+        })
+        .finally(() => {
+          data.stream.close()
+        })
+    }, {
+      maxInboundStreams: this.init.maxInboundStreams,
+      maxOutboundStreams: this.init.maxOutboundStreams
     })
     this.started = true
   }
@@ -74,35 +92,69 @@ export class FetchService implements Startable {
     log('dialing %s to %p', this.protocol, peer)
 
     const connection = await this.components.getConnectionManager().openConnection(peer, options)
-    const { stream } = await connection.newStream([this.protocol], options)
-    let source: Duplex<Uint8Array> = stream
-
-    // make stream abortable if AbortSignal passed
-    if (options.signal != null) {
-      source = abortableDuplex(stream, options.signal)
+    let timeoutController
+    let signal = options.signal
+    let stream: Stream | undefined
+
+    // create a timeout if no abort signal passed
+    if (signal == null) {
+      timeoutController = new TimeoutController(this.init.timeout)
+      signal = timeoutController.signal
+
+      try {
+        // fails on node < 15.4
+        setMaxListeners?.(Infinity, timeoutController.signal)
+      } catch {}
     }
 
-    const shake = handshake(source)
-
-    // send message
-    shake.write(lp.encode.single(FetchRequest.encode({ identifier: key })).slice())
+    try {
+      stream = await connection.newStream([this.protocol], {
+        signal
+      })
 
-    // read response
-    // @ts-expect-error fromReader returns a Source which has no .next method
-    const response = FetchResponse.decode((await lp.decode.fromReader(shake.reader).next()).value.slice())
-    switch (response.status) {
-      case (FetchResponse.StatusCode.OK): {
-        return response.data
-      }
-      case (FetchResponse.StatusCode.NOT_FOUND): {
-        return null
+      // make stream abortable
+      const source = abortableDuplex(stream, signal)
+
+      const result = await pipe(
+        [FetchRequest.encode({ identifier: key })],
+        lp.encode(),
+        source,
+        lp.decode(),
+        async function (source) {
+          const buf = await first(source)
+
+          if (buf == null) {
+            throw errCode(new Error('No data received'), codes.ERR_INVALID_MESSAGE)
+          }
+
+          const response = FetchResponse.decode(buf)
+
+          switch (response.status) {
+            case (FetchResponse.StatusCode.OK): {
+              return response.data
+            }
+            case (FetchResponse.StatusCode.NOT_FOUND): {
+              return null
+            }
+            case (FetchResponse.StatusCode.ERROR): {
+              const errmsg = (new TextDecoder()).decode(response.data)
+              throw errCode(new Error('Error in fetch protocol response: ' + errmsg), codes.ERR_INVALID_PARAMETERS)
+            }
+            default: {
+              throw errCode(new Error('Unknown response status'), codes.ERR_INVALID_MESSAGE)
+            }
+          }
+        }
+      )
+
+      return result ?? null
+    } finally {
+      if (timeoutController != null) {
+        timeoutController.clear()
       }
-      case (FetchResponse.StatusCode.ERROR): {
-        const errmsg = (new TextDecoder()).decode(response.data)
-        throw errCode(new Error('Error in fetch protocol response: ' + errmsg), codes.ERR_INVALID_PARAMETERS)
-      }
-      default: {
-        throw errCode(new Error('Unknown response status'), codes.ERR_INVALID_MESSAGE)
+
+      if (stream != null) {
+        stream.close()
       }
     }
   }
@@ -114,25 +166,40 @@ export class FetchService implements Startable {
    */
   async handleMessage (data: IncomingStreamData) {
     const { stream } = data
-    const shake = handshake(stream)
-    // @ts-expect-error fromReader returns a Source which has no .next method
-    const request = FetchRequest.decode((await lp.decode.fromReader(shake.reader).next()).value.slice())
-
-    let response: FetchResponse
-    const lookup = this._getLookupFunction(request.identifier)
-    if (lookup != null) {
-      const data = await lookup(request.identifier)
-      if (data != null) {
-        response = { status: FetchResponse.StatusCode.OK, data }
-      } else {
-        response = { status: FetchResponse.StatusCode.NOT_FOUND, data: new Uint8Array(0) }
-      }
-    } else {
-      const errmsg = (new TextEncoder()).encode('No lookup function registered for key: ' + request.identifier)
-      response = { status: FetchResponse.StatusCode.ERROR, data: errmsg }
-    }
-
-    shake.write(lp.encode.single(FetchResponse.encode(response)).slice())
+    const self = this
+
+    await pipe(
+      stream,
+      lp.decode(),
+      async function * (source) {
+        const buf = await first(source)
+
+        if (buf == null) {
+          throw errCode(new Error('No data received'), codes.ERR_INVALID_MESSAGE)
+        }
+
+        // for await (const buf of source) {
+        const request = FetchRequest.decode(buf)
+
+        let response: FetchResponse
+        const lookup = self._getLookupFunction(request.identifier)
+        if (lookup != null) {
+          const data = await lookup(request.identifier)
+          if (data != null) {
+            response = { status: FetchResponse.StatusCode.OK, data }
+          } else {
+            response = { status: FetchResponse.StatusCode.NOT_FOUND, data: new Uint8Array(0) }
+          }
+        } else {
+          const errmsg = (new TextEncoder()).encode('No lookup function registered for key: ' + request.identifier)
+          response = { status: FetchResponse.StatusCode.ERROR, data: errmsg }
+        }
+
+        yield FetchResponse.encode(response)
+      },
+      lp.encode(),
+      stream
+    )
   }
 
   /**

package/src/fetch/pb/proto.ts

@@ -1,7 +1,8 @@
 /* eslint-disable import/export */
 /* eslint-disable @typescript-eslint/no-namespace */
 
-import { encodeMessage, decodeMessage, message, string, enumeration, bytes } from 'protons-runtime'
+import { encodeMessage, decodeMessage, message, enumeration } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
 import type { Codec } from 'protons-runtime'
 
 export interface FetchRequest {
@@ -9,17 +10,59 @@ export interface FetchRequest {
 }
 
 export namespace FetchRequest {
+  let _codec: Codec<FetchRequest>
+
   export const codec = (): Codec<FetchRequest> => {
-    return message<FetchRequest>({
-      1: { name: 'identifier', codec: string }
-    })
+    if (_codec == null) {
+      _codec = message<FetchRequest>((obj, writer, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          writer.fork()
+        }
+
+        if (obj.identifier != null) {
+          writer.uint32(10)
+          writer.string(obj.identifier)
+        } else {
+          throw new Error('Protocol error: required field "identifier" was not found in object')
+        }
+
+        if (opts.lengthDelimited !== false) {
+          writer.ldelim()
+        }
+      }, (reader, length) => {
+        const obj: any = {}
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1:
+              obj.identifier = reader.string()
+              break
+            default:
+              reader.skipType(tag & 7)
+              break
+          }
+        }
+
+        if (obj.identifier == null) {
+          throw new Error('Protocol error: value for required field "identifier" was not found in protobuf')
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
   }
 
   export const encode = (obj: FetchRequest): Uint8Array => {
     return encodeMessage(obj, FetchRequest.codec())
   }
 
-  export const decode = (buf: Uint8Array): FetchRequest => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): FetchRequest => {
     return decodeMessage(buf, FetchRequest.codec())
   }
 }
@@ -44,22 +87,77 @@ export namespace FetchResponse {
 
   export namespace StatusCode {
     export const codec = () => {
-      return enumeration<typeof StatusCode>(__StatusCodeValues)
+      return enumeration<StatusCode>(__StatusCodeValues)
     }
   }
 
+  let _codec: Codec<FetchResponse>
+
   export const codec = (): Codec<FetchResponse> => {
-    return message<FetchResponse>({
-      1: { name: 'status', codec: FetchResponse.StatusCode.codec() },
-      2: { name: 'data', codec: bytes }
-    })
+    if (_codec == null) {
+      _codec = message<FetchResponse>((obj, writer, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          writer.fork()
+        }
+
+        if (obj.status != null) {
+          writer.uint32(8)
+          FetchResponse.StatusCode.codec().encode(obj.status, writer)
+        } else {
+          throw new Error('Protocol error: required field "status" was not found in object')
+        }
+
+        if (obj.data != null) {
+          writer.uint32(18)
+          writer.bytes(obj.data)
+        } else {
+          throw new Error('Protocol error: required field "data" was not found in object')
+        }
+
+        if (opts.lengthDelimited !== false) {
+          writer.ldelim()
+        }
+      }, (reader, length) => {
+        const obj: any = {}
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1:
+              obj.status = FetchResponse.StatusCode.codec().decode(reader)
+              break
+            case 2:
+              obj.data = reader.bytes()
+              break
+            default:
+              reader.skipType(tag & 7)
+              break
+          }
+        }
+
+        if (obj.status == null) {
+          throw new Error('Protocol error: value for required field "status" was not found in protobuf')
+        }
+
+        if (obj.data == null) {
+          throw new Error('Protocol error: value for required field "data" was not found in protobuf')
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
   }
 
   export const encode = (obj: FetchResponse): Uint8Array => {
     return encodeMessage(obj, FetchResponse.codec())
   }
 
-  export const decode = (buf: Uint8Array): FetchResponse => {
+  export const decode = (buf: Uint8Array | Uint8ArrayList): FetchResponse => {
     return decodeMessage(buf, FetchResponse.codec())
   }
 }

package/src/get-peer.ts

@@ -2,9 +2,9 @@ import { peerIdFromString } from '@libp2p/peer-id'
 import { Multiaddr } from '@multiformats/multiaddr'
 import errCode from 'err-code'
 import { codes } from './errors.js'
-import { isPeerId } from '@libp2p/interfaces/peer-id'
-import type { PeerId } from '@libp2p/interfaces/peer-id'
-import type { PeerInfo } from '@libp2p/interfaces/peer-info'
+import { isPeerId } from '@libp2p/interface-peer-id'
+import type { PeerId } from '@libp2p/interface-peer-id'
+import type { PeerInfo } from '@libp2p/interface-peer-info'
 
 function peerIdFromMultiaddr (ma: Multiaddr) {
   const idStr = ma.getPeerId()

package/src/identify/index.ts

@@ -18,21 +18,18 @@ import {
   MULTICODEC_IDENTIFY_PUSH_PROTOCOL_VERSION
 } from './consts.js'
 import { codes } from '../errors.js'
-import type { IncomingStreamData } from '@libp2p/interfaces/registrar'
-import type { Connection, Stream } from '@libp2p/interfaces/connection'
+import type { IncomingStreamData } from '@libp2p/interface-registrar'
+import type { Connection, Stream } from '@libp2p/interface-connection'
 import type { Startable } from '@libp2p/interfaces/startable'
 import { peerIdFromKeys } from '@libp2p/peer-id'
-import type { Components } from '@libp2p/interfaces/components'
+import type { Components } from '@libp2p/components'
 import { TimeoutController } from 'timeout-abort-controller'
 import type { AbortOptions } from '@libp2p/interfaces'
 import { abortableDuplex } from 'abortable-iterator'
-import type { Duplex } from 'it-stream-types'
+import { setMaxListeners } from 'events'
 
 const log = logger('libp2p:identify')
 
-// https://github.com/libp2p/go-libp2p/blob/8d2e54e1637041d5cf4fac1e531287560bd1f4ac/p2p/protocol/identify/id.go#L48
-const IDENTIFY_TIMEOUT = 60000
-
 // https://github.com/libp2p/go-libp2p/blob/8d2e54e1637041d5cf4fac1e531287560bd1f4ac/p2p/protocol/identify/id.go#L52
 const MAX_IDENTIFY_MESSAGE_SIZE = 1024 * 8
 
@@ -54,12 +51,18 @@ export interface IdentifyServiceInit {
   /**
    * How long we should wait for a remote peer to send their identify response
    */
-  timeout?: number
+  timeout: number
 
   /**
    * Identify responses larger than this in bytes will be rejected (default: 8192)
    */
   maxIdentifyMessageSize?: number
+
+  maxInboundStreams: number
+  maxOutboundStreams: number
+
+  maxPushIncomingStreams: number
+  maxPushOutgoingStreams: number
 }
 
 export class IdentifyService implements Startable {
@@ -79,8 +82,6 @@ export class IdentifyService implements Startable {
     this.started = false
     this.init = init
 
-    this.handleMessage = this.handleMessage.bind(this)
-
     this.identifyProtocolStr = `/${init.protocolPrefix}/${MULTICODEC_IDENTIFY_PROTOCOL_NAME}/${MULTICODEC_IDENTIFY_PROTOCOL_VERSION}`
     this.identifyPushProtocolStr = `/${init.protocolPrefix}/${MULTICODEC_IDENTIFY_PUSH_PROTOCOL_NAME}/${MULTICODEC_IDENTIFY_PUSH_PROTOCOL_VERSION}`
 
@@ -127,13 +128,21 @@ export class IdentifyService implements Startable {
     await this.components.getPeerStore().metadataBook.setValue(this.components.getPeerId(), 'AgentVersion', uint8ArrayFromString(this.host.agentVersion))
     await this.components.getPeerStore().metadataBook.setValue(this.components.getPeerId(), 'ProtocolVersion', uint8ArrayFromString(this.host.protocolVersion))
 
-    await this.components.getRegistrar().handle([
-      this.identifyProtocolStr,
-      this.identifyPushProtocolStr
-    ], (data) => {
-      void this.handleMessage(data)?.catch(err => {
+    await this.components.getRegistrar().handle(this.identifyProtocolStr, (data) => {
+      void this._handleIdentify(data).catch(err => {
+        log.error(err)
+      })
+    }, {
+      maxInboundStreams: this.init.maxInboundStreams,
+      maxOutboundStreams: this.init.maxOutboundStreams
+    })
+    await this.components.getRegistrar().handle(this.identifyPushProtocolStr, (data) => {
+      void this._handlePush(data).catch(err => {
         log.error(err)
       })
+    }, {
+      maxInboundStreams: this.init.maxPushIncomingStreams,
+      maxOutboundStreams: this.init.maxPushOutgoingStreams
     })
 
     this.started = true
@@ -155,17 +164,21 @@ export class IdentifyService implements Startable {
     const protocols = await this.components.getPeerStore().protoBook.get(this.components.getPeerId())
 
     const pushes = connections.map(async connection => {
-      const timeoutController = new TimeoutController(this.init.timeout ?? IDENTIFY_TIMEOUT)
       let stream: Stream | undefined
+      const timeoutController = new TimeoutController(this.init.timeout)
 
       try {
-        const data = await connection.newStream([this.identifyPushProtocolStr], {
+        // fails on node < 15.4
+        setMaxListeners?.(Infinity, timeoutController.signal)
+      } catch {}
+
+      try {
+        stream = await connection.newStream([this.identifyPushProtocolStr], {
           signal: timeoutController.signal
         })
-        stream = data.stream
 
         // make stream abortable
-        const source: Duplex<Uint8Array> = abortableDuplex(stream, timeoutController.signal)
+        const source = abortableDuplex(stream, timeoutController.signal)
 
         await pipe(
           [Identify.encode({
@@ -218,21 +231,29 @@ export class IdentifyService implements Startable {
   }
 
   async _identify (connection: Connection, options: AbortOptions = {}): Promise<Identify> {
-    const { stream } = await connection.newStream([this.identifyProtocolStr], options)
-    let source: Duplex<Uint8Array> = stream
     let timeoutController
     let signal = options.signal
+    let stream: Stream | undefined
 
     // create a timeout if no abort signal passed
     if (signal == null) {
-      timeoutController = new TimeoutController(this.init.timeout ?? IDENTIFY_TIMEOUT)
+      timeoutController = new TimeoutController(this.init.timeout)
       signal = timeoutController.signal
-    }
 
-    // make stream abortable if AbortSignal passed
-    source = abortableDuplex(stream, signal)
+      try {
+        // fails on node < 15.4
+        setMaxListeners?.(Infinity, timeoutController.signal)
+      } catch {}
+    }
 
     try {
+      stream = await connection.newStream([this.identifyProtocolStr], {
+        signal
+      })
+
+      // make stream abortable
+      const source = abortableDuplex(stream, signal)
+
       const data = await pipe(
         [],
         source,
@@ -256,7 +277,9 @@ export class IdentifyService implements Startable {
         timeoutController.clear()
       }
 
-      stream.close()
+      if (stream != null) {
+        stream.close()
+      }
     }
   }
 
@@ -353,29 +376,18 @@ export class IdentifyService implements Startable {
     // this.components.getAddressManager().addObservedAddr(observedAddr)
   }
 
-  /**
-   * A handler to register with Libp2p to process identify messages
-   */
-  handleMessage (data: IncomingStreamData) {
-    const { protocol } = data
-
-    switch (protocol) {
-      case this.identifyProtocolStr:
-        return this._handleIdentify(data)
-      case this.identifyPushProtocolStr:
-        return this._handlePush(data)
-      default:
-        log.error('cannot handle unknown protocol %s', protocol)
-    }
-  }
-
   /**
    * Sends the `Identify` response with the Signed Peer Record
    * to the requesting peer over the given `connection`
    */
   async _handleIdentify (data: IncomingStreamData) {
     const { connection, stream } = data
-    const timeoutController = new TimeoutController(this.init.timeout ?? IDENTIFY_TIMEOUT)
+    const timeoutController = new TimeoutController(this.init.timeout)
+
+    try {
+      // fails on node < 15.4
+      setMaxListeners?.(Infinity, timeoutController.signal)
+    } catch {}
 
     try {
       const publicKey = this.components.getPeerId().publicKey ?? new Uint8Array(0)
@@ -391,7 +403,7 @@ export class IdentifyService implements Startable {
 
         const envelope = await RecordEnvelope.seal(peerRecord, this.components.getPeerId())
         await this.components.getPeerStore().addressBook.consumePeerRecord(envelope)
-        signedPeerRecord = envelope.marshal()
+        signedPeerRecord = envelope.marshal().subarray()
       }
 
       const message = Identify.encode({
@@ -405,7 +417,7 @@ export class IdentifyService implements Startable {
       })
 
       // make stream abortable
-      const source: Duplex<Uint8Array> = abortableDuplex(stream, timeoutController.signal)
+      const source = abortableDuplex(stream, timeoutController.signal)
 
       await pipe(
         [message],
@@ -426,12 +438,17 @@ export class IdentifyService implements Startable {
    */
   async _handlePush (data: IncomingStreamData) {
     const { connection, stream } = data
-    const timeoutController = new TimeoutController(this.init.timeout ?? IDENTIFY_TIMEOUT)
+    const timeoutController = new TimeoutController(this.init.timeout)
+
+    try {
+      // fails on node < 15.4
+      setMaxListeners?.(Infinity, timeoutController.signal)
+    } catch {}
 
     let message: Identify | undefined
     try {
       // make stream abortable
-      const source: Duplex<Uint8Array> = abortableDuplex(stream, timeoutController.signal)
+      const source = abortableDuplex(stream, timeoutController.signal)
 
       const data = await pipe(
         [],