libp2p 0.43.4 → 0.44.0-17eb162a

package/src/connection-manager/utils.ts

@@ -0,0 +1,76 @@
+import { AbortOptions, multiaddr } from '@multiformats/multiaddr'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import { logger } from '@libp2p/logger'
+import { ClearableSignal, anySignal } from 'any-signal'
+import { setMaxListeners } from 'events'
+
+const log = logger('libp2p:connection-manager:utils')
+
+/**
+ * Resolve multiaddr recursively
+ */
+export async function resolveMultiaddrs (ma: Multiaddr, options: AbortOptions): Promise<Multiaddr[]> {
+  // TODO: recursive logic should live in multiaddr once dns4/dns6 support is in place
+  // Now only supporting resolve for dnsaddr
+  const resolvableProto = ma.protoNames().includes('dnsaddr')
+
+  // Multiaddr is not resolvable? End recursion!
+  if (!resolvableProto) {
+    return [ma]
+  }
+
+  const resolvedMultiaddrs = await resolveRecord(ma, options)
+  const recursiveMultiaddrs = await Promise.all(resolvedMultiaddrs.map(async (nm) => {
+    return await resolveMultiaddrs(nm, options)
+  }))
+
+  const addrs = recursiveMultiaddrs.flat()
+  const output = addrs.reduce<Multiaddr[]>((array, newM) => {
+    if (array.find(m => m.equals(newM)) == null) {
+      array.push(newM)
+    }
+    return array
+  }, ([]))
+
+  log('resolved %s to', ma, output.map(ma => ma.toString()))
+
+  return output
+}
+
+/**
+ * Resolve a given multiaddr. If this fails, an empty array will be returned
+ */
+async function resolveRecord (ma: Multiaddr, options: AbortOptions): Promise<Multiaddr[]> {
+  try {
+    ma = multiaddr(ma.toString()) // Use current multiaddr module
+    const multiaddrs = await ma.resolve(options)
+    return multiaddrs
+  } catch (err) {
+    log.error(`multiaddr ${ma.toString()} could not be resolved`, err)
+    return []
+  }
+}
+
+export function combineSignals (...signals: Array<AbortSignal | undefined>): ClearableSignal {
+  const sigs: AbortSignal[] = []
+
+  for (const sig of signals) {
+    if (sig != null) {
+      try {
+        // fails on node < 15.4
+        setMaxListeners?.(Infinity, sig)
+      } catch { }
+      sigs.push(sig)
+    }
+  }
+
+  // let any signal abort the dial
+  const signal = anySignal(sigs)
+
+  try {
+    // fails on node < 15.4
+    setMaxListeners?.(Infinity, signal)
+  } catch {}
+
+  return signal
+}

package/src/content-routing/index.ts

@@ -54,7 +54,7 @@ export class CompoundContentRouting implements ContentRouting, Startable {
    */
   async * findProviders (key: CID, options: AbortOptions = {}): AsyncIterable<PeerInfo> {
     if (this.routers.length === 0) {
-      throw new CodeError('No content this.routers available', codes.ERR_NO_ROUTERS_AVAILABLE)
+      throw new CodeError('No content routers available', codes.ERR_NO_ROUTERS_AVAILABLE)
     }
 
     yield * pipe(

package/src/content-routing/utils.ts

@@ -11,7 +11,9 @@ import type { PeerStore } from '@libp2p/interface-peer-store'
 export async function * storeAddresses (source: Source<PeerInfo>, peerStore: PeerStore): AsyncIterable<PeerInfo> {
   yield * map(source, async (peer) => {
     // ensure we have the addresses for a given peer
-    await peerStore.addressBook.add(peer.id, peer.multiaddrs)
+    await peerStore.merge(peer.id, {
+      multiaddrs: peer.multiaddrs
+    })
 
     return peer
   })

package/src/fetch/index.ts

@@ -127,9 +127,9 @@ export class FetchService implements Startable {
 
       const result = await pipe(
         [FetchRequest.encode({ identifier: key })],
-        lp.encode(),
+        (source) => lp.encode(source),
         source,
-        lp.decode(),
+        (source) => lp.decode(source),
         async function (source) {
           const buf = await first(source)
 
@@ -184,7 +184,7 @@ export class FetchService implements Startable {
 
     await pipe(
       stream,
-      lp.decode(),
+      (source) => lp.decode(source),
       async function * (source) {
         const buf = await first(source)
 
@@ -215,7 +215,7 @@ export class FetchService implements Startable {
 
         yield FetchResponse.encode(response)
       },
-      lp.encode(),
+      (source) => lp.encode(source),
       stream
     )
   }

package/src/identify/index.ts

@@ -27,8 +27,12 @@ import { abortableDuplex } from 'abortable-iterator'
 import { setMaxListeners } from 'events'
 import type { ConnectionManager } from '@libp2p/interface-connection-manager'
 import type { PeerId } from '@libp2p/interface-peer-id'
-import type { PeerStore } from '@libp2p/interface-peer-store'
+import type { Peer, PeerStore } from '@libp2p/interface-peer-store'
 import type { AddressManager } from '@libp2p/interface-address-manager'
+import type { EventEmitter } from '@libp2p/interfaces/events'
+import type { Libp2pEvents } from '@libp2p/interface-libp2p'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { pbStream } from 'it-pb-stream'
 
 const log = logger('libp2p:identify')
 
@@ -74,6 +78,7 @@ export interface IdentifyServiceComponents {
   connectionManager: ConnectionManager
   registrar: Registrar
   addressManager: AddressManager
+  events: EventEmitter<Libp2pEvents>
 }
 
 export class IdentifyService implements Startable {
@@ -103,27 +108,14 @@ export class IdentifyService implements Startable {
     }
 
     // When a new connection happens, trigger identify
-    this.components.connectionManager.addEventListener('peer:connect', (evt) => {
+    this.components.events.addEventListener('connection:open', (evt) => {
       const connection = evt.detail
-      this.identify(connection).catch(log.error)
+      this.identify(connection).catch(err => { log.error('error during identify trigged by connection:open', err) })
     })
 
-    // When self multiaddrs change, trigger identify-push
-    this.components.peerStore.addEventListener('change:multiaddrs', (evt) => {
-      const { peerId } = evt.detail
-
-      if (this.components.peerId.equals(peerId)) {
-        void this.pushToPeerStore().catch(err => { log.error(err) })
-      }
-    })
-
-    // When self protocols change, trigger identify-push
-    this.components.peerStore.addEventListener('change:protocols', (evt) => {
-      const { peerId } = evt.detail
-
-      if (this.components.peerId.equals(peerId)) {
-        void this.pushToPeerStore().catch(err => { log.error(err) })
-      }
+    // When self peer record changes, trigger identify-push
+    this.components.events.addEventListener('self:peer:update', (evt) => {
+      void this.pushToPeerStore().catch(err => { log.error(err) })
     })
   }
 
@@ -136,8 +128,12 @@ export class IdentifyService implements Startable {
       return
     }
 
-    await this.components.peerStore.metadataBook.setValue(this.components.peerId, 'AgentVersion', uint8ArrayFromString(this.host.agentVersion))
-    await this.components.peerStore.metadataBook.setValue(this.components.peerId, 'ProtocolVersion', uint8ArrayFromString(this.host.protocolVersion))
+    await this.components.peerStore.merge(this.components.peerId, {
+      metadata: {
+        AgentVersion: uint8ArrayFromString(this.host.agentVersion),
+        ProtocolVersion: uint8ArrayFromString(this.host.protocolVersion)
+      }
+    })
 
     await this.components.registrar.handle(this.identifyProtocolStr, (data) => {
       void this._handleIdentify(data).catch(err => {
@@ -170,9 +166,16 @@ export class IdentifyService implements Startable {
    * Send an Identify Push update to the list of connections
    */
   async push (connections: Connection[]): Promise<void> {
-    const signedPeerRecord = await this.components.peerStore.addressBook.getRawEnvelope(this.components.peerId)
-    const listenAddrs = this.components.addressManager.getAddresses().map((ma) => ma.bytes)
-    const protocols = await this.components.peerStore.protoBook.get(this.components.peerId)
+    const listenAddresses = this.components.addressManager.getAddresses().map(ma => ma.decapsulateCode(protocols('p2p').code))
+    const peerRecord = new PeerRecord({
+      peerId: this.components.peerId,
+      multiaddrs: listenAddresses
+    })
+    const signedPeerRecord = await RecordEnvelope.seal(peerRecord, this.components.peerId)
+    const supportedProtocols = this.components.registrar.getProtocols()
+    const peer = await this.components.peerStore.get(this.components.peerId)
+    const agentVersion = uint8ArrayToString(peer.metadata.get('AgentVersion') ?? uint8ArrayFromString(this.host.agentVersion))
+    const protocolVersion = uint8ArrayToString(peer.metadata.get('ProtocolVersion') ?? uint8ArrayFromString(this.host.protocolVersion))
 
     const pushes = connections.map(async connection => {
       let stream: Stream | undefined
@@ -193,11 +196,13 @@ export class IdentifyService implements Startable {
 
         await source.sink(pipe(
           [Identify.encode({
-            listenAddrs,
-            signedPeerRecord,
-            protocols
+            listenAddrs: listenAddresses.map(ma => ma.bytes),
+            signedPeerRecord: signedPeerRecord.marshal(),
+            protocols: supportedProtocols,
+            agentVersion,
+            protocolVersion
           })],
-          lp.encode()
+          (source) => lp.encode(source)
         ))
       } catch (err: any) {
         // Just log errors
@@ -266,7 +271,7 @@ export class IdentifyService implements Startable {
       const data = await pipe(
         [],
         source,
-        lp.decode({
+        (source) => lp.decode(source, {
           maxDataLength: this.init.maxIdentifyMessageSize ?? MAX_IDENTIFY_MESSAGE_SIZE
         }),
         async (source) => await first(source)
@@ -299,15 +304,10 @@ export class IdentifyService implements Startable {
    */
   async identify (connection: Connection, options: AbortOptions = {}): Promise<void> {
     const message = await this._identify(connection, options)
-
     const {
       publicKey,
-      listenAddrs,
       protocols,
-      observedAddr,
-      signedPeerRecord,
-      agentVersion,
-      protocolVersion
+      observedAddr
     } = message
 
     if (publicKey == null) {
@@ -327,57 +327,6 @@ export class IdentifyService implements Startable {
     // Get the observedAddr if there is one
     const cleanObservedAddr = IdentifyService.getCleanMultiaddr(observedAddr)
 
-    if (signedPeerRecord != null) {
-      log('received signed peer record from %p', id)
-
-      try {
-        const envelope = await RecordEnvelope.openAndCertify(signedPeerRecord, PeerRecord.DOMAIN)
-
-        if (!envelope.peerId.equals(id)) {
-          throw new CodeError('identified peer does not match the expected peer', codes.ERR_INVALID_PEER)
-        }
-
-        if (await this.components.peerStore.addressBook.consumePeerRecord(envelope)) {
-          await this.components.peerStore.protoBook.set(id, protocols)
-
-          if (agentVersion != null) {
-            await this.components.peerStore.metadataBook.setValue(id, 'AgentVersion', uint8ArrayFromString(agentVersion))
-          }
-
-          if (protocolVersion != null) {
-            await this.components.peerStore.metadataBook.setValue(id, 'ProtocolVersion', uint8ArrayFromString(protocolVersion))
-          }
-
-          log('identify completed for peer %p with and protocols %o', id, protocols)
-
-          return
-        }
-      } catch (err: any) {
-        log('received invalid envelope, discard it and fallback to listenAddrs is available', err)
-      }
-    } else {
-      log('no signed peer record received from %p', id)
-    }
-
-    log('falling back to legacy addresses from %p', id)
-
-    // LEGACY: Update peers data in PeerStore
-    try {
-      await this.components.peerStore.addressBook.set(id, listenAddrs.map((addr) => multiaddr(addr)))
-    } catch (err: any) {
-      log.error('received invalid addrs', err)
-    }
-
-    await this.components.peerStore.protoBook.set(id, protocols)
-
-    if (agentVersion != null) {
-      await this.components.peerStore.metadataBook.setValue(id, 'AgentVersion', uint8ArrayFromString(agentVersion))
-    }
-
-    if (protocolVersion != null) {
-      await this.components.peerStore.metadataBook.setValue(id, 'ProtocolVersion', uint8ArrayFromString(protocolVersion))
-    }
-
     log('identify completed for peer %p and protocols %o', id, protocols)
     log('our observed address is %s', cleanObservedAddr)
 
@@ -386,6 +335,8 @@ export class IdentifyService implements Startable {
       log('storing our observed address %s', cleanObservedAddr?.toString())
       this.components.addressManager.addObservedAddr(cleanObservedAddr)
     }
+
+    await this.#consumeIdentifyMessage(connection.remotePeer, message)
   }
 
   /**
@@ -414,7 +365,6 @@ export class IdentifyService implements Startable {
         })
 
         const envelope = await RecordEnvelope.seal(peerRecord, this.components.peerId)
-        await this.components.peerStore.addressBook.consumePeerRecord(envelope)
         signedPeerRecord = envelope.marshal().subarray()
       }
 
@@ -431,7 +381,7 @@ export class IdentifyService implements Startable {
       // make stream abortable
       const source = abortableDuplex(stream, timeoutController.signal)
 
-      const msgWithLenPrefix = pipe([message], lp.encode())
+      const msgWithLenPrefix = pipe([message], (source) => lp.encode(source))
       await source.sink(msgWithLenPrefix)
     } catch (err: any) {
       log.error('could not respond to identify request', err)
@@ -446,88 +396,98 @@ export class IdentifyService implements Startable {
    */
   async _handlePush (data: IncomingStreamData): Promise<void> {
     const { connection, stream } = data
-    const timeoutController = new TimeoutController(this.init.timeout)
 
     try {
-      // fails on node < 15.4
-      setMaxListeners?.(Infinity, timeoutController.signal)
-    } catch {}
+      if (this.components.peerId.equals(connection.remotePeer)) {
+        throw new Error('received push from ourselves?')
+      }
 
-    let message: Identify | undefined
-    try {
       // make stream abortable
-      const source = abortableDuplex(stream, timeoutController.signal)
-
-      const data = await pipe(
-        [],
-        source,
-        lp.decode({
-          maxDataLength: this.init.maxIdentifyMessageSize ?? MAX_IDENTIFY_MESSAGE_SIZE
-        }),
-        async (source) => await first(source)
-      )
+      const source = abortableDuplex(stream, AbortSignal.timeout(this.init.timeout))
+      const pb = pbStream(source, {
+        maxDataLength: this.init.maxIdentifyMessageSize ?? MAX_IDENTIFY_MESSAGE_SIZE
+      })
+      const message = await pb.readPB(Identify)
 
-      if (data != null) {
-        message = Identify.decode(data)
-      }
+      await this.#consumeIdentifyMessage(connection.remotePeer, message)
     } catch (err: any) {
       log.error('received invalid message', err)
       return
     } finally {
       stream.close()
-      timeoutController.clear()
     }
 
+    log('handled push from %p', connection.remotePeer)
+  }
+
+  async #consumeIdentifyMessage (remotePeer: PeerId, message: Identify): Promise<void> {
     if (message == null) {
-      log.error('received invalid message')
-      return
+      throw new Error('Message was null or undefined')
     }
 
-    const id = connection.remotePeer
+    log('received identify from %p', remotePeer)
 
-    if (this.components.peerId.equals(id)) {
-      log('received push from ourselves?')
+    if (message.signedPeerRecord == null) {
       return
     }
 
-    log('received push from %p', id)
+    const envelope = await RecordEnvelope.openAndCertify(message.signedPeerRecord, PeerRecord.DOMAIN)
+    const peerRecord = PeerRecord.createFromProtobuf(envelope.payload)
 
-    if (message.signedPeerRecord != null) {
-      log('received signedPeerRecord in push')
+    // Verify peerId
+    if (!peerRecord.peerId.equals(envelope.peerId)) {
+      throw new Error('signing key does not match PeerId in the PeerRecord')
+    }
 
-      try {
-        const envelope = await RecordEnvelope.openAndCertify(message.signedPeerRecord, PeerRecord.DOMAIN)
+    // Make sure remote peer is the one sending the record
+    if (!remotePeer.equals(peerRecord.peerId)) {
+      throw new Error('signing key does not match remote PeerId')
+    }
 
-        if (await this.components.peerStore.addressBook.consumePeerRecord(envelope)) {
-          log('consumed signedPeerRecord sent in push')
+    let peer: Peer | undefined
 
-          await this.components.peerStore.protoBook.set(id, message.protocols)
-          return
-        } else {
-          log('failed to consume signedPeerRecord sent in push')
-        }
-      } catch (err: any) {
-        log('received invalid envelope, discard it and fallback to listenAddrs is available', err)
+    try {
+      peer = await this.components.peerStore.get(peerRecord.peerId)
+    } catch (err: any) {
+      if (err.code !== 'ERR_NOT_FOUND') {
+        throw err
       }
-    } else {
-      log('did not receive signedPeerRecord in push')
     }
 
-    // LEGACY: Update peers data in PeerStore
-    try {
-      await this.components.peerStore.addressBook.set(id, message.listenAddrs.map((addr) => multiaddr(addr)))
-    } catch (err: any) {
-      log.error('received invalid addrs', err)
+    log('received signedPeerRecord in push from %p', remotePeer)
+    let metadata = new Map()
+
+    if (peer?.peerRecordEnvelope != null) {
+      const storedEnvelope = await RecordEnvelope.createFromProtobuf(peer.peerRecordEnvelope)
+      const storedRecord = PeerRecord.createFromProtobuf(storedEnvelope.payload)
+
+      // ensure seq is greater than, or equal to, the last received
+      if (storedRecord.seqNumber >= peerRecord.seqNumber) {
+        log('sequence number was lower or equal to existing sequence number - stored: %d received: %d', storedRecord.seqNumber, peerRecord.seqNumber)
+      }
+
+      metadata = peer.metadata
     }
 
-    // Update the protocols
-    try {
-      await this.components.peerStore.protoBook.set(id, message.protocols)
-    } catch (err: any) {
-      log.error('received invalid protocols', err)
+    if (message.agentVersion != null) {
+      metadata.set('AgentVersion', uint8ArrayFromString(message.agentVersion))
     }
 
-    log('handled push from %p', id)
+    if (message.protocolVersion != null) {
+      metadata.set('ProtocolVersion', uint8ArrayFromString(message.protocolVersion))
+    }
+
+    await this.components.peerStore.patch(peerRecord.peerId, {
+      peerRecordEnvelope: message.signedPeerRecord,
+      protocols: message.protocols,
+      addresses: peerRecord.multiaddrs.map(multiaddr => ({
+        isCertified: true,
+        multiaddr
+      })),
+      metadata
+    })
+
+    log('consumed signedPeerRecord sent in push from %p', remotePeer)
   }
 
   /**

package/src/index.ts

@@ -20,7 +20,6 @@ import type { TransportManagerInit } from './transport-manager.js'
 import type { IdentifyServiceInit } from './identify/index.js'
 import type { DualDHT } from '@libp2p/interface-dht'
 import type { Datastore } from 'interface-datastore'
-import type { PeerStoreInit } from '@libp2p/interface-peer-store'
 import type { PeerId } from '@libp2p/interface-peer-id'
 import type { PeerDiscovery } from '@libp2p/interface-peer-discovery'
 import type { ConnectionProtector } from '@libp2p/interface-connection'
@@ -32,7 +31,6 @@ import type { PeerRouting } from '@libp2p/interface-peer-routing'
 import type { ContentRouting } from '@libp2p/interface-content-routing'
 import type { PubSub } from '@libp2p/interface-pubsub'
 import type { Metrics } from '@libp2p/interface-metrics'
-import type { PeerInfo } from '@libp2p/interface-peer-info'
 import type { PingServiceInit } from './ping/index.js'
 import type { FetchServiceInit } from './fetch/index.js'
 import type { AutonatServiceInit } from './autonat/index.js'
@@ -44,6 +42,7 @@ import type { AddressManagerInit } from './address-manager/index.js'
 import type { PeerRoutingInit } from './peer-routing.js'
 import type { ConnectionManagerInit } from './connection-manager/index.js'
 import type { CircuitRelayService } from './circuit-relay/index.js'
+import type { PersistentPeerStoreInit } from '@libp2p/peer-store'
 
 /**
  * For Libp2p configurations and modules details read the [Configuration Document](./CONFIGURATION.md).
@@ -84,7 +83,7 @@ export interface Libp2pInit {
   /**
    * libp2p PeerStore configuration
    */
-  peerStore: PeerStoreInit
+  peerStore: PersistentPeerStoreInit
 
   /**
    * libp2p Peer routing service configuration
@@ -158,23 +157,6 @@ export interface Libp2pInit {
   connectionProtector?: (components: Components) => ConnectionProtector
 }
 
-/**
- * Once you have a libp2p instance, you can listen to several events it emits, so that you can be notified of relevant network events.
- */
-export interface Libp2pEvents {
-  /**
-   * @example
-   *
-   * ```js
-   * libp2p.addEventListener('peer:discovery', (event) => {
-   *    const peerInfo = event.detail
-   *    // ...
-   * })
-   * ```
-   */
-  'peer:discovery': CustomEvent<PeerInfo>
-}
-
 export type { Libp2p }
 
 export type Libp2pOptions = RecursivePartial<Libp2pInit> & { start?: boolean }

package/src/insecure/index.ts

@@ -30,7 +30,7 @@ import { Exchange, KeyType } from './pb/proto.js'
 import type { PeerId } from '@libp2p/interface-peer-id'
 import { peerIdFromBytes, peerIdFromKeys } from '@libp2p/peer-id'
 import type { ConnectionEncrypter, SecuredConnection } from '@libp2p/interface-connection-encrypter'
-import type { Duplex } from 'it-stream-types'
+import type { Duplex, Source } from 'it-stream-types'
 import map from 'it-map'
 import type { Uint8ArrayList } from 'uint8arraylist'
 
@@ -46,7 +46,7 @@ function lpEncodeExchange (exchange: Exchange): Uint8ArrayList {
 /**
  * Encrypt connection
  */
-async function encrypt (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: PeerId): Promise<SecuredConnection> {
+async function encrypt (localId: PeerId, conn: Duplex<AsyncGenerator<Uint8Array>, Source<Uint8Array>, Promise<void>>, remoteId?: PeerId): Promise<SecuredConnection> {
   const shake = handshake(conn)
 
   let type = KeyType.RSA
@@ -71,8 +71,12 @@ async function encrypt (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: Pe
   log('write pubkey exchange to peer %p', remoteId)
 
   // Get the Exchange message
-  // @ts-expect-error needs to be generator
   const response = (await lp.decode.fromReader(shake.reader).next()).value
+
+  if (response == null) {
+    throw new Error('Did not read response')
+  }
+
   const id = Exchange.decode(response)
   log('read pubkey exchange from peer %p', remoteId)
 
@@ -120,11 +124,11 @@ async function encrypt (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: Pe
 class Plaintext implements ConnectionEncrypter {
   public protocol: string = PROTOCOL
 
-  async secureInbound (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: PeerId): Promise<SecuredConnection> {
+  async secureInbound (localId: PeerId, conn: Duplex<AsyncGenerator<Uint8Array>, Source<Uint8Array>, Promise<void>>, remoteId?: PeerId): Promise<SecuredConnection> {
     return await encrypt(localId, conn, remoteId)
   }
 
-  async secureOutbound (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: PeerId): Promise<SecuredConnection> {
+  async secureOutbound (localId: PeerId, conn: Duplex<AsyncGenerator<Uint8Array>, Source<Uint8Array>, Promise<void>>, remoteId?: PeerId): Promise<SecuredConnection> {
     return await encrypt(localId, conn, remoteId)
   }
 }