@libp2p/circuit-relay-v2 2.1.3-e99e8f448 → 2.1.4-5d199f9b6

package/src/transport/reservation-store.ts

@@ -1,15 +1,14 @@
-import { KEEP_ALIVE, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
+import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
 import { PeerMap } from '@libp2p/peer-collections'
 import { createBloomFilter } from '@libp2p/utils/filters'
 import { PeerQueue } from '@libp2p/utils/peer-queue'
 import { multiaddr } from '@multiformats/multiaddr'
 import { pbStream } from 'it-protobuf-stream'
-import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
-import { DEFAULT_MAX_RESERVATION_QUEUE_LENGTH, DEFAULT_RESERVATION_COMPLETION_TIMEOUT, DEFAULT_RESERVATION_CONCURRENCY, RELAY_TAG, RELAY_V2_HOP_CODEC } from '../constants.js'
+import { DEFAULT_MAX_RESERVATION_QUEUE_LENGTH, DEFAULT_RESERVATION_COMPLETION_TIMEOUT, DEFAULT_RESERVATION_CONCURRENCY, KEEP_ALIVE_TAG, RELAY_TAG, RELAY_V2_HOP_CODEC } from '../constants.js'
 import { HopMessage, Status } from '../pb/index.js'
 import { getExpirationMilliseconds } from '../utils.js'
 import type { Reservation } from '../pb/index.js'
-import type { TypedEventTarget, Libp2pEvents, AbortOptions, ComponentLogger, Logger, Connection, PeerId, PeerStore, Startable, Metrics } from '@libp2p/interface'
+import type { TypedEventTarget, Libp2pEvents, AbortOptions, ComponentLogger, Logger, Connection, PeerId, PeerStore, Startable, Metrics, Peer } from '@libp2p/interface'
 import type { ConnectionManager, TransportManager } from '@libp2p/interface-internal'
 import type { Filter } from '@libp2p/utils/filters'
 
@@ -22,7 +21,7 @@ const REFRESH_TIMEOUT = (60 * 1000) * 5
 // minimum duration before which a reservation must not be refreshed
 const REFRESH_TIMEOUT_MIN = 30 * 1000
 
-export interface RelayStoreComponents {
+export interface ReservationStoreComponents {
   peerId: PeerId
   connectionManager: ConnectionManager
   transportManager: TransportManager
@@ -32,7 +31,7 @@ export interface RelayStoreComponents {
   metrics?: Metrics
 }
 
-export interface RelayStoreInit {
+export interface ReservationStoreInit {
   /**
    * Multiple relays may be discovered simultaneously - to prevent listening
    * on too many relays, this value controls how many to attempt to reserve a
@@ -71,6 +70,11 @@ interface RelayEntry {
   timeout: ReturnType<typeof setTimeout>
   type: RelayType
   reservation: Reservation
+
+  /**
+   * Stores the id of the connection we have to the relay
+   */
+  connection: string
 }
 
 export interface ReservationStoreEvents {
@@ -94,7 +98,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
   private readonly log: Logger
   private readonly relayFilter: Filter
 
-  constructor (components: RelayStoreComponents, init?: RelayStoreInit) {
+  constructor (components: ReservationStoreComponents, init?: ReservationStoreInit) {
     super()
 
     this.log = components.logger.forComponent('libp2p:circuit-relay:transport:reservation-store')
@@ -117,11 +121,21 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
       metrics: components.metrics
     })
 
-    // When a peer disconnects, if we had a reservation on that peer
-    // remove the reservation and multiaddr and maybe trigger search
-    // for new relays
-    this.events.addEventListener('peer:disconnect', (evt) => {
-      this.#removeRelay(evt.detail)
+    // reservations are only valid while we are still connected to the relay.
+    // if we had a reservation opened via that connection, remove it and maybe
+    // trigger a search for new relays
+    this.events.addEventListener('connection:close', (evt) => {
+      const reservation = [...this.reservations.values()]
+        .find(reservation => reservation.connection === evt.detail.id)
+
+      if (reservation == null) {
+        return
+      }
+
+      this.#removeReservation(evt.detail.remotePeer, reservation)
+        .catch(err => {
+          this.log('could not remove relay %p - %e', evt.detail, err)
+        })
     })
   }
 
@@ -134,10 +148,37 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
   }
 
   afterStart (): void {
-    if (this.reservations.size < this.maxDiscoveredRelays) {
-      this.log('not enough relays %d/%d', this.reservations.size, this.maxDiscoveredRelays)
-      this.safeDispatchEvent('relay:not-enough-relays', {})
-    }
+    // remove old relay tags
+    void Promise.resolve()
+      .then(async () => {
+        const relayPeers: Peer[] = await this.peerStore.all({
+          filters: [(peer) => {
+            return peer.tags.has(RELAY_TAG)
+          }]
+        })
+
+        this.log('removing tag from %d old relays', relayPeers.length)
+
+        // remove old relay tag and redial
+        await Promise.all(
+          relayPeers.map(async peer => {
+            await this.peerStore.merge(peer.id, {
+              tags: {
+                [RELAY_TAG]: undefined,
+                [KEEP_ALIVE_TAG]: undefined
+              }
+            })
+          })
+        )
+
+        if (this.reservations.size < this.maxDiscoveredRelays) {
+          this.log('not enough relays %d/%d', this.reservations.size, this.maxDiscoveredRelays)
+          this.safeDispatchEvent('relay:not-enough-relays', {})
+        }
+      })
+      .catch(err => {
+        this.log.error(err)
+      })
   }
 
   stop (): void {
@@ -157,26 +198,26 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
    */
   async addRelay (peerId: PeerId, type: RelayType): Promise<void> {
     if (this.peerId.equals(peerId)) {
-      this.log('not trying to use self as relay')
+      this.log.trace('not trying to use self as relay')
       return
     }
 
     if (this.reserveQueue.size > this.maxReservationQueueLength) {
-      this.log('not adding potential relay peer %p as the queue is full', peerId)
+      this.log.trace('not adding potential relay peer %p as the queue is full', peerId)
       return
     }
 
     if (this.reserveQueue.has(peerId)) {
-      this.log('potential relay peer %p is already in the reservation queue', peerId)
+      this.log.trace('potential relay peer %p is already in the reservation queue', peerId)
       return
     }
 
     if (this.relayFilter.has(peerId.toMultihash().bytes)) {
-      this.log('potential relay peer %p has failed previously, not trying again', peerId)
+      this.log.trace('potential relay peer %p has failed previously, not trying again', peerId)
       return
     }
 
-    this.log('try to reserve relay slot with %p', peerId)
+    this.log.trace('try to reserve relay slot with %p', peerId)
 
     await this.reserveQueue.add(async () => {
       const start = Date.now()
@@ -186,13 +227,24 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
         const existingReservation = this.reservations.get(peerId)
 
         if (existingReservation != null) {
-          if (getExpirationMilliseconds(existingReservation.reservation.expire) > REFRESH_WINDOW) {
-            this.log('already have reservation on relay peer %p and it expires in more than 10 minutes', peerId)
+          const connections = this.connectionManager.getConnections(peerId)
+          let connected = false
+
+          if (connections.length === 0) {
+            this.log('already have relay reservation with %p but we are no longer connected', peerId)
+          }
+
+          if (connections.map(conn => conn.id).includes(existingReservation.connection)) {
+            this.log('already have relay reservation with %p and the original connection is still open', peerId)
+            connected = true
+          }
+
+          if (connected && getExpirationMilliseconds(existingReservation.reservation.expire) > REFRESH_WINDOW) {
+            this.log('already have relay reservation with %p but we are still connected and it does not expire soon', peerId)
             return
           }
 
-          clearTimeout(existingReservation.timeout)
-          this.reservations.delete(peerId)
+          await this.#removeReservation(peerId, existingReservation)
         }
 
         if (type === 'discovered' && [...this.reservations.values()].reduce((acc, curr) => {
@@ -202,7 +254,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
 
           return acc
         }, 0) >= this.maxDiscoveredRelays) {
-          this.log('already have enough discovered relays')
+          this.log.trace('already have enough discovered relays')
           return
         }
 
@@ -222,25 +274,41 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
           signal
         })
 
-        this.log('created reservation on relay peer %p', peerId)
-
         const expiration = getExpirationMilliseconds(reservation.expire)
 
+        this.log('created reservation on relay peer %p, expiry date is %s', peerId, new Date(Date.now() + expiration).toString())
+
         // sets a lower bound on the timeout, and also don't let it go over
         // 2^31 - 1 (setTimeout will only accept signed 32 bit integers)
         const timeoutDuration = Math.min(Math.max(expiration - REFRESH_TIMEOUT, REFRESH_TIMEOUT_MIN), Math.pow(2, 31) - 1)
 
         const timeout = setTimeout(() => {
-          this.addRelay(peerId, type).catch(err => {
-            this.log.error('could not refresh reservation to relay %p', peerId, err)
-          })
+          this.log('refresh reservation to relay %p', peerId)
+
+          this.addRelay(peerId, type)
+            .catch(async err => {
+              this.log.error('could not refresh reservation to relay %p - %e', peerId, err)
+
+              const reservation = this.reservations.get(peerId)
+
+              if (reservation == null) {
+                this.log.error('did not have reservation after refreshing reservation failed %p', peerId)
+                return
+              }
+
+              await this.#removeReservation(peerId, reservation)
+            })
+            .catch(err => {
+              this.log.error('could not remove expired reservation to relay %p - %e', peerId, err)
+            })
         }, timeoutDuration)
 
         // we've managed to create a reservation successfully
         this.reservations.set(peerId, {
           timeout,
           reservation,
-          type
+          type,
+          connection: connection.id
         })
 
         // ensure we don't close the connection to the relay
@@ -250,7 +318,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
               value: 1,
               ttl: expiration
             },
-            [KEEP_ALIVE]: {
+            [KEEP_ALIVE_TAG]: {
               value: 1,
               ttl: expiration
             }
@@ -258,7 +326,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
         })
 
         // listen on multiaddr that only the circuit transport is listening for
-        await this.transportManager.listen([multiaddr(`/p2p/${peerId.toString()}/p2p-circuit`)])
+        await this.transportManager.listen([multiaddr(`/p2p/${peerId.toString()}/p2p-circuit/p2p/${this.peerId.toString()}`)])
 
         this.safeDispatchEvent('relay:created-reservation', {
           detail: peerId
@@ -266,18 +334,19 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
       } catch (err) {
         this.log.error('could not reserve slot on %p after %dms', peerId, Date.now() - start, err)
 
+        // don't try this peer again
+        this.relayFilter.add(peerId.toMultihash().bytes)
+
         // cancel the renewal timeout if it's been set
         const reservation = this.reservations.get(peerId)
 
+        // if listening failed, remove the reservation
         if (reservation != null) {
-          clearTimeout(reservation.timeout)
+          this.#removeReservation(peerId, reservation)
+            .catch(err => {
+              this.log.error('could not remove reservation on %p after reserving slot failed - %e', peerId, err)
+            })
         }
-
-        // if listening failed, remove the reservation
-        this.reservations.delete(peerId)
-
-        // don't try this peer again
-        this.relayFilter.add(peerId.toMultihash().bytes)
       }
     }, {
       peerId
@@ -296,6 +365,14 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
     return this.reservations.size
   }
 
+  async cancelReservations (): Promise<void> {
+    await Promise.all(
+      [...this.reservations.entries()].map(async ([peerId, reservation]) => {
+        await this.#removeReservation(peerId, reservation)
+      })
+    )
+  }
+
   async #createReservation (connection: Connection, options: AbortOptions): Promise<Reservation> {
     options.signal?.throwIfAborted()
 
@@ -318,24 +395,31 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
       }
     }
 
-    if (response.status === Status.OK && (response.reservation != null)) {
+    if (response.status === Status.OK && response.reservation != null) {
       // check that the returned relay has the relay address - this can be
       // omitted when requesting a reservation from a go-libp2p relay we
       // already have a reservation on
-      let hasRelayAddress = false
-      const relayAddressBytes = connection.remoteAddr.bytes
+      const addresses = new Set<string>()
+      addresses.add(connection.remoteAddr.toString())
 
       for (const buf of response.reservation.addrs) {
-        if (uint8ArrayEquals(relayAddressBytes, buf)) {
-          hasRelayAddress = true
-          break
+        let ma = multiaddr(buf)
+
+        if (ma.getPeerId() == null) {
+          ma = ma.encapsulate(`/p2p/${connection.remotePeer}`)
         }
-      }
 
-      if (!hasRelayAddress) {
-        response.reservation.addrs.push(relayAddressBytes)
+        // TODO: workaround for https://github.com/libp2p/go-libp2p/issues/3003
+        ma = multiaddr(ma.toString().replace(
+          `/p2p/${connection.remotePeer}/p2p/${connection.remotePeer}`,
+          `/p2p/${connection.remotePeer}`
+        ))
+
+        addresses.add(ma.toString())
       }
 
+      response.reservation.addrs = [...addresses].map(str => multiaddr(str).bytes)
+
       return response.reservation
     }
 
@@ -348,18 +432,24 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
   /**
    * Remove listen relay
    */
-  #removeRelay (peerId: PeerId): void {
-    const existingReservation = this.reservations.get(peerId)
-
-    if (existingReservation == null) {
+  async #removeReservation (peerId: PeerId, reservation: RelayEntry): Promise<void> {
+    if (!this.reservations.has(peerId)) {
+      this.log('not removing relay reservation with %p from local store as we do not have a reservation with this peer', peerId)
       return
     }
 
-    this.log('connection to relay %p closed, removing reservation from local store', peerId)
-
-    clearTimeout(existingReservation.timeout)
+    this.log('removing relay reservation with %p from local store', peerId)
+    clearTimeout(reservation.timeout)
     this.reservations.delete(peerId)
 
+    // untag the relay
+    await this.peerStore.merge(peerId, {
+      tags: {
+        [RELAY_TAG]: undefined,
+        [KEEP_ALIVE_TAG]: undefined
+      }
+    })
+
     this.safeDispatchEvent('relay:removed', { detail: peerId })
 
     if (this.reservations.size < this.maxDiscoveredRelays) {

package/src/transport/transport.ts

@@ -2,14 +2,14 @@ import { DialError, InvalidMessageError, serviceCapabilities, serviceDependencie
 import { peerFilter } from '@libp2p/peer-collections'
 import { peerIdFromMultihash, peerIdFromString } from '@libp2p/peer-id'
 import { streamToMaConnection } from '@libp2p/utils/stream-to-ma-conn'
-import * as mafmt from '@multiformats/mafmt'
 import { multiaddr } from '@multiformats/multiaddr'
+import { Circuit } from '@multiformats/multiaddr-matcher'
 import { pbStream } from 'it-protobuf-stream'
 import * as Digest from 'multiformats/hashes/digest'
 import { CustomProgressEvent } from 'progress-events'
 import { CIRCUIT_PROTO_CODE, DEFAULT_DISCOVERY_FILTER_ERROR_RATE, DEFAULT_DISCOVERY_FILTER_SIZE, MAX_CONNECTIONS, RELAY_V2_HOP_CODEC, RELAY_V2_STOP_CODEC } from '../constants.js'
 import { StopMessage, HopMessage, Status } from '../pb/index.js'
-import { LimitTracker } from '../utils.js'
+import { CircuitListen, LimitTracker } from '../utils.js'
 import { RelayDiscovery } from './discovery.js'
 import { createListener } from './listener.js'
 import { ReservationStore } from './reservation-store.js'
@@ -184,9 +184,9 @@ export class CircuitRelayTransport implements Transport<CircuitRelayDialEvents>
     const destinationId = destinationAddr.getPeerId()
 
     if (relayId == null || destinationId == null) {
-      const errMsg = `Circuit relay dial to ${ma.toString()} failed as address did not have peer ids`
-      this.log.error(errMsg)
-      throw new DialError(errMsg)
+      const errMsg = `ircuit relay dial to ${ma.toString()} failed as address did not have both relay and destination PeerIDs`
+      this.log.error(`c${errMsg}`)
+      throw new DialError(`C${errMsg}`)
     }
 
     const relayPeer = peerIdFromString(relayId)
@@ -281,7 +281,7 @@ export class CircuitRelayTransport implements Transport<CircuitRelayDialEvents>
         onProgress
       })
     } catch (err: any) {
-      this.log.error(`Circuit relay dial to destination ${destinationPeer.toString()} via relay ${connection.remotePeer.toString()} failed`, err)
+      this.log.error(`circuit relay dial to destination ${destinationPeer.toString()} via relay ${connection.remotePeer.toString()} failed`, err)
       disconnectOnFailure && await connection.close()
       throw err
     }
@@ -305,7 +305,7 @@ export class CircuitRelayTransport implements Transport<CircuitRelayDialEvents>
     multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs]
 
     return multiaddrs.filter((ma) => {
-      return mafmt.Circuit.matches(ma)
+      return CircuitListen.exactMatch(ma)
     })
   }
 
@@ -313,7 +313,11 @@ export class CircuitRelayTransport implements Transport<CircuitRelayDialEvents>
    * Filter check for all Multiaddrs that this transport can dial
    */
   dialFilter (multiaddrs: Multiaddr[]): Multiaddr[] {
-    return this.listenFilter(multiaddrs)
+    multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs]
+
+    return multiaddrs.filter((ma) => {
+      return Circuit.exactMatch(ma)
+    })
   }
 
   /**

package/src/utils.ts

@@ -1,7 +1,10 @@
+import { P2P } from '@multiformats/multiaddr-matcher'
+import { fmt, peerId, literal, optional, and } from '@multiformats/multiaddr-matcher/utils'
 import { anySignal } from 'any-signal'
 import { CID } from 'multiformats/cid'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { DurationLimitError, TransferLimitError } from './errors.js'
+import type { RelayReservation } from './index.js'
 import type { Limit } from './pb/index.js'
 import type { ConnectionLimits, LoggerOptions, Stream } from '@libp2p/interface'
 import type { Source } from 'it-stream-types'
@@ -34,16 +37,18 @@ async function * countStreamBytes (source: Source<Uint8Array | Uint8ArrayList>,
   }
 }
 
-export function createLimitedRelay (src: Stream, dst: Stream, abortSignal: AbortSignal, limit: Limit | undefined, options: LoggerOptions): void {
+export function createLimitedRelay (src: Stream, dst: Stream, abortSignal: AbortSignal, reservation: RelayReservation, options: LoggerOptions): void {
   function abortStreams (err: Error): void {
     src.abort(err)
     dst.abort(err)
   }
 
-  const signals = [abortSignal]
+  // combine shutdown signal and reservation expiry signal
+  const signals = [abortSignal, reservation.signal]
 
-  if (limit?.duration != null) {
-    signals.push(AbortSignal.timeout(limit.duration))
+  if (reservation.limit?.duration != null) {
+    options.log('limiting relayed connection duration to %dms', reservation.limit.duration)
+    signals.push(AbortSignal.timeout(reservation.limit.duration))
   }
 
   const signal = anySignal(signals)
@@ -53,15 +58,16 @@ export function createLimitedRelay (src: Stream, dst: Stream, abortSignal: Abort
 
   let dataLimit: { remaining: bigint } | undefined
 
-  if (limit?.data != null) {
+  if (reservation.limit?.data != null) {
     dataLimit = {
-      remaining: limit.data
+      remaining: reservation.limit.data
     }
   }
 
   queueMicrotask(() => {
     const onAbort = (): void => {
-      dst.abort(new DurationLimitError(`duration limit of ${limit?.duration} ms exceeded`))
+      options.log('relayed connection reached time limit')
+      dst.abort(new DurationLimitError(`duration limit of ${reservation.limit?.duration} ms exceeded`))
     }
 
     signal.addEventListener('abort', onAbort, { once: true })
@@ -83,7 +89,8 @@ export function createLimitedRelay (src: Stream, dst: Stream, abortSignal: Abort
 
   queueMicrotask(() => {
     const onAbort = (): void => {
-      src.abort(new DurationLimitError(`duration limit of ${limit?.duration} ms exceeded`))
+      options.log('relayed connection reached time limit')
+      src.abort(new DurationLimitError(`duration limit of ${reservation.limit?.duration} ms exceeded`))
     }
 
     signal.addEventListener('abort', onAbort, { once: true })
@@ -185,3 +192,9 @@ export class LimitTracker {
     return output
   }
 }
+
+/**
+ * A custom matcher that makes the `/p2p/peer-id` part of circuit relay
+ * addresses optional
+ */
+export const CircuitListen = fmt(and(P2P.matchers[0], literal('p2p-circuit'), optional(peerId())))