@libp2p/circuit-relay-v2 2.1.4 → 2.1.5-3bc9769b8

package/src/transport/reservation-store.ts

@@ -1,15 +1,18 @@
-import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
+import { ListenError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
 import { PeerMap } from '@libp2p/peer-collections'
-import { createBloomFilter } from '@libp2p/utils/filters'
+import { createScalableCuckooFilter } from '@libp2p/utils/filters'
 import { PeerQueue } from '@libp2p/utils/peer-queue'
 import { multiaddr } from '@multiformats/multiaddr'
+import { Circuit } from '@multiformats/multiaddr-matcher'
 import { pbStream } from 'it-protobuf-stream'
-import { DEFAULT_MAX_RESERVATION_QUEUE_LENGTH, DEFAULT_RESERVATION_COMPLETION_TIMEOUT, DEFAULT_RESERVATION_CONCURRENCY, KEEP_ALIVE_TAG, RELAY_TAG, RELAY_V2_HOP_CODEC } from '../constants.js'
+import { nanoid } from 'nanoid'
+import { DEFAULT_MAX_RESERVATION_QUEUE_LENGTH, DEFAULT_RESERVATION_COMPLETION_TIMEOUT, DEFAULT_RESERVATION_CONCURRENCY, KEEP_ALIVE_TAG, RELAY_V2_HOP_CODEC } from '../constants.js'
+import { DoubleRelayError, HadEnoughRelaysError, RelayQueueFullError } from '../errors.js'
 import { HopMessage, Status } from '../pb/index.js'
 import { getExpirationMilliseconds } from '../utils.js'
 import type { Reservation } from '../pb/index.js'
-import type { TypedEventTarget, Libp2pEvents, AbortOptions, ComponentLogger, Logger, Connection, PeerId, PeerStore, Startable, Metrics, Peer } from '@libp2p/interface'
-import type { ConnectionManager, TransportManager } from '@libp2p/interface-internal'
+import type { AbortOptions, TypedEventTarget, Libp2pEvents, ComponentLogger, Logger, PeerId, PeerStore, Startable, Metrics, Peer, Connection } from '@libp2p/interface'
+import type { ConnectionManager } from '@libp2p/interface-internal'
 import type { Filter } from '@libp2p/utils/filters'
 
 // allow refreshing a relay reservation if it will expire in the next 10 minutes
@@ -24,7 +27,6 @@ const REFRESH_TIMEOUT_MIN = 30 * 1000
 export interface ReservationStoreComponents {
   peerId: PeerId
   connectionManager: ConnectionManager
-  transportManager: TransportManager
   peerStore: PeerStore
   events: TypedEventTarget<Libp2pEvents>
   logger: ComponentLogger
@@ -44,11 +46,6 @@ export interface ReservationStoreInit {
    */
   reservationConcurrency?: number
 
-  /**
-   * How many discovered relays to allow in the reservation store
-   */
-  discoverRelays?: number
-
   /**
    * Limit the number of potential relays we will dial
    *
@@ -66,37 +63,60 @@ export interface ReservationStoreInit {
 
 export type RelayType = 'discovered' | 'configured'
 
-interface RelayEntry {
+export interface DiscoveredRelayEntry {
   timeout: ReturnType<typeof setTimeout>
-  type: RelayType
+  type: 'discovered'
   reservation: Reservation
 
   /**
    * Stores the id of the connection we have to the relay
    */
   connection: string
+
+  /**
+   * Stores the identifier returned when the reservation was requested
+   */
+  id: string
+}
+
+export interface ConfiguredRelayEntry {
+  timeout: ReturnType<typeof setTimeout>
+  type: 'configured'
+  reservation: Reservation
+
+  /**
+   * Stores the id of the connection we have to the relay
+   */
+  connection: string
+}
+
+export type RelayEntry = DiscoveredRelayEntry | ConfiguredRelayEntry
+
+export interface RelayReservation {
+  relay: PeerId
+  details: RelayEntry
 }
 
 export interface ReservationStoreEvents {
   'relay:not-enough-relays': CustomEvent
-  'relay:removed': CustomEvent<PeerId>
-  'relay:created-reservation': CustomEvent<PeerId>
+  'relay:found-enough-relays': CustomEvent
+  'relay:removed': CustomEvent<RelayReservation>
+  'relay:created-reservation': CustomEvent<RelayReservation>
 }
 
 export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents> implements Startable {
   private readonly peerId: PeerId
   private readonly connectionManager: ConnectionManager
-  private readonly transportManager: TransportManager
   private readonly peerStore: PeerStore
   private readonly events: TypedEventTarget<Libp2pEvents>
-  private readonly reserveQueue: PeerQueue
+  private readonly reserveQueue: PeerQueue<RelayReservation>
   private readonly reservations: PeerMap<RelayEntry>
-  private readonly maxDiscoveredRelays: number
+  private readonly pendingReservations: string[]
   private readonly maxReservationQueueLength: number
   private readonly reservationCompletionTimeout: number
   private started: boolean
   private readonly log: Logger
-  private readonly relayFilter: Filter
+  private relayFilter: Filter
 
   constructor (components: ReservationStoreComponents, init?: ReservationStoreInit) {
     super()
@@ -104,15 +124,14 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
     this.log = components.logger.forComponent('libp2p:circuit-relay:transport:reservation-store')
     this.peerId = components.peerId
     this.connectionManager = components.connectionManager
-    this.transportManager = components.transportManager
     this.peerStore = components.peerStore
     this.events = components.events
     this.reservations = new PeerMap()
-    this.maxDiscoveredRelays = init?.discoverRelays ?? 0
+    this.pendingReservations = []
     this.maxReservationQueueLength = init?.maxReservationQueueLength ?? DEFAULT_MAX_RESERVATION_QUEUE_LENGTH
     this.reservationCompletionTimeout = init?.reservationCompletionTimeout ?? DEFAULT_RESERVATION_COMPLETION_TIMEOUT
     this.started = false
-    this.relayFilter = createBloomFilter(100)
+    this.relayFilter = createScalableCuckooFilter(100)
 
     // ensure we don't listen on multiple relays simultaneously
     this.reserveQueue = new PeerQueue({
@@ -132,7 +151,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
         return
       }
 
-      this.#removeReservation(evt.detail.remotePeer, reservation)
+      this.#removeReservation(evt.detail.remotePeer)
         .catch(err => {
           this.log('could not remove relay %p - %e', evt.detail, err)
         })
@@ -153,7 +172,7 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
       .then(async () => {
         const relayPeers: Peer[] = await this.peerStore.all({
           filters: [(peer) => {
-            return peer.tags.has(RELAY_TAG)
+            return peer.tags.has(KEEP_ALIVE_TAG)
           }]
         })
 
@@ -164,17 +183,18 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
           relayPeers.map(async peer => {
             await this.peerStore.merge(peer.id, {
               tags: {
-                [RELAY_TAG]: undefined,
                 [KEEP_ALIVE_TAG]: undefined
               }
             })
           })
         )
 
-        if (this.reservations.size < this.maxDiscoveredRelays) {
-          this.log('not enough relays %d/%d', this.reservations.size, this.maxDiscoveredRelays)
-          this.safeDispatchEvent('relay:not-enough-relays', {})
-        }
+        this.log('redialing %d old relays', relayPeers.length)
+        await Promise.all(
+          relayPeers.map(async peer => this.addRelay(peer.id, 'discovered'))
+        )
+
+        this.#checkReservationCount()
       })
       .catch(err => {
         this.log.error(err)
@@ -190,36 +210,46 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
     this.started = false
   }
 
+  reserveRelay (): string {
+    const id = nanoid()
+
+    this.pendingReservations.push(id)
+
+    this.#checkReservationCount()
+
+    return id
+  }
+
   /**
    * If the number of current relays is beneath the configured `maxReservations`
    * value, and the passed peer id is not our own, and we have a non-relayed
    * connection to the remote, and the remote peer speaks the hop protocol, try
    * to reserve a slot on the remote peer
    */
-  async addRelay (peerId: PeerId, type: RelayType): Promise<void> {
+  async addRelay (peerId: PeerId, type: RelayType): Promise<RelayReservation> {
     if (this.peerId.equals(peerId)) {
       this.log.trace('not trying to use self as relay')
-      return
+      throw new ListenError('Cannot use self as relay')
     }
 
     if (this.reserveQueue.size > this.maxReservationQueueLength) {
-      this.log.trace('not adding potential relay peer %p as the queue is full', peerId)
-      return
+      throw new RelayQueueFullError('The reservation queue is full')
     }
 
-    if (this.reserveQueue.has(peerId)) {
+    const existingJob = this.reserveQueue.find(peerId)
+
+    if (existingJob != null) {
       this.log.trace('potential relay peer %p is already in the reservation queue', peerId)
-      return
+      return existingJob.join()
     }
 
     if (this.relayFilter.has(peerId.toMultihash().bytes)) {
-      this.log.trace('potential relay peer %p has failed previously, not trying again', peerId)
-      return
+      throw new ListenError('The relay was previously invalid')
     }
 
     this.log.trace('try to reserve relay slot with %p', peerId)
 
-    await this.reserveQueue.add(async () => {
+    return this.reserveQueue.add(async () => {
       const start = Date.now()
 
       try {
@@ -227,24 +257,31 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
         const existingReservation = this.reservations.get(peerId)
 
         if (existingReservation != null) {
-          if (this.connectionManager.getConnections(peerId).map(conn => conn.id).includes(existingReservation.connection) && getExpirationMilliseconds(existingReservation.reservation.expire) > REFRESH_WINDOW) {
-            this.log('already have relay reservation with %p but we are still connected and it does not expire soon', peerId)
-            return
+          const connections = this.connectionManager.getConnections(peerId)
+          let connected = false
+
+          if (connections.length === 0) {
+            this.log('already have relay reservation with %p but we are no longer connected', peerId)
           }
 
-          this.log('already have relay reservation with %p but the original connection is no longer open', peerId)
-          await this.#removeReservation(peerId, existingReservation)
-        }
+          if (connections.map(conn => conn.id).includes(existingReservation.connection)) {
+            this.log('already have relay reservation with %p and the original connection is still open', peerId)
+            connected = true
+          }
 
-        if (type === 'discovered' && [...this.reservations.values()].reduce((acc, curr) => {
-          if (curr.type === 'discovered') {
-            acc++
+          if (connected && getExpirationMilliseconds(existingReservation.reservation.expire) > REFRESH_WINDOW) {
+            this.log('already have relay reservation with %p but we are still connected and it does not expire soon', peerId)
+            return {
+              relay: peerId,
+              details: existingReservation
+            } satisfies RelayReservation
           }
 
-          return acc
-        }, 0) >= this.maxDiscoveredRelays) {
-          this.log.trace('already have enough discovered relays')
-          return
+          await this.#removeReservation(peerId)
+        }
+
+        if (type === 'discovered' && this.pendingReservations.length === 0) {
+          throw new HadEnoughRelaysError('Not making reservation on discovered relay because we do not need any more relays')
         }
 
         const signal = AbortSignal.timeout(this.reservationCompletionTimeout)
@@ -254,44 +291,67 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
           signal
         })
 
-        if (connection.remoteAddr.protoNames().includes('p2p-circuit')) {
-          this.log('not creating reservation over relayed connection')
-          return
+        if (Circuit.matches(connection.remoteAddr)) {
+          throw new DoubleRelayError('not creating reservation over relayed connection')
         }
 
         const reservation = await this.#createReservation(connection, {
           signal
         })
 
-        this.log('created reservation on relay peer %p', peerId)
-
         const expiration = getExpirationMilliseconds(reservation.expire)
 
+        this.log('created reservation on relay peer %p, expiry date is %s', peerId, new Date(Date.now() + expiration).toString())
+
         // sets a lower bound on the timeout, and also don't let it go over
         // 2^31 - 1 (setTimeout will only accept signed 32 bit integers)
         const timeoutDuration = Math.min(Math.max(expiration - REFRESH_TIMEOUT, REFRESH_TIMEOUT_MIN), Math.pow(2, 31) - 1)
 
         const timeout = setTimeout(() => {
-          this.addRelay(peerId, type).catch(err => {
-            this.log.error('could not refresh reservation to relay %p', peerId, err)
-          })
+          this.log('refresh reservation to relay %p', peerId)
+
+          this.addRelay(peerId, type)
+            .catch(async err => {
+              this.log.error('could not refresh reservation to relay %p - %e', peerId, err)
+              await this.#removeReservation(peerId)
+            })
+            .catch(err => {
+              this.log.error('could not remove expired reservation to relay %p - %e', peerId, err)
+            })
         }, timeoutDuration)
 
+        let res: RelayEntry
+
+        // assign a reservation id if one was requested
+        if (type === 'discovered') {
+          const id = this.pendingReservations.pop()
+
+          if (id == null) {
+            throw new HadEnoughRelaysError('Made reservation on relay but did not need any more discovered relays')
+          }
+
+          res = {
+            timeout,
+            reservation,
+            type,
+            connection: connection.id,
+            id
+          }
+        } else {
+          res = {
+            timeout,
+            reservation,
+            type,
+            connection: connection.id
+          }
+        }
+
         // we've managed to create a reservation successfully
-        this.reservations.set(peerId, {
-          timeout,
-          reservation,
-          type,
-          connection: connection.id
-        })
+        this.reservations.set(peerId, res)
 
         // ensure we don't close the connection to the relay
         await this.peerStore.merge(peerId, {
           tags: {
-            [RELAY_TAG]: {
-              value: 1,
-              ttl: expiration
-            },
             [KEEP_ALIVE_TAG]: {
               value: 1,
               ttl: expiration
@@ -299,27 +359,37 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
           }
         })
 
-        // listen on multiaddr that only the circuit transport is listening for
-        await this.transportManager.listen([multiaddr(`/p2p/${peerId.toString()}/p2p-circuit`)])
+        // check to see if we have discovered enough relays
+        this.#checkReservationCount()
+
+        const result: RelayReservation = {
+          relay: peerId,
+          details: res
+        }
 
         this.safeDispatchEvent('relay:created-reservation', {
-          detail: peerId
+          detail: result
         })
-      } catch (err) {
-        this.log.error('could not reserve slot on %p after %dms', peerId, Date.now() - start, err)
 
-        // cancel the renewal timeout if it's been set
-        const reservation = this.reservations.get(peerId)
+        return result
+      } catch (err: any) {
+        if (!(type === 'discovered' && err.name === 'HadEnoughRelaysError')) {
+          this.log.error('could not reserve slot on %p after %dms - %e', peerId, Date.now() - start, err)
+        }
 
-        if (reservation != null) {
-          clearTimeout(reservation.timeout)
+        // don't try this peer again if dialing failed or they do not support
+        // the hop protocol
+        if (err.name === 'DialError' || err.name === 'UnsupportedProtocolError') {
+          this.relayFilter.add(peerId.toMultihash().bytes)
         }
 
         // if listening failed, remove the reservation
-        this.reservations.delete(peerId)
+        this.#removeReservation(peerId)
+          .catch(err => {
+            this.log.error('could not remove reservation on %p after reserving slot failed - %e', peerId, err)
+          })
 
-        // don't try this peer again
-        this.relayFilter.add(peerId.toMultihash().bytes)
+        throw err
       }
     }, {
       peerId
@@ -334,16 +404,26 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
     return this.reservations.get(peerId)?.reservation
   }
 
-  reservationCount (): number {
-    return this.reservations.size
+  reservationCount (type?: RelayType): number {
+    if (type == null) {
+      return this.reservations.size
+    }
+
+    return [...this.reservations.values()].reduce((acc, curr) => {
+      if (curr.type === type) {
+        acc++
+      }
+
+      return acc
+    }, 0)
   }
 
-  async cancelReservations (): Promise<void> {
-    await Promise.all(
-      [...this.reservations.entries()].map(async ([peerId, reservation]) => {
-        await this.#removeReservation(peerId, reservation)
-      })
-    )
+  cancelReservations (): void {
+    [...this.reservations.values()].forEach(reservation => {
+      clearTimeout(reservation.timeout)
+    })
+
+    this.reservations.clear()
   }
 
   async #createReservation (connection: Connection, options: AbortOptions): Promise<Reservation> {
@@ -353,11 +433,14 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
     const stream = await connection.newStream(RELAY_V2_HOP_CODEC, options)
     const pbstr = pbStream(stream)
     const hopstr = pbstr.pb(HopMessage)
+
+    this.log.trace('send RESERVE to %p', connection.remotePeer)
     await hopstr.write({ type: HopMessage.Type.RESERVE }, options)
 
     let response: HopMessage
 
     try {
+      this.log.trace('read response from %p', connection.remotePeer)
       response = await hopstr.read(options)
     } catch (err: any) {
       stream.abort(err)
@@ -368,6 +451,8 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
       }
     }
 
+    this.log.trace('read response from %p', response)
+
     if (response.status === Status.OK && response.reservation != null) {
       // check that the returned relay has the relay address - this can be
       // omitted when requesting a reservation from a go-libp2p relay we
@@ -405,24 +490,53 @@ export class ReservationStore extends TypedEventEmitter<ReservationStoreEvents>
   /**
    * Remove listen relay
    */
-  async #removeReservation (peerId: PeerId, reservation: RelayEntry): Promise<void> {
+  async #removeReservation (peerId: PeerId): Promise<void> {
+    const reservation = this.reservations.get(peerId)
+
+    if (reservation == null) {
+      return
+    }
+
     this.log('removing relay reservation with %p from local store', peerId)
     clearTimeout(reservation.timeout)
     this.reservations.delete(peerId)
 
+    // discover a new relay for this discovery request
+    if (reservation.type === 'discovered') {
+      this.pendingReservations.push(
+        reservation.id
+      )
+    }
+
     // untag the relay
     await this.peerStore.merge(peerId, {
       tags: {
-        [RELAY_TAG]: undefined,
         [KEEP_ALIVE_TAG]: undefined
       }
     })
 
-    this.safeDispatchEvent('relay:removed', { detail: peerId })
+    this.safeDispatchEvent('relay:removed', {
+      detail: {
+        relay: peerId,
+        details: reservation
+      }
+    })
+
+    // maybe trigger discovery of new relays
+    this.#checkReservationCount()
+  }
+
+  #checkReservationCount (): void {
+    if (this.pendingReservations.length === 0) {
+      this.log.trace('have discovered enough relays')
+      this.reserveQueue.clear()
+      this.safeDispatchEvent('relay:found-enough-relays')
 
-    if (this.reservations.size < this.maxDiscoveredRelays) {
-      this.log('not enough relays %d/%d', this.reservations.size, this.maxDiscoveredRelays)
-      this.safeDispatchEvent('relay:not-enough-relays', {})
+      return
     }
+
+    this.relayFilter = createScalableCuckooFilter(100)
+    this.log('not discovered enough relays %d/%d', this.reservations.size, this.pendingReservations.length)
+    this.safeDispatchEvent('relay:not-enough-relays')
   }
 }