@libp2p/kad-dht 13.1.2 → 14.0.0-0d326d102

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@libp2p/kad-dht",
-  "version": "13.1.2",
+  "version": "14.0.0-0d326d102",
   "description": "JavaScript implementation of the Kad-DHT for libp2p",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/libp2p/js-libp2p/tree/main/packages/kad-dht#readme",
@@ -57,13 +57,13 @@
     "doc-check": "aegir doc-check"
   },
   "dependencies": {
-    "@libp2p/crypto": "^5.0.4",
-    "@libp2p/interface": "^2.1.2",
-    "@libp2p/interface-internal": "^2.0.6",
-    "@libp2p/peer-collections": "^6.0.6",
-    "@libp2p/peer-id": "^5.0.4",
-    "@libp2p/record": "^4.0.4",
-    "@libp2p/utils": "^6.0.6",
+    "@libp2p/crypto": "5.0.4-0d326d102",
+    "@libp2p/interface": "2.1.2-0d326d102",
+    "@libp2p/interface-internal": "2.0.7-0d326d102",
+    "@libp2p/peer-collections": "6.0.7-0d326d102",
+    "@libp2p/peer-id": "5.0.4-0d326d102",
+    "@libp2p/record": "4.0.4-0d326d102",
+    "@libp2p/utils": "6.1.0-0d326d102",
     "@multiformats/multiaddr": "^12.2.3",
     "any-signal": "^4.1.1",
     "hashlru": "^2.3.0",
@@ -89,9 +89,9 @@
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
-    "@libp2p/interface-compliance-tests": "^6.1.4",
-    "@libp2p/logger": "^5.1.0",
-    "@libp2p/peer-store": "^11.0.6",
+    "@libp2p/interface-compliance-tests": "6.1.5-0d326d102",
+    "@libp2p/logger": "5.1.0-0d326d102",
+    "@libp2p/peer-store": "11.0.7-0d326d102",
     "@types/lodash.random": "^3.2.9",
     "@types/lodash.range": "^3.2.9",
     "@types/sinon": "^17.0.3",
@@ -109,7 +109,6 @@
     "lodash.random": "^3.2.0",
     "lodash.range": "^3.2.0",
     "p-retry": "^6.2.0",
-    "p-wait-for": "^5.0.2",
     "protons": "^7.5.0",
     "sinon": "^18.0.0",
     "sinon-ts": "^2.0.0",

package/src/constants.ts

@@ -5,6 +5,8 @@
 // until the year 2020 (a great time in the future). For that record to stick around
 // it must be rebroadcasted more frequently than once every 'MaxRecordAge'
 
+import { KEEP_ALIVE } from '@libp2p/interface'
+
 export const second = 1000
 export const minute = 60 * second
 export const hour = 60 * minute
@@ -51,3 +53,6 @@ export const TABLE_REFRESH_QUERY_TIMEOUT = 30 * second
 
 // When a timeout is not specified, run a query for this long
 export const DEFAULT_QUERY_TIMEOUT = 180 * second
+
+// used to ensure connections to our closest peers remain open
+export const KEEP_ALIVE_TAG = `${KEEP_ALIVE}-kad-dht`

package/src/index.ts

@@ -397,12 +397,10 @@ export interface KadDHTInit {
   logPrefix?: string
 
   /**
-   * How long to wait in ms when pinging DHT peers to decide if they
-   * should be evicted from the routing table or not.
-   *
-   * @default 10000
+   * Settings for how long to wait in ms when pinging DHT peers to decide if
+   * they should be evicted from the routing table or not.
    */
-  pingTimeout?: number
+  pingOldContactTimeout?: Omit<AdaptiveTimeoutInit, 'metricsName' | 'metrics'>
 
   /**
    * How many peers to ping in parallel when deciding if they should
@@ -410,7 +408,35 @@ export interface KadDHTInit {
    *
    * @default 10
    */
-  pingConcurrency?: number
+  pingOldContactConcurrency?: number
+
+  /**
+   * How long the queue to ping peers is allowed to grow
+   *
+   * @default 100
+   */
+  pingOldContactMaxQueueSize?: number
+
+  /**
+   * Settings for how long to wait in ms when pinging DHT peers to decide if
+   * they should be added to the routing table or not.
+   */
+  pingNewContactTimeout?: Omit<AdaptiveTimeoutInit, 'metricsName' | 'metrics'>
+
+  /**
+   * How many peers to ping in parallel when deciding if they should be added to
+   * the routing table or not
+   *
+   * @default 10
+   */
+  pingNewContactConcurrency?: number
+
+  /**
+   * How long the queue to ping peers is allowed to grow
+   *
+   * @default 100
+   */
+  pingNewContactMaxQueueSize?: number
 
   /**
    * How many parallel incoming streams to allow on the DHT protocol per

package/src/kad-dht.ts

@@ -138,8 +138,6 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       querySelfInterval,
       protocol,
       logPrefix,
-      pingTimeout,
-      pingConcurrency,
       maxInboundStreams,
       maxOutboundStreams,
       providers: providersInit
@@ -156,13 +154,6 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
     this.maxInboundStreams = maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS
     this.maxOutboundStreams = maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS
     this.peerInfoMapper = init.peerInfoMapper ?? removePrivateAddressesMapper
-    this.routingTable = new RoutingTable(components, {
-      kBucketSize,
-      pingTimeout,
-      pingConcurrency,
-      protocol: this.protocol,
-      logPrefix: loggingPrefix
-    })
 
     this.providers = new Providers(components, providersInit ?? {})
 
@@ -179,6 +170,21 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       logPrefix: loggingPrefix
     })
 
+    this.routingTable = new RoutingTable(components, {
+      kBucketSize,
+      pingOldContactTimeout: init.pingOldContactTimeout,
+      pingOldContactConcurrency: init.pingOldContactConcurrency,
+      pingOldContactMaxQueueSize: init.pingOldContactMaxQueueSize,
+      pingNewContactTimeout: init.pingNewContactTimeout,
+      pingNewContactConcurrency: init.pingNewContactConcurrency,
+      pingNewContactMaxQueueSize: init.pingNewContactMaxQueueSize,
+      protocol: this.protocol,
+      logPrefix: loggingPrefix,
+      prefixLength: init.prefixLength,
+      splitThreshold: init.kBucketSplitThreshold,
+      network: this.network
+    })
+
     // all queries should wait for the initial query-self query to run so we have
     // some peers and don't force consumers to use arbitrary timeouts
     const initialQuerySelfHasRun = pDefer<any>()
@@ -374,11 +380,17 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
 
     await this.components.registrar.unhandle(this.protocol)
 
+    // check again after async work
+    if (mode === this.getMode() && !force) {
+      this.log('already in %s mode', mode)
+      return
+    }
+
     if (mode === 'client') {
-      this.log('enabling client mode')
+      this.log('enabling client mode while in %s mode', this.getMode())
       this.clientMode = true
     } else {
-      this.log('enabling server mode')
+      this.log('enabling server mode while in %s mode', this.getMode())
       this.clientMode = false
       await this.components.registrar.handle(this.protocol, this.rpc.onIncomingStream.bind(this.rpc), {
         maxInboundStreams: this.maxInboundStreams,
@@ -397,14 +409,18 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
     await this.setMode(this.clientMode ? 'client' : 'server', true)
 
     await start(
-      this.querySelf,
+      this.routingTable,
       this.providers,
       this.queryManager,
       this.network,
-      this.routingTable,
       this.topologyListener,
       this.routingTableRefresh
     )
+
+    // Query self after other components are configured
+    await start(
+      this.querySelf
+    )
   }
 
   /**

package/src/network.ts

@@ -85,7 +85,7 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
   }
 
   /**
-   * Send a request and record RTT for latency measurements
+   * Send a request and read a response
    */
   async * sendRequest (to: PeerId, msg: Partial<Message>, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
     if (!this.running) {
@@ -204,7 +204,6 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
   async _writeMessage (stream: Stream, msg: Partial<Message>, options: AbortOptions): Promise<void> {
     const pb = pbStream(stream)
     await pb.write(msg, Message, options)
-    await pb.unwrap().close(options)
   }
 
   /**
@@ -219,8 +218,6 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
 
     const message = await pb.read(Message, options)
 
-    await pb.unwrap().close(options)
-
     // tell any listeners about new peers we've seen
     message.closer.forEach(peerData => {
       this.safeDispatchEvent<PeerInfo>('peer', {

package/src/{peer-list/peer-distance-list.ts → peer-distance-list.ts}

@@ -1,6 +1,6 @@
 import { xor as uint8ArrayXor } from 'uint8arrays/xor'
 import { xorCompare as uint8ArrayXorCompare } from 'uint8arrays/xor-compare'
-import { convertPeerId } from '../utils.js'
+import { convertPeerId } from './utils.js'
 import type { PeerId, PeerInfo } from '@libp2p/interface'
 
 interface PeerDistance {

package/src/peer-routing/index.ts

@@ -5,7 +5,7 @@ import { Libp2pRecord } from '@libp2p/record'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { QueryError, InvalidRecordError } from '../errors.js'
 import { MessageType } from '../message/dht.js'
-import { PeerDistanceList } from '../peer-list/peer-distance-list.js'
+import { PeerDistanceList } from '../peer-distance-list.js'
 import {
   queryErrorEvent,
   finalPeerEvent,

package/src/query-self.ts

@@ -106,19 +106,27 @@ export class QuerySelf implements Startable {
 
     if (this.started) {
       this.controller = new AbortController()
-      const timeoutSignal = AbortSignal.timeout(this.queryTimeout)
-      const signal = anySignal([this.controller.signal, timeoutSignal])
+      const signals = [this.controller.signal]
 
-      // this controller will get used for lots of dial attempts so make sure we don't cause warnings to be logged
-      setMaxListeners(Infinity, signal, this.controller.signal, timeoutSignal)
+      // add a shorter timeout if we've already run our initial self query
+      if (this.initialQuerySelfHasRun == null) {
+        const timeoutSignal = AbortSignal.timeout(this.queryTimeout)
+        setMaxListeners(Infinity, timeoutSignal)
+        signals.push(timeoutSignal)
+      }
+
+      const signal = anySignal(signals)
+      setMaxListeners(Infinity, signal, this.controller.signal)
 
       try {
         if (this.routingTable.size === 0) {
           this.log('routing table was empty, waiting for some peers before running query')
-          // wait to discover at least one DHT peer
+          // wait to discover at least one DHT peer that isn't us
           await pEvent(this.routingTable, 'peer:add', {
-            signal
+            signal,
+            filter: (event) => !this.peerId.equals(event.detail)
           })
+          this.log('routing table has peers, continuing with query')
         }
 
         this.log('run self-query, look for %d peers timing out after %dms', this.count, this.queryTimeout)

package/src/routing-table/closest-peers.ts

@@ -0,0 +1,113 @@
+import { PeerSet } from '@libp2p/peer-collections'
+import { KEEP_ALIVE_TAG } from '../constants.js'
+import { PeerDistanceList } from '../peer-distance-list.js'
+import { convertPeerId } from '../utils.js'
+import type { RoutingTable } from './index.js'
+import type { ComponentLogger, Logger, Metrics, PeerId, PeerStore, Startable } from '@libp2p/interface'
+
+export const PEER_SET_SIZE = 20
+export const REFRESH_INTERVAL = 5000
+export const KAD_CLOSE_TAG_NAME = 'kad-close'
+export const KAD_CLOSE_TAG_VALUE = 50
+
+export interface ClosestPeersInit {
+  logPrefix: string
+  routingTable: RoutingTable
+  peerSetSize?: number
+  refreshInterval?: number
+  closeTagName?: string
+  closeTagValue?: number
+}
+
+export interface ClosestPeersComponents {
+  peerId: PeerId
+  peerStore: PeerStore
+  metrics?: Metrics
+  logger: ComponentLogger
+}
+
+/**
+ * Contains a list of the kad-closest peers encountered on the network.
+ *
+ * Once every few seconds, if the list has changed, it tags the closest peers.
+ */
+export class ClosestPeers implements Startable {
+  private readonly routingTable: RoutingTable
+  private readonly components: ClosestPeersComponents
+  private closestPeers: PeerSet
+  private newPeers?: PeerDistanceList
+  private readonly refreshInterval: number
+  private readonly peerSetSize: number
+  private timeout?: ReturnType<typeof setTimeout>
+  private readonly closeTagName: string
+  private readonly closeTagValue: number
+  private readonly log: Logger
+
+  constructor (components: ClosestPeersComponents, init: ClosestPeersInit) {
+    this.components = components
+    this.log = components.logger.forComponent(`${init.logPrefix}:routing-table`)
+    this.routingTable = init.routingTable
+    this.refreshInterval = init.refreshInterval ?? REFRESH_INTERVAL
+    this.peerSetSize = init.peerSetSize ?? PEER_SET_SIZE
+    this.closeTagName = init.closeTagName ?? KAD_CLOSE_TAG_NAME
+    this.closeTagValue = init.closeTagValue ?? KAD_CLOSE_TAG_VALUE
+
+    this.closestPeers = new PeerSet()
+    this.onPeerPing = this.onPeerPing.bind(this)
+  }
+
+  async start (): Promise<void> {
+    const targetKadId = await convertPeerId(this.components.peerId)
+    this.newPeers = new PeerDistanceList(targetKadId, this.peerSetSize)
+    this.routingTable.addEventListener('peer:ping', this.onPeerPing)
+
+    this.timeout = setInterval(() => {
+      this.updatePeerTags()
+        .catch(err => {
+          this.log.error('error updating peer tags - %e', err)
+        })
+    }, this.refreshInterval)
+  }
+
+  stop (): void {
+    this.routingTable.removeEventListener('peer:ping', this.onPeerPing)
+    clearTimeout(this.timeout)
+  }
+
+  onPeerPing (event: CustomEvent<PeerId>): void {
+    this.newPeers?.add({ id: event.detail, multiaddrs: [] })
+      .catch(err => {
+        this.log.error('error adding peer to distance list - %e', err)
+      })
+  }
+
+  async updatePeerTags (): Promise<void> {
+    const newClosest = new PeerSet(this.newPeers?.peers.map(peer => peer.id))
+    const added = newClosest.difference(this.closestPeers)
+    const removed = this.closestPeers.difference(newClosest)
+    this.closestPeers = newClosest
+
+    await Promise.all([
+      ...[...added].map(async peerId => {
+        await this.components.peerStore.merge(peerId, {
+          tags: {
+            [this.closeTagName]: {
+              value: this.closeTagValue
+            },
+            [KEEP_ALIVE_TAG]: {
+              value: 1
+            }
+          }
+        })
+      }),
+      ...[...removed].map(async peerId => {
+        await this.components.peerStore.merge(peerId, {
+          tags: {
+            [this.closeTagName]: undefined,
+            [KEEP_ALIVE_TAG]: undefined
+          }
+        })
+      })
+    ])
+  }
+}