@libp2p/kad-dht 0.28.6

package/src/routing-table/index.ts

@@ -0,0 +1,265 @@
+// @ts-expect-error no types
+import KBuck from 'k-bucket'
+import * as utils from '../utils.js'
+import Queue from 'p-queue'
+import { PROTOCOL_DHT } from '../constants.js'
+import { TimeoutController } from 'timeout-abort-controller'
+import { logger } from '@libp2p/logger'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { Dialer, Startable } from '@libp2p/interfaces'
+import type { ComponentMetricsTracker } from '@libp2p/interfaces/metrics'
+import type { Logger } from '@libp2p/logger'
+
+export interface KBucketPeer {
+  id: Uint8Array
+  peer: PeerId
+}
+
+export interface KBucket {
+  id: Uint8Array
+  contacts: KBucketPeer[]
+  dontSplit: boolean
+  left: KBucket
+  right: KBucket
+}
+
+export interface KBucketTree {
+  root: KBucket
+  localNodeId: Uint8Array
+  on: (event: 'ping', callback: (oldContacts: KBucketPeer[], newContact: KBucketPeer) => void) => void
+  closest: (key: Uint8Array, count: number) => KBucketPeer[]
+  closestPeer: (key: Uint8Array) => KBucketPeer
+  remove: (key: Uint8Array) => void
+  add: (peer: KBucketPeer) => void
+  get: (key: Uint8Array) => Uint8Array
+  count: () => number
+  toIterable: () => Iterable<KBucket>
+}
+
+const METRIC_ROUTING_TABLE_SIZE = 'routing-table-size'
+
+export interface RoutingTableOptions {
+  peerId: PeerId
+  dialer: Dialer
+  lan: boolean
+  metrics?: ComponentMetricsTracker
+  kBucketSize?: number
+  pingTimeout?: number
+}
+
+/**
+ * A wrapper around `k-bucket`, to provide easy store and
+ * retrieval for peers.
+ */
+export class RoutingTable implements Startable {
+  private readonly log: Logger
+  private readonly peerId: PeerId
+  public dialer: Dialer
+  private readonly lan: boolean
+  private readonly metrics?: ComponentMetricsTracker
+  public kBucketSize: number
+  private readonly pingTimeout: number
+  public kb?: KBucketTree
+  private running: boolean
+  public pingQueue: Queue
+
+  constructor (options: RoutingTableOptions) {
+    const { peerId, dialer, kBucketSize, pingTimeout, lan, metrics } = options
+
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:routing-table`)
+    this.peerId = peerId
+    this.dialer = dialer
+    this.kBucketSize = kBucketSize ?? 20
+    this.pingTimeout = pingTimeout ?? 10000
+    this.lan = lan
+    this.metrics = metrics
+    this.pingQueue = new Queue({ concurrency: 1 })
+    this.running = false
+
+    this._onPing = this._onPing.bind(this)
+  }
+
+  isStarted () {
+    return this.running
+  }
+
+  async start () {
+    this.running = true
+
+    const kBuck = new KBuck({
+      localNodeId: await utils.convertPeerId(this.peerId),
+      numberOfNodesPerKBucket: this.kBucketSize,
+      numberOfNodesToPing: 1
+    })
+    kBuck.on('ping', this._onPing)
+    this.kb = kBuck
+  }
+
+  async stop () {
+    this.running = false
+    this.pingQueue.clear()
+    this.kb = undefined
+  }
+
+  /**
+   * Called on the `ping` event from `k-bucket` when a bucket is full
+   * and cannot split.
+   *
+   * `oldContacts.length` is defined by the `numberOfNodesToPing` param
+   * passed to the `k-bucket` constructor.
+   *
+   * `oldContacts` will not be empty and is the list of contacts that
+   * have not been contacted for the longest.
+   */
+  _onPing (oldContacts: KBucketPeer[], newContact: KBucketPeer) {
+    // add to a queue so multiple ping requests do not overlap and we don't
+    // flood the network with ping requests if lots of newContact requests
+    // are received
+    this.pingQueue.add(async () => {
+      if (!this.running) {
+        return
+      }
+
+      let responded = 0
+
+      try {
+        await Promise.all(
+          oldContacts.map(async oldContact => {
+            let timeoutController
+
+            try {
+              timeoutController = new TimeoutController(this.pingTimeout)
+
+              this.log('pinging old contact %p', oldContact.peer)
+              const { stream } = await this.dialer.dialProtocol(oldContact.peer, PROTOCOL_DHT, {
+                signal: timeoutController.signal
+              })
+              await stream.close()
+              responded++
+            } catch (err: any) {
+              if (this.running && this.kb != null) {
+                // only evict peers if we are still running, otherwise we evict when dialing is
+                // cancelled due to shutdown in progress
+                this.log.error('could not ping peer %p', oldContact.peer, err)
+                this.log('evicting old contact after ping failed %p', oldContact)
+                this.kb.remove(oldContact.id)
+              }
+            } finally {
+              if (timeoutController != null) {
+                timeoutController.clear()
+              }
+
+              this.metrics?.updateComponentMetric({
+                system: 'libp2p',
+                component: `kad-dht-${this.lan ? 'lan' : 'wan'}`,
+                metric: METRIC_ROUTING_TABLE_SIZE,
+                value: this.size
+              })
+            }
+          })
+        )
+
+        if (this.running && responded < oldContacts.length && this.kb != null) {
+          this.log('adding new contact %p', newContact.peer)
+          this.kb.add(newContact)
+        }
+      } catch (err: any) {
+        this.log.error('could not process k-bucket ping event', err)
+      }
+    })
+      .catch(err => {
+        this.log.error('could not process k-bucket ping event', err)
+      })
+  }
+
+  // -- Public Interface
+
+  /**
+   * Amount of currently stored peers
+   */
+  get size () {
+    if (this.kb == null) {
+      return 0
+    }
+
+    return this.kb.count()
+  }
+
+  /**
+   * Find a specific peer by id
+   */
+  async find (peer: PeerId) {
+    const key = await utils.convertPeerId(peer)
+    const closest = this.closestPeer(key)
+
+    if (closest != null && peer.equals(closest)) {
+      return closest
+    }
+  }
+
+  /**
+   * Retrieve the closest peers to the given key
+   */
+  closestPeer (key: Uint8Array) {
+    const res = this.closestPeers(key, 1)
+
+    if (res.length > 0) {
+      return res[0]
+    }
+  }
+
+  /**
+   * Retrieve the `count`-closest peers to the given key
+   */
+  closestPeers (key: Uint8Array, count = this.kBucketSize) {
+    if (this.kb == null) {
+      return []
+    }
+
+    const closest = this.kb.closest(key, count)
+
+    return closest.map(p => p.peer)
+  }
+
+  /**
+   * Add or update the routing table with the given peer
+   */
+  async add (peer: PeerId) {
+    if (this.kb == null) {
+      throw new Error('RoutingTable is not started')
+    }
+
+    const id = await utils.convertPeerId(peer)
+
+    this.kb.add({ id: id, peer: peer })
+
+    this.log('added %p with kad id %b', peer, id)
+
+    this.metrics?.updateComponentMetric({
+      system: 'libp2p',
+      component: `kad-dht-${this.lan ? 'lan' : 'wan'}`,
+      metric: METRIC_ROUTING_TABLE_SIZE,
+      value: this.size
+    })
+  }
+
+  /**
+   * Remove a given peer from the table
+   */
+  async remove (peer: PeerId) {
+    if (this.kb == null) {
+      throw new Error('RoutingTable is not started')
+    }
+
+    const id = await utils.convertPeerId(peer)
+
+    this.kb.remove(id)
+
+    this.metrics?.updateComponentMetric({
+      system: 'libp2p',
+      component: `kad-dht-${this.lan ? 'lan' : 'wan'}`,
+      metric: METRIC_ROUTING_TABLE_SIZE,
+      value: this.size
+    })
+  }
+}

package/src/routing-table/refresh.ts

@@ -0,0 +1,263 @@
+import { xor as uint8ArrayXor } from 'uint8arrays/xor'
+import GENERATED_PREFIXES from './generated-prefix-list.js'
+import { sha256 } from 'multiformats/hashes/sha2'
+import { randomBytes } from '@libp2p/crypto'
+import { peerIdFromBytes } from '@libp2p/peer-id'
+import { logger } from '@libp2p/logger'
+import length from 'it-length'
+import { TimeoutController } from 'timeout-abort-controller'
+import { TABLE_REFRESH_INTERVAL, TABLE_REFRESH_QUERY_TIMEOUT } from '../constants.js'
+import type { RoutingTable } from './index.js'
+import type { Logger } from '@libp2p/logger'
+import type { PeerRouting } from '../peer-routing/index.js'
+
+/**
+ * Cannot generate random KadIds longer than this + 1
+ */
+const MAX_COMMON_PREFIX_LENGTH = 15
+
+export interface RoutingTableRefreshOptions {
+  peerRouting: PeerRouting
+  routingTable: RoutingTable
+  lan: boolean
+  refreshInterval?: number
+  refreshQueryTimeout?: number
+}
+
+/**
+ * A wrapper around `k-bucket`, to provide easy store and
+ * retrieval for peers.
+ */
+export class RoutingTableRefresh {
+  private readonly log: Logger
+  private readonly peerRouting: PeerRouting
+  private readonly routingTable: RoutingTable
+  private readonly refreshInterval: number
+  private readonly refreshQueryTimeout: number
+  private readonly commonPrefixLengthRefreshedAt: Date[]
+  private refreshTimeoutId?: NodeJS.Timer
+
+  constructor (options: RoutingTableRefreshOptions) {
+    const { peerRouting, routingTable, refreshInterval, refreshQueryTimeout, lan } = options
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:routing-table:refresh`)
+    this.peerRouting = peerRouting
+    this.routingTable = routingTable
+    this.refreshInterval = refreshInterval ?? TABLE_REFRESH_INTERVAL
+    this.refreshQueryTimeout = refreshQueryTimeout ?? TABLE_REFRESH_QUERY_TIMEOUT
+    this.commonPrefixLengthRefreshedAt = []
+
+    this.refreshTable = this.refreshTable.bind(this)
+  }
+
+  async start () {
+    this.log(`refreshing routing table every ${this.refreshInterval}ms`)
+    this.refreshTable(true)
+  }
+
+  async stop () {
+    if (this.refreshTimeoutId != null) {
+      clearTimeout(this.refreshTimeoutId)
+    }
+  }
+
+  /**
+   * To speed lookups, we seed the table with random PeerIds. This means
+   * when we are asked to locate a peer on the network, we can find a KadId
+   * that is close to the requested peer ID and query that, then network
+   * peers will tell us who they know who is close to the fake ID
+   */
+  refreshTable (force: boolean = false) {
+    this.log('refreshing routing table')
+
+    const prefixLength = this._maxCommonPrefix()
+    const refreshCpls = this._getTrackedCommonPrefixLengthsForRefresh(prefixLength)
+
+    this.log(`max common prefix length ${prefixLength}`)
+    this.log(`tracked CPLs [ ${refreshCpls.map(date => date.toISOString()).join(', ')} ]`)
+
+    /**
+     * If we see a gap at a common prefix length in the Routing table, we ONLY refresh up until
+     * the maximum cpl we have in the Routing Table OR (2 * (Cpl+ 1) with the gap), whichever
+     * is smaller.
+     *
+     * This is to prevent refreshes for Cpls that have no peers in the network but happen to be
+     * before a very high max Cpl for which we do have peers in the network.
+     *
+     * The number of 2 * (Cpl + 1) can be proved and a proof would have been written here if
+     * the programmer had paid more attention in the Math classes at university.
+     *
+     * So, please be patient and a doc explaining it will be published soon.
+     *
+     * https://github.com/libp2p/go-libp2p-kad-dht/commit/2851c88acb0a3f86bcfe3cfd0f4604a03db801d8#diff-ad45f4ba97ffbc4083c2eb87a4420c1157057b233f048030d67c6b551855ccf6R219
+     */
+    Promise.all(
+      refreshCpls.map(async (lastRefresh, index) => {
+        try {
+          await this._refreshCommonPrefixLength(index, lastRefresh, force)
+
+          if (this._numPeersForCpl(prefixLength) === 0) {
+            const lastCpl = Math.min(2 * (index + 1), refreshCpls.length - 1)
+
+            for (let n = index + 1; n < lastCpl + 1; n++) {
+              try {
+                await this._refreshCommonPrefixLength(n, lastRefresh, force)
+              } catch (err: any) {
+                this.log.error(err)
+              }
+            }
+          }
+        } catch (err: any) {
+          this.log.error(err)
+        }
+      })
+    ).catch(err => {
+      this.log.error(err)
+    }).then(() => {
+      this.refreshTimeoutId = setTimeout(this.refreshTable, this.refreshInterval)
+
+      if (this.refreshTimeoutId.unref != null) {
+        this.refreshTimeoutId.unref()
+      }
+    }).catch(err => {
+      this.log.error(err)
+    })
+  }
+
+  async _refreshCommonPrefixLength (cpl: number, lastRefresh: Date, force: boolean) {
+    if (!force && lastRefresh.getTime() > (Date.now() - this.refreshInterval)) {
+      this.log('not running refresh for cpl %s as time since last refresh not above interval', cpl)
+      return
+    }
+
+    // gen a key for the query to refresh the cpl
+    const peerId = await this._generateRandomPeerId(cpl)
+
+    this.log('starting refreshing cpl %s with key %p (routing table size was %s)', cpl, peerId, this.routingTable.size)
+
+    const controller = new TimeoutController(this.refreshQueryTimeout)
+
+    try {
+      const peers = await length(this.peerRouting.getClosestPeers(peerId.toBytes(), { signal: controller.signal }))
+
+      this.log(`found ${peers} peers that were close to imaginary peer %p`, peerId)
+      this.log('finished refreshing cpl %s with key %p (routing table size is now %s)', cpl, peerId, this.routingTable.size)
+    } finally {
+      controller.clear()
+    }
+  }
+
+  _getTrackedCommonPrefixLengthsForRefresh (maxCommonPrefix: number) {
+    if (maxCommonPrefix > MAX_COMMON_PREFIX_LENGTH) {
+      maxCommonPrefix = MAX_COMMON_PREFIX_LENGTH
+    }
+
+    const dates = []
+
+    for (let i = 0; i <= maxCommonPrefix; i++) {
+      // defaults to the zero value if we haven't refreshed it yet.
+      dates[i] = this.commonPrefixLengthRefreshedAt[i] ?? new Date()
+    }
+
+    return dates
+  }
+
+  async _generateRandomPeerId (targetCommonPrefixLength: number) {
+    if (this.routingTable.kb == null) {
+      throw new Error('Routing table not started')
+    }
+
+    const randomData = randomBytes(2)
+    const randomUint16 = (randomData[1] << 8) + randomData[0]
+
+    const key = await this._makePeerId(this.routingTable.kb.localNodeId, randomUint16, targetCommonPrefixLength)
+
+    return peerIdFromBytes(key)
+  }
+
+  async _makePeerId (localKadId: Uint8Array, randomPrefix: number, targetCommonPrefixLength: number) {
+    if (targetCommonPrefixLength > MAX_COMMON_PREFIX_LENGTH) {
+      throw new Error(`Cannot generate peer ID for common prefix length greater than ${MAX_COMMON_PREFIX_LENGTH}`)
+    }
+
+    const view = new DataView(localKadId.buffer, localKadId.byteOffset, localKadId.byteLength)
+    const localPrefix = view.getUint16(0, false)
+
+    // For host with ID `L`, an ID `K` belongs to a bucket with ID `B` ONLY IF CommonPrefixLen(L,K) is EXACTLY B.
+    // Hence, to achieve a targetPrefix `T`, we must toggle the (T+1)th bit in L & then copy (T+1) bits from L
+    // to our randomly generated prefix.
+    const toggledLocalPrefix = localPrefix ^ (0x8000 >> targetCommonPrefixLength)
+
+    // Combine the toggled local prefix and the random bits at the correct offset
+    // such that ONLY the first `targetCommonPrefixLength` bits match the local ID.
+    const mask = 65535 << (16 - (targetCommonPrefixLength + 1))
+    const targetPrefix = (toggledLocalPrefix & mask) | (randomPrefix & ~mask)
+
+    // Convert to a known peer ID.
+    const keyPrefix = GENERATED_PREFIXES[targetPrefix]
+
+    const keyBuffer = new ArrayBuffer(34)
+    const keyView = new DataView(keyBuffer, 0, keyBuffer.byteLength)
+    keyView.setUint8(0, sha256.code)
+    keyView.setUint8(1, 32)
+    keyView.setUint32(2, keyPrefix, false)
+
+    return new Uint8Array(keyView.buffer, keyView.byteOffset, keyView.byteLength)
+  }
+
+  /**
+   * returns the maximum common prefix length between any peer in the table
+   * and the current peer
+   */
+  _maxCommonPrefix () {
+    // xor our KadId with every KadId in the k-bucket tree,
+    // return the longest id prefix that is the same
+    let prefixLength = 0
+
+    for (const length of this._prefixLengths()) {
+      if (length > prefixLength) {
+        prefixLength = length
+      }
+    }
+
+    return prefixLength
+  }
+
+  /**
+   * Returns the number of peers in the table with a given prefix length
+   */
+  _numPeersForCpl (prefixLength: number) {
+    let count = 0
+
+    for (const length of this._prefixLengths()) {
+      if (length === prefixLength) {
+        count++
+      }
+    }
+
+    return count
+  }
+
+  /**
+   * Yields the common prefix length of every peer in the table
+   */
+  * _prefixLengths () {
+    if (this.routingTable.kb == null) {
+      return
+    }
+
+    for (const { id } of this.routingTable.kb.toIterable()) {
+      const distance = uint8ArrayXor(this.routingTable.kb.localNodeId, id)
+      let leadingZeros = 0
+
+      for (const byte of distance) {
+        if (byte === 0) {
+          leadingZeros++
+        } else {
+          break
+        }
+      }
+
+      yield leadingZeros
+    }
+  }
+}

package/src/rpc/handlers/add-provider.ts

@@ -0,0 +1,63 @@
+import { CID } from 'multiformats/cid'
+import errcode from 'err-code'
+import { logger } from '@libp2p/logger'
+import type { Providers } from '../../providers'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { DHTMessageHandler } from '../index.js'
+import type { Message } from '../../message/index.js'
+
+const log = logger('libp2p:kad-dht:rpc:handlers:add-provider')
+
+export interface AddProviderHandlerOptions {
+  providers: Providers
+}
+
+export class AddProviderHandler implements DHTMessageHandler {
+  private readonly providers: Providers
+
+  constructor (options: AddProviderHandlerOptions) {
+    const { providers } = options
+    this.providers = providers
+  }
+
+  async handle (peerId: PeerId, msg: Message) {
+    log('start')
+
+    if (msg.key == null || msg.key.length === 0) {
+      throw errcode(new Error('Missing key'), 'ERR_MISSING_KEY')
+    }
+
+    let cid: CID
+    try {
+      // this is actually just the multihash, not the whole CID
+      cid = CID.decode(msg.key)
+    } catch (err: any) {
+      throw errcode(new Error('Invalid CID'), 'ERR_INVALID_CID')
+    }
+
+    if (msg.providerPeers == null || msg.providerPeers.length === 0) {
+      log.error('no providers found in message')
+    }
+
+    await Promise.all(
+      msg.providerPeers.map(async (pi) => {
+        // Ignore providers not from the originator
+        if (!pi.id.equals(peerId)) {
+          log('invalid provider peer %p from %p', pi.id, peerId)
+          return
+        }
+
+        if (pi.multiaddrs.length < 1) {
+          log('no valid addresses for provider %p. Ignore', peerId)
+          return
+        }
+
+        log('received provider %p for %s (addrs %s)', peerId, cid, pi.multiaddrs.map((m) => m.toString()))
+
+        await this.providers.addProvider(cid, pi.id)
+      })
+    )
+
+    return undefined
+  }
+}

package/src/rpc/handlers/find-node.ts

@@ -0,0 +1,57 @@
+import { Message } from '../../message/index.js'
+import { logger } from '@libp2p/logger'
+import {
+  removePrivateAddresses,
+  removePublicAddresses
+} from '../../utils.js'
+import { pipe } from 'it-pipe'
+import type { DHTMessageHandler } from '../index.js'
+import type { PeerRouting } from '../../peer-routing/index.js'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import map from 'it-map'
+import filter from 'it-filter'
+import all from 'it-all'
+
+const log = logger('libp2p:kad-dht:rpc:handlers:find-node')
+
+export interface FindNodeHandlerOptions {
+  peerRouting: PeerRouting
+  lan: boolean
+}
+
+export class FindNodeHandler implements DHTMessageHandler {
+  private readonly peerRouting: PeerRouting
+  private readonly lan: boolean
+
+  constructor (options: FindNodeHandlerOptions) {
+    const { peerRouting, lan } = options
+    this.peerRouting = peerRouting
+    this.lan = Boolean(lan)
+  }
+
+  /**
+   * Process `FindNode` DHT messages
+   */
+  async handle (peerId: PeerId, msg: Message) {
+    log('incoming request from %p for peers closer to %b', peerId, msg.key)
+
+    const mapper = this.lan ? removePublicAddresses : removePrivateAddresses
+
+    const closer = await pipe(
+      await this.peerRouting.getCloserPeersOffline(msg.key, peerId),
+      (source) => map(source, mapper),
+      (source) => filter(source, ({ multiaddrs }) => multiaddrs.length > 0),
+      async (source) => await all(source)
+    )
+
+    const response = new Message(msg.type, new Uint8Array(0), msg.clusterLevel)
+
+    if (closer.length > 0) {
+      response.closerPeers = closer
+    } else {
+      log('could not find any peers closer to %b than %p', msg.key, peerId)
+    }
+
+    return response
+  }
+}