@libp2p/kad-dht 0.28.6

package/src/query/events.ts

@@ -0,0 +1,126 @@
+import { MESSAGE_TYPE_LOOKUP } from '../message/index.js'
+import type { SendingQueryEvent, PeerResponseEvent, MessageType, DialingPeerEvent, AddingPeerEvent, ValueEvent, ProviderEvent, QueryErrorEvent, FinalPeerEvent } from '@libp2p/interfaces/dht'
+import type { PeerData } from '@libp2p/interfaces/peer-data'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { Libp2pRecord } from '@libp2p/record'
+
+const MESSAGE_NAMES = [
+  'PUT_VALUE',
+  'GET_VALUE',
+  'ADD_PROVIDER',
+  'GET_PROVIDERS',
+  'FIND_NODE',
+  'PING'
+]
+
+export interface QueryEventFields {
+  to: PeerId
+  type: MessageType
+}
+
+export function sendingQueryEvent (fields: QueryEventFields): SendingQueryEvent {
+  return {
+    ...fields,
+    name: 'SENDING_QUERY',
+    type: 0,
+    // @ts-expect-error cannot look up values like this
+    messageName: MESSAGE_TYPE_LOOKUP[fields.type],
+    messageType: fields.type
+  }
+}
+
+export interface PeerResponseEventField {
+  from: PeerId
+  messageType: MessageType
+  closer?: PeerData[]
+  providers?: PeerData[]
+  record?: Libp2pRecord
+}
+
+export function peerResponseEvent (fields: PeerResponseEventField): PeerResponseEvent {
+  return {
+    ...fields,
+    name: 'PEER_RESPONSE',
+    type: 1,
+    // @ts-expect-error cannot look up values like this
+    messageName: MESSAGE_NAMES[fields.messageType],
+    closer: (fields.closer != null) ? fields.closer : [],
+    providers: (fields.providers != null) ? fields.providers : []
+  }
+}
+
+export interface FinalPeerEventFields {
+  from: PeerId
+  peer: PeerData
+}
+
+export function finalPeerEvent (fields: FinalPeerEventFields): FinalPeerEvent {
+  return {
+    ...fields,
+    name: 'FINAL_PEER',
+    type: 2
+  }
+}
+
+export interface ErrorEventFields {
+  from: PeerId
+  error: Error
+}
+
+export function queryErrorEvent (fields: ErrorEventFields): QueryErrorEvent {
+  return {
+    ...fields,
+    name: 'QUERY_ERROR',
+    type: 3
+  }
+}
+
+export interface ProviderEventFields {
+  from: PeerId
+  providers: PeerData[]
+}
+
+export function providerEvent (fields: ProviderEventFields): ProviderEvent {
+  return {
+    ...fields,
+    name: 'PROVIDER',
+    type: 4
+  }
+}
+
+export interface ValueEventFields {
+  from: PeerId
+  value: Uint8Array
+}
+
+export function valueEvent (fields: ValueEventFields): ValueEvent {
+  return {
+    ...fields,
+    name: 'VALUE',
+    type: 5
+  }
+}
+
+export interface PeerEventFields {
+  peer: PeerId
+}
+
+export function addingPeerEvent (fields: PeerEventFields): AddingPeerEvent {
+  return {
+    ...fields,
+    name: 'ADDING_PEER',
+    type: 6
+  }
+}
+
+export interface DialingPeerEventFields {
+  peer: PeerId
+}
+
+export function dialingPeerEvent (fields: DialingPeerEventFields): DialingPeerEvent {
+  return {
+    ...fields,
+    name: 'DIALING_PEER',
+    type: 7
+  }
+}

package/src/query/manager.ts

@@ -0,0 +1,188 @@
+import { TimeoutController } from 'timeout-abort-controller'
+import { anySignal } from 'any-signal'
+import {
+  ALPHA, K, DEFAULT_QUERY_TIMEOUT
+} from '../constants.js'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { queryPath } from './query-path.js'
+import merge from 'it-merge'
+import {
+  // @ts-expect-error only defined in node 15+
+  setMaxListeners
+} from 'events'
+import { EventEmitter, CustomEvent } from '@libp2p/interfaces'
+import { logger } from '@libp2p/logger'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { ComponentMetricsTracker } from '@libp2p/interfaces/metrics'
+import type { Startable } from '@libp2p/interfaces'
+import type { QueryFunc } from './types.js'
+import type { QueryOptions } from '@libp2p/interfaces/dht'
+
+const METRIC_RUNNING_QUERIES = 'running-queries'
+
+export interface CleanUpEvents {
+  'cleanup': CustomEvent
+}
+
+export interface QueryManagerOptions {
+  peerId: PeerId
+  lan?: boolean
+  metrics?: ComponentMetricsTracker
+  disjointPaths?: number
+  alpha?: number
+}
+
+/**
+ * Keeps track of all running queries
+ */
+export class QueryManager implements Startable {
+  private readonly peerId: PeerId
+  private readonly lan: boolean
+  private readonly metrics?: ComponentMetricsTracker
+  public disjointPaths: number
+  private readonly alpha: number
+  private readonly controllers: Set<AbortController>
+  private running: boolean
+  private queries: number
+
+  constructor (options: QueryManagerOptions) {
+    const { peerId, lan = false, metrics, disjointPaths = K, alpha = ALPHA } = options
+    this.peerId = peerId
+    this.disjointPaths = disjointPaths ?? K
+    this.controllers = new Set()
+    this.running = false
+    this.alpha = alpha ?? ALPHA
+    this.lan = lan
+    this.metrics = metrics
+    this.queries = 0
+  }
+
+  isStarted () {
+    return this.running
+  }
+
+  /**
+   * Starts the query manager
+   */
+  async start () {
+    this.running = true
+  }
+
+  /**
+   * Stops all queries
+   */
+  async stop () {
+    this.running = false
+
+    for (const controller of this.controllers) {
+      controller.abort()
+    }
+
+    this.controllers.clear()
+  }
+
+  async * run (key: Uint8Array, peers: PeerId[], queryFunc: QueryFunc, options: QueryOptions = {}) {
+    if (!this.running) {
+      throw new Error('QueryManager not started')
+    }
+
+    let timeoutController
+
+    if (options.signal == null) {
+      // don't let queries run forever
+      timeoutController = new TimeoutController(DEFAULT_QUERY_TIMEOUT)
+      options.signal = timeoutController.signal
+    }
+
+    // allow us to stop queries on shut down
+    const abortController = new AbortController()
+    this.controllers.add(abortController)
+    const signals = [abortController.signal]
+
+    if (options.signal != null) {
+      signals.push(options.signal)
+    }
+
+    const signal = anySignal(signals)
+
+    // this signal will get listened to for every invocation of queryFunc
+    // so make sure we don't make a lot of noise in the logs
+    try {
+      if (setMaxListeners != null) {
+        setMaxListeners(0, signal)
+      }
+    } catch {} // fails on node < 15.4
+
+    const log = logger(`libp2p:kad-dht:${this.lan ? 'lan' : 'wan'}:query:` + uint8ArrayToString(key, 'base58btc'))
+
+    // query a subset of peers up to `kBucketSize / 2` in length
+    const peersToQuery = peers.slice(0, Math.min(this.disjointPaths, peers.length))
+    const startTime = Date.now()
+    const cleanUp = new EventEmitter<CleanUpEvents>()
+
+    try {
+      log('query:start')
+      this.queries++
+      this.metrics?.updateComponentMetric({
+        system: 'libp2p',
+        component: `kad-dht-${this.lan ? 'lan' : 'wan'}`,
+        metric: METRIC_RUNNING_QUERIES,
+        value: this.queries
+      })
+
+      if (peers.length === 0) {
+        log.error('Running query with no peers')
+        return
+      }
+
+      // Create query paths from the starting peers
+      const paths = peersToQuery.map((peer, index) => {
+        return queryPath({
+          key,
+          startingPeer: peer,
+          ourPeerId: this.peerId,
+          signal,
+          query: queryFunc,
+          pathIndex: index,
+          numPaths: peersToQuery.length,
+          alpha: this.alpha,
+          cleanUp,
+          queryFuncTimeout: options.queryFuncTimeout,
+          log
+        })
+      })
+
+      // Execute the query along each disjoint path and yield their results as they become available
+      for await (const event of merge(...paths)) {
+        yield event
+
+        if (event.name === 'QUERY_ERROR') {
+          log('error', event.error)
+        }
+      }
+    } catch (err: any) {
+      if (!this.running && err.code === 'ERR_QUERY_ABORTED') {
+        // ignore query aborted errors that were thrown during query manager shutdown
+      } else {
+        throw err
+      }
+    } finally {
+      this.controllers.delete(abortController)
+
+      if (timeoutController != null) {
+        timeoutController.clear()
+      }
+
+      this.queries--
+      this.metrics?.updateComponentMetric({
+        system: 'libp2p',
+        component: `kad-dht-${this.lan ? 'lan' : 'wan'}`,
+        metric: METRIC_RUNNING_QUERIES,
+        value: this.queries
+      })
+
+      cleanUp.dispatchEvent(new CustomEvent('cleanup'))
+      log('query:done in %dms', Date.now() - startTime)
+    }
+  }
+}

package/src/query/query-path.ts

@@ -0,0 +1,263 @@
+import Queue from 'p-queue'
+import { xor } from 'uint8arrays/xor'
+import { toString } from 'uint8arrays/to-string'
+import defer from 'p-defer'
+import errCode from 'err-code'
+import { convertPeerId, convertBuffer } from '../utils.js'
+import { TimeoutController } from 'timeout-abort-controller'
+import { anySignal } from 'any-signal'
+import { queryErrorEvent } from './events.js'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { EventEmitter } from '@libp2p/interfaces'
+import type { CleanUpEvents } from './manager.js'
+import type { Logger } from '@libp2p/logger'
+import type { QueryFunc } from '../query/types.js'
+import { base58btc } from 'multiformats/bases/base58'
+import type { QueryEvent } from '@libp2p/interfaces/dht'
+
+const MAX_XOR = BigInt('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
+
+export interface QueryPathOptions {
+  /**
+   * What are we trying to find
+   */
+  key: Uint8Array
+
+  /**
+   * Where we start our query
+   */
+  startingPeer: PeerId
+
+  /**
+   * Who we are
+   */
+  ourPeerId: PeerId
+
+  /**
+   * When to stop querying
+   */
+  signal: AbortSignal
+
+  /**
+   * The query function to run with each peer
+   */
+  query: QueryFunc
+
+  /**
+   * How many concurrent node/value lookups to run
+   */
+  alpha: number
+
+  /**
+   * How many concurrent node/value lookups to run
+   */
+  pathIndex: number
+
+  /**
+   * How many concurrent node/value lookups to run
+   */
+  numPaths: number
+
+  /**
+   * will emit a 'cleanup' event if the caller exits the for..await of early
+   */
+  cleanUp: EventEmitter<CleanUpEvents>
+
+  /**
+   * A timeout for queryFunc in ms
+   */
+  queryFuncTimeout?: number
+
+  /**
+   * Query log
+   */
+  log: Logger
+}
+
+/**
+ * Walks a path through the DHT, calling the passed query function for
+ * every peer encountered that we have not seen before
+ */
+export async function * queryPath (options: QueryPathOptions) {
+  const { key, startingPeer, ourPeerId, signal, query, alpha, pathIndex, numPaths, cleanUp, queryFuncTimeout, log } = options
+  // Only ALPHA node/value lookups are allowed at any given time for each process
+  // https://github.com/libp2p/specs/tree/master/kad-dht#alpha-concurrency-parameter-%CE%B1
+  const queue = new Queue({
+    concurrency: alpha
+  })
+
+  // perform lookups on kadId, not the actual value
+  const kadId = await convertBuffer(key)
+
+  // make sure we don't get trapped in a loop
+  const peersSeen = new Set()
+
+  /**
+   * Adds the passed peer to the query queue if it's not us and no
+   * other path has passed through this peer
+   */
+  function queryPeer (peer: PeerId, peerKadId: Uint8Array) {
+    if (peer == null) {
+      return
+    }
+
+    peersSeen.add(peer.toString(base58btc))
+
+    const peerXor = BigInt('0x' + toString(xor(peerKadId, kadId), 'base16'))
+
+    queue.add(async () => {
+      let timeout
+      const signals = [signal]
+
+      if (queryFuncTimeout != null) {
+        timeout = new TimeoutController(queryFuncTimeout)
+        signals.push(timeout.signal)
+      }
+
+      const compoundSignal = anySignal(signals)
+
+      try {
+        for await (const event of query({
+          key,
+          peer,
+          signal: compoundSignal,
+          pathIndex,
+          numPaths
+        })) {
+          if (compoundSignal.aborted) {
+            return
+          }
+
+          // if there are closer peers and the query has not completed, continue the query
+          if (event.name === 'PEER_RESPONSE') {
+            for (const closerPeer of event.closer) {
+              if (peersSeen.has(closerPeer.id.toString(base58btc))) { // eslint-disable-line max-depth
+                log('already seen %p in query', closerPeer.id)
+                continue
+              }
+
+              if (ourPeerId.equals(closerPeer.id)) { // eslint-disable-line max-depth
+                log('not querying ourselves')
+                continue
+              }
+
+              const closerPeerKadId = await convertPeerId(closerPeer.id)
+              const closerPeerXor = BigInt('0x' + toString(xor(closerPeerKadId, kadId), 'base16'))
+
+              // only continue query if closer peer is actually closer
+              if (closerPeerXor > peerXor) { // eslint-disable-line max-depth
+                log('skipping %p as they are not closer to %b than %p', closerPeer.id, key, peer)
+                continue
+              }
+
+              log('querying closer peer %p', closerPeer.id)
+              queryPeer(closerPeer.id, closerPeerKadId)
+            }
+          }
+
+          // TODO: we have upgraded to p-queue@7, this should no longer be necessary
+          queue.emit('completed', event)
+        }
+
+        timeout?.clear()
+      } catch (err: any) {
+        if (signal.aborted) {
+          // TODO: we have upgraded to p-queue@7, this should no longer be necessary
+          queue.emit('error', err)
+        } else {
+          // TODO: we have upgraded to p-queue@7, this should no longer be necessary
+          queue.emit('completed', queryErrorEvent({
+            from: peer,
+            error: err
+          }))
+        }
+      } finally {
+        timeout?.clear()
+      }
+    }, {
+      // use xor value as the queue priority - closer peers should execute first
+      // subtract it from MAX_XOR because higher priority values execute sooner
+
+      // @ts-expect-error this is supposed to be a Number but it's ok to use BigInts
+      // as long as all priorities are BigInts since we won't mix BigInts and Number
+      // values in arithmetic operations
+      priority: MAX_XOR - peerXor
+    }).catch(err => {
+      log.error(err)
+    })
+  }
+
+  // begin the query with the starting peer
+  queryPeer(startingPeer, await convertPeerId(startingPeer))
+
+  // yield results as they come in
+  yield * toGenerator(queue, signal, cleanUp, log)
+}
+
+async function * toGenerator (queue: Queue, signal: AbortSignal, cleanUp: EventEmitter<CleanUpEvents>, log: Logger) {
+  let deferred = defer()
+  let running = true
+  const results: QueryEvent[] = []
+
+  const cleanup = () => {
+    if (!running) {
+      return
+    }
+
+    log('clean up queue, results %d, queue size %d, pending tasks %d', results.length, queue.size, queue.pending)
+
+    running = false
+    queue.clear()
+    results.splice(0, results.length)
+  }
+
+  queue.on('completed', result => {
+    results.push(result)
+    deferred.resolve()
+  })
+  queue.on('error', err => {
+    log('queue error', err)
+    cleanup()
+    deferred.reject(err)
+  })
+  queue.on('idle', () => {
+    log('queue idle')
+    running = false
+    deferred.resolve()
+  })
+
+  // clear the queue and throw if the query is aborted
+  signal.addEventListener('abort', () => {
+    log('abort queue')
+    const wasRunning = running
+    cleanup()
+
+    if (wasRunning) {
+      deferred.reject(errCode(new Error('Query aborted'), 'ERR_QUERY_ABORTED'))
+    }
+  })
+
+  // the user broke out of the loop early, ensure we resolve the deferred result
+  // promise and clear the queue of any remaining jobs
+  cleanUp.addEventListener('cleanup', () => {
+    cleanup()
+    deferred.resolve()
+  })
+
+  while (running) { // eslint-disable-line no-unmodified-loop-condition
+    await deferred.promise
+    deferred = defer()
+
+    // yield all available results
+    while (results.length > 0) {
+      const result = results.shift()
+
+      if (result != null) {
+        yield result
+      }
+    }
+  }
+
+  // yield any remaining results
+  yield * results
+}

package/src/query/types.ts

@@ -0,0 +1,22 @@
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { QueryEvent } from '@libp2p/interfaces/dht'
+
+export interface QueryContext {
+  // the key we are looking up
+  key: Uint8Array
+  // the current peer being queried
+  peer: PeerId
+  // if this signal emits an 'abort' event, any long-lived processes or requests started as part of this query should be terminated
+  signal: AbortSignal
+  // which disjoint path we are following
+  pathIndex: number
+  // the total number of disjoint paths being executed
+  numPaths: number
+}
+
+/**
+ * Query function
+ */
+export interface QueryFunc {
+  (context: QueryContext): AsyncIterable<QueryEvent>
+}

package/src/query-self.ts

@@ -0,0 +1,106 @@
+// @ts-expect-error setMaxListeners is missing from the types
+import { setMaxListeners } from 'events'
+import take from 'it-take'
+import length from 'it-length'
+import { QUERY_SELF_INTERVAL, QUERY_SELF_TIMEOUT, K } from './constants.js'
+import { TimeoutController } from 'timeout-abort-controller'
+import { anySignal } from 'any-signal'
+import { logger, Logger } from '@libp2p/logger'
+import type { PeerRouting } from './peer-routing/index.js'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { Startable } from '@libp2p/interfaces'
+import { pipe } from 'it-pipe'
+
+export interface QuerySelfOptions {
+  peerId: PeerId
+  lan: boolean
+  peerRouting: PeerRouting
+  count?: number
+  interval?: number
+  queryTimeout?: number
+}
+
+/**
+ * Receives notifications of new peers joining the network that support the DHT protocol
+ */
+export class QuerySelf implements Startable {
+  private readonly log: Logger
+  private readonly peerId: PeerId
+  private readonly peerRouting: PeerRouting
+  private readonly count: number
+  private readonly interval: number
+  private readonly queryTimeout: number
+  private running: boolean
+  private timeoutId?: NodeJS.Timer
+  private controller?: AbortController
+
+  constructor (options: QuerySelfOptions) {
+    const { peerId, peerRouting, lan, count, interval, queryTimeout } = options
+
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:query-self`)
+    this.running = false
+    this.peerId = peerId
+    this.peerRouting = peerRouting
+    this.count = count ?? K
+    this.interval = interval ?? QUERY_SELF_INTERVAL
+    this.queryTimeout = queryTimeout ?? QUERY_SELF_TIMEOUT
+  }
+
+  isStarted () {
+    return this.running
+  }
+
+  async start () {
+    if (this.running) {
+      return
+    }
+
+    this.running = true
+    this._querySelf()
+  }
+
+  async stop () {
+    this.running = false
+
+    if (this.timeoutId != null) {
+      clearTimeout(this.timeoutId)
+    }
+
+    if (this.controller != null) {
+      this.controller.abort()
+    }
+  }
+
+  _querySelf () {
+    Promise.resolve().then(async () => {
+      const timeoutController = new TimeoutController(this.queryTimeout)
+
+      try {
+        this.controller = new AbortController()
+        const signal = anySignal([this.controller.signal, timeoutController.signal])
+        // this controller will get used for lots of dial attempts so make sure we don't cause warnings to be logged
+        try {
+          if (setMaxListeners != null) {
+            setMaxListeners(Infinity, signal)
+          }
+        } catch {} // fails on node < 15.4
+        const found = await pipe(
+          this.peerRouting.getClosestPeers(this.peerId.toBytes(), {
+            signal
+          }),
+          (source) => take(source, this.count),
+          async (source) => await length(source)
+        )
+
+        this.log('query ran successfully - found %d peers', found)
+      } catch (err: any) {
+        this.log('query error', err)
+      } finally {
+        this.timeoutId = setTimeout(this._querySelf.bind(this), this.interval)
+        timeoutController.clear()
+      }
+    }).catch(err => {
+      this.log('query error', err)
+    })
+  }
+}