@libp2p/kad-dht 15.0.2 → 15.1.0

package/src/query/query-path.ts

@@ -1,16 +1,15 @@
-import { setMaxListeners } from '@libp2p/interface'
 import { Queue } from '@libp2p/utils/queue'
-import { anySignal } from 'any-signal'
+import { pushable } from 'it-pushable'
 import { xor as uint8ArrayXor } from 'uint8arrays/xor'
 import { xorCompare as uint8ArrayXorCompare } from 'uint8arrays/xor-compare'
 import { QueryAbortedError } from '../errors.js'
 import { convertPeerId, convertBuffer } from '../utils.js'
-import { queryErrorEvent } from './events.js'
+import { pathEndedEvent, queryErrorEvent } from './events.js'
 import type { QueryEvent } from '../index.js'
 import type { QueryFunc } from '../query/types.js'
-import type { Logger, PeerId, RoutingOptions, AbortOptions } from '@libp2p/interface'
+import type { Logger, PeerId, RoutingOptions, AbortOptions, PeerInfo } from '@libp2p/interface'
 import type { ConnectionManager } from '@libp2p/interface-internal'
-import type { PeerSet } from '@libp2p/peer-collections'
+import type { Filter } from '@libp2p/utils/filters'
 
 export interface QueryPathOptions extends RoutingOptions {
   /**
@@ -21,18 +20,13 @@ export interface QueryPathOptions extends RoutingOptions {
   /**
    * Where we start our query
    */
-  startingPeer: PeerId
+  startingPeers: PeerId[]
 
   /**
    * Who we are
    */
   ourPeerId: PeerId
 
-  /**
-   * When to stop querying
-   */
-  signal: AbortSignal
-
   /**
    * The query function to run with each peer
    */
@@ -44,20 +38,15 @@ export interface QueryPathOptions extends RoutingOptions {
   alpha: number
 
   /**
-   * How many concurrent node/value lookups to run
+   * The index within `k` this path represents
    */
-  pathIndex: number
+  path: number
 
   /**
-   * How many concurrent node/value lookups to run
+   * How many disjoint paths are in this query
    */
   numPaths: number
 
-  /**
-   * A timeout for queryFunc in ms
-   */
-  queryFuncTimeout?: number
-
   /**
    * Query log
    */
@@ -66,12 +55,17 @@ export interface QueryPathOptions extends RoutingOptions {
   /**
    * Set of peers seen by this and other paths
    */
-  peersSeen: PeerSet
+  peersSeen: Filter
 
   /**
    * The libp2p connection manager
    */
   connectionManager: ConnectionManager
+
+  /**
+   * The overall query abort signal
+   */
+  signal: AbortSignal
 }
 
 interface QueryQueueOptions extends AbortOptions {
@@ -83,60 +77,75 @@ interface QueryQueueOptions extends AbortOptions {
  * every peer encountered that we have not seen before
  */
 export async function * queryPath (options: QueryPathOptions): AsyncGenerator<QueryEvent, void, undefined> {
-  const { key, startingPeer, ourPeerId, signal, query, alpha, pathIndex, numPaths, queryFuncTimeout, log, peersSeen, connectionManager } = options
+  const { key, startingPeers, ourPeerId, query, alpha, path, numPaths, log, peersSeen, connectionManager, signal } = options
+  const events = pushable<QueryEvent>({
+    objectMode: true
+  })
+
   // Only ALPHA node/value lookups are allowed at any given time for each process
   // https://github.com/libp2p/specs/tree/master/kad-dht#alpha-concurrency-parameter-%CE%B1
-  const queue = new Queue<QueryEvent | undefined, QueryQueueOptions>({
+  const queue = new Queue<undefined, QueryQueueOptions>({
     concurrency: alpha,
     sort: (a, b) => uint8ArrayXorCompare(a.options.distance, b.options.distance)
   })
+  queue.addEventListener('idle', () => {
+    events.push(pathEndedEvent({
+      path: {
+        index: path,
+        queued: queue.queued,
+        running: queue.running,
+        total: queue.size
+      }
+    }, options))
+
+    events.end()
+  })
+  queue.addEventListener('error', (evt) => {
+    log.error('error during query - %e', evt.detail)
+  })
+
+  signal.addEventListener('abort', () => {
+    queue.abort()
+    events.end(new QueryAbortedError())
+  })
 
   // perform lookups on kadId, not the actual value
   const kadId = await convertBuffer(key)
 
   /**
-   * Adds the passed peer to the query queue if it's not us and no
-   * other path has passed through this peer
+   * Adds the passed peer to the query queue if it's not us and no other path
+   * has passed through this peer
    */
-  function queryPeer (peer: PeerId, peerKadId: Uint8Array): void {
+  function queryPeer (peer: PeerInfo, peerKadId: Uint8Array): void {
     if (peer == null) {
       return
     }
 
-    peersSeen.add(peer)
+    peersSeen.add(peer.id.toMultihash().bytes)
 
     const peerXor = uint8ArrayXor(peerKadId, kadId)
 
     queue.add(async () => {
-      const signals = [signal]
-
-      if (queryFuncTimeout != null) {
-        signals.push(AbortSignal.timeout(queryFuncTimeout))
-      }
-
-      const compoundSignal = anySignal(signals)
-
-      // this signal can get listened to a lot
-      setMaxListeners(Infinity, compoundSignal)
-
       try {
         for await (const event of query({
           ...options,
           key,
           peer,
-          signal: compoundSignal,
-          pathIndex,
-          numPaths
+          path: {
+            index: path,
+            queued: queue.queued,
+            running: queue.running,
+            total: queue.size
+          },
+          numPaths,
+          peerKadId,
+          signal
         })) {
-          if (compoundSignal.aborted) {
-            return
-          }
-
           // if there are closer peers and the query has not completed, continue the query
           if (event.name === 'PEER_RESPONSE') {
             for (const closerPeer of event.closer) {
-              if (peersSeen.has(closerPeer.id)) { // eslint-disable-line max-depth
-                log.trace('already seen %p in query', closerPeer.id)
+              if (peersSeen.has(closerPeer.id.toMultihash().bytes)) { // eslint-disable-line max-depth
+                log('already seen %p in query', closerPeer.id)
                 continue
               }
 
@@ -155,51 +164,52 @@ export async function * queryPath (options: QueryPathOptions): AsyncGenerator<Qu
 
               // only continue query if closer peer is actually closer
               if (uint8ArrayXorCompare(closerPeerXor, peerXor) !== -1) { // eslint-disable-line max-depth
-                log.trace('skipping %p as they are not closer to %b than %p', closerPeer.id, key, peer)
+                log('skipping %p as they are not closer to %b than %p', closerPeer.id, key, peer)
                 continue
               }
 
-              log.trace('querying closer peer %p', closerPeer.id)
-              queryPeer(closerPeer.id, closerPeerKadId)
+              log('querying closer peer %p', closerPeer.id)
+              queryPeer(closerPeer, closerPeerKadId)
             }
           }
 
-          queue.safeDispatchEvent('completed', {
-            detail: event
+          events.push({
+            ...event,
+            path: {
+              index: path,
+              queued: queue.queued,
+              running: queue.running,
+              total: queue.size
+            }
           })
         }
       } catch (err: any) {
-        if (!signal.aborted) {
-          return queryErrorEvent({
-            from: peer,
-            error: err
-          }, options)
-        }
-      } finally {
-        compoundSignal.clear()
+        // yield error event if query is continuing
+        events.push(queryErrorEvent({
+          from: peer.id,
+          error: err,
+          path: {
+            index: path,
+            queued: queue.queued,
+            running: queue.running - 1,
+            total: queue.size - 1
+          }
+        }, options))
       }
     }, {
       distance: peerXor
     }).catch(err => {
-      log.error(err)
+      log.error('error during query - %e', err)
     })
   }
 
-  // begin the query with the starting peer
-  queryPeer(startingPeer, await convertPeerId(startingPeer))
-
-  try {
-    // yield results as they come in
-    for await (const event of queue.toGenerator({ signal })) {
-      if (event != null) {
-        yield event
-      }
-    }
-  } catch (err) {
-    if (signal.aborted) {
-      throw new QueryAbortedError('Query aborted')
-    }
+  // begin the query with the starting peers
+  await Promise.all(
+    startingPeers.map(async startingPeer => {
+      queryPeer({ id: startingPeer, multiaddrs: [] }, await convertPeerId(startingPeer))
+    })
+  )
 
-    throw err
-  }
+  // yield results as they come in
+  yield * events
 }

package/src/query/types.ts

@@ -1,15 +1,17 @@
-import type { QueryEvent } from '../index.js'
-import type { PeerId } from '@libp2p/interface'
+import type { DisjointPath, QueryEvent } from '../index.js'
+import type { PeerInfo } from '@libp2p/interface'
 
 export interface QueryContext {
   // the key we are looking up
   key: Uint8Array
   // the current peer being queried
-  peer: PeerId
+  peer: PeerInfo
+  // the KAD ID of the peer being queried
+  peerKadId: Uint8Array
   // if this signal emits an 'abort' event, any long-lived processes or requests started as part of this query should be terminated
-  signal: AbortSignal
+  signal?: AbortSignal
   // which disjoint path we are following
-  pathIndex: number
+  path: DisjointPath
   // the total number of disjoint paths being executed
   numPaths: number
 }

package/src/query-self.ts

@@ -4,18 +4,16 @@ import length from 'it-length'
 import { pipe } from 'it-pipe'
 import take from 'it-take'
 import pDefer from 'p-defer'
-import { pEvent } from 'p-event'
 import { QUERY_SELF_INTERVAL, QUERY_SELF_TIMEOUT, K, QUERY_SELF_INITIAL_INTERVAL } from './constants.js'
 import { timeOperationMethod } from './utils.js'
 import type { OperationMetrics } from './kad-dht.js'
 import type { PeerRouting } from './peer-routing/index.js'
-import type { RoutingTable } from './routing-table/index.js'
 import type { ComponentLogger, Logger, Metrics, PeerId, Startable } from '@libp2p/interface'
 import type { DeferredPromise } from 'p-defer'
+
 export interface QuerySelfInit {
   logPrefix: string
   peerRouting: PeerRouting
-  routingTable: RoutingTable
   count?: number
   interval?: number
   initialInterval?: number
@@ -28,6 +26,7 @@ export interface QuerySelfComponents {
   peerId: PeerId
   logger: ComponentLogger
   metrics?: Metrics
+  events: EventTarget
 }
 
 /**
@@ -37,7 +36,7 @@ export class QuerySelf implements Startable {
   private readonly log: Logger
   private readonly peerId: PeerId
   private readonly peerRouting: PeerRouting
-  private readonly routingTable: RoutingTable
+  private readonly events: EventTarget
   private readonly count: number
   private readonly interval: number
   private readonly initialInterval: number
@@ -51,9 +50,9 @@ export class QuerySelf implements Startable {
   constructor (components: QuerySelfComponents, init: QuerySelfInit) {
     this.peerId = components.peerId
     this.log = components.logger.forComponent(`${init.logPrefix}:query-self`)
+    this.events = components.events
     this.running = false
     this.peerRouting = init.peerRouting
-    this.routingTable = init.routingTable
     this.count = init.count ?? K
     this.interval = init.interval ?? QUERY_SELF_INTERVAL
     this.initialInterval = init.initialInterval ?? QUERY_SELF_INITIAL_INTERVAL
@@ -122,20 +121,10 @@ export class QuerySelf implements Startable {
       setMaxListeners(Infinity, signal, this.controller.signal)
 
       try {
-        if (this.routingTable.size === 0) {
-          this.log('routing table was empty, waiting for some peers before running query')
-          // wait to discover at least one DHT peer that isn't us
-          await pEvent(this.routingTable, 'peer:add', {
-            signal,
-            filter: (event) => !this.peerId.equals(event.detail)
-          })
-          this.log('routing table has peers, continuing with query')
-        }
-
         this.log('run self-query, look for %d peers timing out after %dms', this.count, this.queryTimeout)
         const start = Date.now()
 
-        const found = await pipe(
+        const peers = await pipe(
           this.peerRouting.getClosestPeers(this.peerId.toMultihash().bytes, {
             signal,
             isSelfQuery: true
@@ -144,7 +133,16 @@ export class QuerySelf implements Startable {
           async (source) => length(source)
         )
 
-        this.log('self-query found %d peers in %dms', found, Date.now() - start)
+        const duration = Date.now() - start
+
+        this.log('self-query found %d peers in %dms', peers, duration)
+
+        this.events.dispatchEvent(new CustomEvent('kad-dht:query:self', {
+          detail: {
+            peers,
+            duration
+          }
+        }))
       } catch (err: any) {
         this.log.error('self-query error', err)
       } finally {

package/src/routing-table/closest-peers.ts

@@ -91,7 +91,7 @@ export class ClosestPeers implements Startable {
   }
 
   async updatePeerTags (): Promise<void> {
-    const newClosest = new PeerSet(this.newPeers?.peers.map(peer => peer.id))
+    const newClosest = new PeerSet(this.newPeers?.peers.map(({ peer }) => peer.id))
     const added = newClosest.difference(this.closestPeers)
     const removed = this.closestPeers.difference(newClosest)
     this.closestPeers = newClosest

package/src/routing-table/k-bucket.ts

@@ -300,7 +300,7 @@ export class KBucket {
       list.addWithKadId({ id: peer.peerId, multiaddrs: [] }, peer.kadId)
     }
 
-    yield * map(list.peers, info => info.id)
+    yield * map(list.peers, ({ peer }) => peer.id)
   }
 
   /**

package/src/rpc/handlers/get-providers.ts

@@ -67,7 +67,7 @@ export class GetProvidersHandler implements DHTMessageHandler {
 
         return info
       })),
-      this.peerRouting.getCloserPeersOffline(msg.key, this.peerId)
+      this.peerRouting.getCloserPeersOffline(msg.key, peerId)
     ])
 
     const response: Message = {

package/src/utils.ts

@@ -140,6 +140,15 @@ export function fromPublicKeyKey (key: Uint8Array): PeerId {
   return peerIdFromMultihash(multihash)
 }
 
+export function uint8ArrayToBigInt (buf: Uint8Array): bigint {
+  return BigInt(
+    `0x${
+      Array.from(buf)
+        .map(val => val.toString(16).padStart(2, '0')).join('')
+    }`
+  )
+}
+
 /**
  * Create a new put record, encodes and signs it if enabled
  */