@libp2p/kad-dht 15.0.1 → 15.0.2-185b23eac

package/src/peer-distance-list.ts

@@ -6,6 +6,7 @@ import type { PeerId, PeerInfo } from '@libp2p/interface'
 interface PeerDistance {
   peer: PeerInfo
   distance: Uint8Array
+  path: number
 }
 
 /**
@@ -40,30 +41,31 @@ export class PeerDistanceList {
   /**
    * The peers in the list, in order of distance from the origin key
    */
-  get peers (): PeerInfo[] {
-    return this.peerDistances.map(pd => pd.peer)
+  get peers (): PeerDistance[] {
+    return [...this.peerDistances]
   }
 
   /**
    * Add a peerId to the list.
    */
-  async add (peer: PeerInfo): Promise<void> {
+  async add (peer: PeerInfo, path: number = -1): Promise<void> {
     const dhtKey = await convertPeerId(peer.id)
 
-    this.addWithKadId(peer, dhtKey)
+    this.addWithKadId(peer, dhtKey, path)
   }
 
   /**
    * Add a peerId to the list.
    */
-  addWithKadId (peer: PeerInfo, kadId: Uint8Array): void {
+  addWithKadId (peer: PeerInfo, kadId: Uint8Array, path: number = -1): void {
     if (this.peerDistances.find(pd => pd.peer.id.equals(peer.id)) != null) {
       return
     }
 
-    const el = {
+    const el: PeerDistance = {
       peer,
-      distance: uint8ArrayXor(this.originDhtKey, kadId)
+      distance: uint8ArrayXor(this.originDhtKey, kadId),
+      path
     }
 
     let added = false

package/src/peer-routing/index.ts

@@ -18,7 +18,7 @@ import { verifyRecord } from '../record/validators.js'
 import { convertBuffer, convertPeerId, keyForPublicKey } from '../utils.js'
 import type { DHTRecord, FinalPeerEvent, QueryEvent, Validators } from '../index.js'
 import type { Message } from '../message/dht.js'
-import type { Network } from '../network.js'
+import type { Network, SendMessageOptions } from '../network.js'
 import type { QueryManager, QueryOptions } from '../query/manager.js'
 import type { QueryFunc } from '../query/types.js'
 import type { RoutingTable } from '../routing-table/index.js'
@@ -110,7 +110,7 @@ export class PeerRouting {
   /**
    * Get a value via rpc call for the given parameters
    */
-  async * _getValueSingle (peer: PeerId, key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
+  async * _getValueSingle (peer: PeerId, key: Uint8Array, options: SendMessageOptions): AsyncGenerator<QueryEvent> {
     const msg: Partial<Message> = {
       type: MessageType.GET_VALUE,
       key
@@ -125,7 +125,10 @@ export class PeerRouting {
   async * getPublicKeyFromNode (peer: PeerId, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
     const pkKey = keyForPublicKey(peer)
 
-    for await (const event of this._getValueSingle(peer, pkKey, options)) {
+    for await (const event of this._getValueSingle(peer, pkKey, {
+      ...options,
+      path: -1
+    })) {
       yield event
 
       if (event.name === 'PEER_RESPONSE' && event.record != null) {
@@ -166,7 +169,8 @@ export class PeerRouting {
         this.log('found local')
         yield finalPeerEvent({
           from: this.peerId,
-          peer: pi
+          peer: pi,
+          path: -1
         }, options)
         return
       }
@@ -177,7 +181,7 @@ export class PeerRouting {
     if (options.useNetwork !== false) {
       const self = this // eslint-disable-line @typescript-eslint/no-this-alias
 
-      const findPeerQuery: QueryFunc = async function * ({ peer, signal }) {
+      const findPeerQuery: QueryFunc = async function * ({ peer, signal, path }) {
         const request: Partial<Message> = {
           type: MessageType.FIND_NODE,
           key: id.toMultihash().bytes
@@ -185,7 +189,8 @@ export class PeerRouting {
 
         for await (const event of self.network.sendRequest(peer, request, {
           ...options,
-          signal
+          signal,
+          path
         })) {
           yield event
 
@@ -194,7 +199,11 @@ export class PeerRouting {
 
             // found the peer
             if (match != null) {
-              yield finalPeerEvent({ from: event.from, peer: match }, options)
+              yield finalPeerEvent({
+                from: event.from,
+                peer: match,
+                path: event.path
+              }, options)
             }
           }
         }
@@ -210,7 +219,10 @@ export class PeerRouting {
     }
 
     if (!foundPeer) {
-      yield queryErrorEvent({ from: this.peerId, error: new NotFoundError('Not found') }, options)
+      yield queryErrorEvent({
+        from: this.peerId,
+        error: new NotFoundError('Not found')
+      }, options)
     }
   }
 
@@ -225,9 +237,9 @@ export class PeerRouting {
     const self = this // eslint-disable-line @typescript-eslint/no-this-alias
 
     const peers = new PeerDistanceList(kadId, this.routingTable.kBucketSize)
-    await Promise.all(tablePeers.map(async peer => { await peers.add({ id: peer, multiaddrs: [] }) }))
+    await Promise.all(tablePeers.map(async peer => { await peers.add({ id: peer, multiaddrs: [] }, -1) }))
 
-    const getCloserPeersQuery: QueryFunc = async function * ({ peer, signal }) {
+    const getCloserPeersQuery: QueryFunc = async function * ({ peer, signal, path }) {
       self.log('closerPeersSingle %s from %p', uint8ArrayToString(key, 'base32'), peer)
       const request: Partial<Message> = {
         type: MessageType.FIND_NODE,
@@ -236,14 +248,15 @@ export class PeerRouting {
 
       yield * self.network.sendRequest(peer, request, {
         ...options,
-        signal
+        signal,
+        path
       })
     }
 
     for await (const event of this.queryManager.run(key, getCloserPeersQuery, options)) {
       if (event.name === 'PEER_RESPONSE') {
         await Promise.all(event.closer.map(async peerData => {
-          await peers.add(peerData)
+          await peers.add(peerData, event.path)
         }))
       }
 
@@ -252,10 +265,11 @@ export class PeerRouting {
 
     this.log('found %d peers close to %b', peers.length, key)
 
-    for (const peer of peers.peers) {
+    for (const { peer, path } of peers.peers) {
       yield finalPeerEvent({
         from: this.peerId,
-        peer
+        peer,
+        path
       }, options)
     }
   }
@@ -266,7 +280,7 @@ export class PeerRouting {
    *
    * Note: The peerStore is updated with new addresses found for the given peer.
    */
-  async * getValueOrPeers (peer: PeerId, key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
+  async * getValueOrPeers (peer: PeerId, key: Uint8Array, options: SendMessageOptions): AsyncGenerator<QueryEvent> {
     for await (const event of this._getValueSingle(peer, key, options)) {
       if (event.name === 'PEER_RESPONSE') {
         if (event.record != null) {
@@ -277,7 +291,11 @@ export class PeerRouting {
             const errMsg = 'invalid record received, discarded'
             this.log(errMsg)
 
-            yield queryErrorEvent({ from: event.from, error: new QueryError(errMsg) }, options)
+            yield queryErrorEvent({
+              from: event.from,
+              error: new QueryError(errMsg),
+              path: options.path
+            }, options)
             continue
           }
         }

package/src/providers.ts

@@ -1,7 +1,7 @@
 import { PeerMap } from '@libp2p/peer-collections'
 import * as varint from 'uint8-varint'
 import { parseProviderKey, readProviderTime, toProviderKey } from './utils.js'
-import type { ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface'
+import type { AbortOptions, ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface'
 import type { Datastore } from 'interface-datastore'
 import type { Mortice } from 'mortice'
 import type { CID } from 'multiformats'
@@ -37,7 +37,7 @@ export class Providers {
   /**
    * Add a new provider for the given CID
    */
-  async addProvider (cid: CID, provider: PeerId): Promise<void> {
+  async addProvider (cid: CID, provider: PeerId, options?: AbortOptions): Promise<void> {
     const release = await this.lock.readLock()
 
     try {
@@ -51,7 +51,7 @@ export class Providers {
   /**
    * Remove a provider for the given CID
    */
-  async removeProvider (cid: CID, provider: PeerId): Promise<void> {
+  async removeProvider (cid: CID, provider: PeerId, options?: AbortOptions): Promise<void> {
     const release = await this.lock.writeLock()
 
     try {
@@ -66,12 +66,12 @@ export class Providers {
   /**
    * Get a list of providers for the given CID
    */
-  async getProviders (cid: CID): Promise<PeerId[]> {
+  async getProviders (cid: CID, options?: AbortOptions): Promise<PeerId[]> {
     const release = await this.lock.readLock()
 
     try {
       this.log('get providers for %c', cid)
-      const provs = await this.loadProviders(cid)
+      const provs = await this.loadProviders(cid, options)
       this.log('got %d providers for %c', provs.size, cid)
 
       return [...provs.keys()]
@@ -83,21 +83,21 @@ export class Providers {
   /**
    * Write a provider into the given store
    */
-  private async writeProviderEntry (cid: CID, peerId: PeerId, time: Date = new Date()): Promise<void> {
+  private async writeProviderEntry (cid: CID, peerId: PeerId, time: Date = new Date(), options?: AbortOptions): Promise<void> {
     const key = toProviderKey(this.datastorePrefix, cid, peerId)
     const buffer = varint.encode(time.getTime())
 
-    await this.datastore.put(key, buffer)
+    await this.datastore.put(key, buffer, options)
   }
 
   /**
    * Load providers for the given CID from the store
    */
-  private async loadProviders (cid: CID): Promise<PeerMap<Date>> {
+  private async loadProviders (cid: CID, options?: AbortOptions): Promise<PeerMap<Date>> {
     const providers = new PeerMap<Date>()
     const key = toProviderKey(this.datastorePrefix, cid)
 
-    for await (const entry of this.datastore.query({ prefix: key.toString() })) {
+    for await (const entry of this.datastore.query({ prefix: key.toString() }, options)) {
       const { peerId } = parseProviderKey(entry.key)
       providers.set(peerId, readProviderTime(entry.value))
     }

package/src/query/events.ts

@@ -1,4 +1,4 @@
-import type { MessageType, SendQueryEvent, PeerResponseEvent, DialPeerEvent, AddPeerEvent, ValueEvent, ProviderEvent, QueryErrorEvent, FinalPeerEvent } from '../index.js'
+import type { MessageType, SendQueryEvent, PeerResponseEvent, AddPeerEvent, ValueEvent, ProviderEvent, QueryErrorEvent, FinalPeerEvent } from '../index.js'
 import type { PeerId, PeerInfo } from '@libp2p/interface'
 import type { Libp2pRecord } from '@libp2p/record'
 import type { ProgressOptions } from 'progress-events'
@@ -6,6 +6,7 @@ import type { ProgressOptions } from 'progress-events'
 export interface QueryEventFields {
   to: PeerId
   type: MessageType
+  path: number
 }
 
 export function sendQueryEvent (fields: QueryEventFields, options: ProgressOptions = {}): SendQueryEvent {
@@ -25,6 +26,7 @@ export function sendQueryEvent (fields: QueryEventFields, options: ProgressOptio
 export interface PeerResponseEventFields {
   from: PeerId
   messageType: MessageType
+  path: number
   closer?: PeerInfo[]
   providers?: PeerInfo[]
   record?: Libp2pRecord
@@ -48,6 +50,7 @@ export function peerResponseEvent (fields: PeerResponseEventFields, options: Pro
 export interface FinalPeerEventFields {
   from: PeerId
   peer: PeerInfo
+  path: number
 }
 
 export function finalPeerEvent (fields: FinalPeerEventFields, options: ProgressOptions = {}): FinalPeerEvent {
@@ -65,6 +68,7 @@ export function finalPeerEvent (fields: FinalPeerEventFields, options: ProgressO
 export interface ErrorEventFields {
   from: PeerId
   error: Error
+  path?: number
 }
 
 export function queryErrorEvent (fields: ErrorEventFields, options: ProgressOptions = {}): QueryErrorEvent {
@@ -113,11 +117,12 @@ export function valueEvent (fields: ValueEventFields, options: ProgressOptions =
   return event
 }
 
-export interface PeerEventFields {
+export interface AddPeerEventFields {
   peer: PeerId
+  path: number
 }
 
-export function addPeerEvent (fields: PeerEventFields, options: ProgressOptions = {}): AddPeerEvent {
+export function addPeerEvent (fields: AddPeerEventFields, options: ProgressOptions = {}): AddPeerEvent {
   const event: AddPeerEvent = {
     ...fields,
     name: 'ADD_PEER',
@@ -128,19 +133,3 @@ export function addPeerEvent (fields: PeerEventFields, options: ProgressOptions
 
   return event
 }
-
-export interface DialPeerEventFields {
-  peer: PeerId
-}
-
-export function dialPeerEvent (fields: DialPeerEventFields, options: ProgressOptions = {}): DialPeerEvent {
-  const event: DialPeerEvent = {
-    ...fields,
-    name: 'DIAL_PEER',
-    type: 7
-  }
-
-  options.onProgress?.(new CustomEvent('kad-dht:query:dial-peer', { detail: event }))
-
-  return event
-}

package/src/query/manager.ts

@@ -2,6 +2,7 @@ import { setMaxListeners } from '@libp2p/interface'
 import { PeerSet } from '@libp2p/peer-collections'
 import { anySignal } from 'any-signal'
 import merge from 'it-merge'
+import { pEvent } from 'p-event'
 import { raceSignal } from 'race-signal'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import {
@@ -26,6 +27,7 @@ export interface QueryManagerInit {
   disjointPaths?: number
   alpha?: number
   initialQuerySelfHasRun: DeferredPromise<void>
+  allowQueryWithZeroPeers?: boolean
   routingTable: RoutingTable
 }
 
@@ -59,6 +61,7 @@ export class QueryManager implements Startable {
   private readonly routingTable: RoutingTable
   private initialQuerySelfHasRun?: DeferredPromise<void>
   private readonly logPrefix: string
+  private readonly allowQueryWithZeroPeers: boolean
 
   constructor (components: QueryManagerComponents, init: QueryManagerInit) {
     this.logPrefix = init.logPrefix
@@ -69,6 +72,7 @@ export class QueryManager implements Startable {
     this.logger = components.logger
     this.peerId = components.peerId
     this.connectionManager = components.connectionManager
+    this.allowQueryWithZeroPeers = init.allowQueryWithZeroPeers ?? false
 
     // allow us to stop queries on shut down
     this.shutDownController = new AbortController()
@@ -146,6 +150,16 @@ export class QueryManager implements Startable {
     let queryFinished = false
 
     try {
+      if (this.routingTable.size === 0 && !this.allowQueryWithZeroPeers) {
+        log('routing table was empty, waiting for some peers before running%s query', options.isSelfQuery === true ? ' self' : '')
+        // wait to discover at least one DHT peer that isn't us
+        await pEvent(this.routingTable, 'peer:add', {
+          signal,
+          filter: (event) => !this.peerId.equals(event.detail)
+        })
+        log('routing table has peers, continuing with%s query', options.isSelfQuery === true ? ' self' : '')
+      }
+
       if (options.isSelfQuery !== true && this.initialQuerySelfHasRun != null) {
         log('waiting for initial query-self query before continuing')
 
@@ -161,7 +175,7 @@ export class QueryManager implements Startable {
       const peersToQuery = peers.slice(0, Math.min(this.disjointPaths, peers.length))
 
       if (peers.length === 0) {
-        log.error('Running query with no peers')
+        log.error('running query with no peers')
         return
       }
 
@@ -177,7 +191,7 @@ export class QueryManager implements Startable {
           ourPeerId: this.peerId,
           signal,
           query: queryFunc,
-          pathIndex: index,
+          path: index,
           numPaths: peersToQuery.length,
           alpha: this.alpha,
           queryFuncTimeout: options.queryFuncTimeout,
@@ -197,22 +211,26 @@ export class QueryManager implements Startable {
         if (event.name === 'PEER_RESPONSE') {
           for (const peer of [...event.closer, ...event.providers]) {
             // eslint-disable-next-line max-depth
-            if (!(await this.connectionManager.isDialable(peer.multiaddrs))) {
+            if (!(await this.connectionManager.isDialable(peer.multiaddrs, {
+              signal
+            }))) {
               continue
             }
 
-            await this.routingTable.add(peer.id)
+            await this.routingTable.add(peer.id, {
+              signal
+            })
           }
         }
 
         yield event
+        signal.throwIfAborted()
       }
 
       queryFinished = true
-    } catch (err: any) {
-      if (!this.running && err.name === 'QueryAbortedError') {
-        // ignore query aborted errors that were thrown during query manager shutdown
-      } else {
+    } catch (err) {
+      if (this.running) {
+        // ignore errors thrown during shut down
         throw err
       }
     } finally {

package/src/query/query-path.ts

@@ -3,7 +3,6 @@ import { Queue } from '@libp2p/utils/queue'
 import { anySignal } from 'any-signal'
 import { xor as uint8ArrayXor } from 'uint8arrays/xor'
 import { xorCompare as uint8ArrayXorCompare } from 'uint8arrays/xor-compare'
-import { QueryAbortedError } from '../errors.js'
 import { convertPeerId, convertBuffer } from '../utils.js'
 import { queryErrorEvent } from './events.js'
 import type { QueryEvent } from '../index.js'
@@ -44,12 +43,12 @@ export interface QueryPathOptions extends RoutingOptions {
   alpha: number
 
   /**
-   * How many concurrent node/value lookups to run
+   * The index within `k` this path represents
    */
-  pathIndex: number
+  path: number
 
   /**
-   * How many concurrent node/value lookups to run
+   * How many disjoint paths are in this query
    */
   numPaths: number
 
@@ -83,7 +82,7 @@ interface QueryQueueOptions extends AbortOptions {
  * every peer encountered that we have not seen before
  */
 export async function * queryPath (options: QueryPathOptions): AsyncGenerator<QueryEvent, void, undefined> {
-  const { key, startingPeer, ourPeerId, signal, query, alpha, pathIndex, numPaths, queryFuncTimeout, log, peersSeen, connectionManager } = options
+  const { key, startingPeer, ourPeerId, signal, query, alpha, path, numPaths, queryFuncTimeout, log, peersSeen, connectionManager } = options
   // Only ALPHA node/value lookups are allowed at any given time for each process
   // https://github.com/libp2p/specs/tree/master/kad-dht#alpha-concurrency-parameter-%CE%B1
   const queue = new Queue<QueryEvent | undefined, QueryQueueOptions>({
@@ -125,7 +124,7 @@ export async function * queryPath (options: QueryPathOptions): AsyncGenerator<Qu
           key,
           peer,
           signal: compoundSignal,
-          pathIndex,
+          path,
           numPaths
         })) {
           if (compoundSignal.aborted) {
@@ -172,7 +171,8 @@ export async function * queryPath (options: QueryPathOptions): AsyncGenerator<Qu
         if (!signal.aborted) {
           return queryErrorEvent({
             from: peer,
-            error: err
+            error: err,
+            path
           }, options)
         }
       } finally {
@@ -188,18 +188,10 @@ export async function * queryPath (options: QueryPathOptions): AsyncGenerator<Qu
   // begin the query with the starting peer
   queryPeer(startingPeer, await convertPeerId(startingPeer))
 
-  try {
-    // yield results as they come in
-    for await (const event of queue.toGenerator({ signal })) {
-      if (event != null) {
-        yield event
-      }
+  // yield results as they come in
+  for await (const event of queue.toGenerator({ signal })) {
+    if (event != null) {
+      yield event
     }
-  } catch (err) {
-    if (signal.aborted) {
-      throw new QueryAbortedError('Query aborted')
-    }
-
-    throw err
   }
 }

package/src/query/types.ts

@@ -9,7 +9,7 @@ export interface QueryContext {
   // if this signal emits an 'abort' event, any long-lived processes or requests started as part of this query should be terminated
   signal: AbortSignal
   // which disjoint path we are following
-  pathIndex: number
+  path: number
   // the total number of disjoint paths being executed
   numPaths: number
 }

package/src/query-self.ts

@@ -4,18 +4,16 @@ import length from 'it-length'
 import { pipe } from 'it-pipe'
 import take from 'it-take'
 import pDefer from 'p-defer'
-import { pEvent } from 'p-event'
 import { QUERY_SELF_INTERVAL, QUERY_SELF_TIMEOUT, K, QUERY_SELF_INITIAL_INTERVAL } from './constants.js'
 import { timeOperationMethod } from './utils.js'
 import type { OperationMetrics } from './kad-dht.js'
 import type { PeerRouting } from './peer-routing/index.js'
-import type { RoutingTable } from './routing-table/index.js'
 import type { ComponentLogger, Logger, Metrics, PeerId, Startable } from '@libp2p/interface'
 import type { DeferredPromise } from 'p-defer'
+
 export interface QuerySelfInit {
   logPrefix: string
   peerRouting: PeerRouting
-  routingTable: RoutingTable
   count?: number
   interval?: number
   initialInterval?: number
@@ -28,6 +26,7 @@ export interface QuerySelfComponents {
   peerId: PeerId
   logger: ComponentLogger
   metrics?: Metrics
+  events: EventTarget
 }
 
 /**
@@ -37,7 +36,7 @@ export class QuerySelf implements Startable {
   private readonly log: Logger
   private readonly peerId: PeerId
   private readonly peerRouting: PeerRouting
-  private readonly routingTable: RoutingTable
+  private readonly events: EventTarget
   private readonly count: number
   private readonly interval: number
   private readonly initialInterval: number
@@ -51,9 +50,9 @@ export class QuerySelf implements Startable {
   constructor (components: QuerySelfComponents, init: QuerySelfInit) {
     this.peerId = components.peerId
     this.log = components.logger.forComponent(`${init.logPrefix}:query-self`)
+    this.events = components.events
     this.running = false
     this.peerRouting = init.peerRouting
-    this.routingTable = init.routingTable
     this.count = init.count ?? K
     this.interval = init.interval ?? QUERY_SELF_INTERVAL
     this.initialInterval = init.initialInterval ?? QUERY_SELF_INITIAL_INTERVAL
@@ -122,20 +121,10 @@ export class QuerySelf implements Startable {
       setMaxListeners(Infinity, signal, this.controller.signal)
 
       try {
-        if (this.routingTable.size === 0) {
-          this.log('routing table was empty, waiting for some peers before running query')
-          // wait to discover at least one DHT peer that isn't us
-          await pEvent(this.routingTable, 'peer:add', {
-            signal,
-            filter: (event) => !this.peerId.equals(event.detail)
-          })
-          this.log('routing table has peers, continuing with query')
-        }
-
         this.log('run self-query, look for %d peers timing out after %dms', this.count, this.queryTimeout)
         const start = Date.now()
 
-        const found = await pipe(
+        const peers = await pipe(
           this.peerRouting.getClosestPeers(this.peerId.toMultihash().bytes, {
             signal,
             isSelfQuery: true
@@ -144,7 +133,16 @@ export class QuerySelf implements Startable {
           async (source) => length(source)
         )
 
-        this.log('self-query found %d peers in %dms', found, Date.now() - start)
+        const duration = Date.now() - start
+
+        this.log('self-query found %d peers in %dms', peers, duration)
+
+        this.events.dispatchEvent(new CustomEvent('kad-dht:query:self', {
+          detail: {
+            peers,
+            duration
+          }
+        }))
       } catch (err: any) {
         this.log.error('self-query error', err)
       } finally {

package/src/routing-table/closest-peers.ts

@@ -91,7 +91,7 @@ export class ClosestPeers implements Startable {
   }
 
   async updatePeerTags (): Promise<void> {
-    const newClosest = new PeerSet(this.newPeers?.peers.map(peer => peer.id))
+    const newClosest = new PeerSet(this.newPeers?.peers.map(({ peer }) => peer.id))
     const added = newClosest.difference(this.closestPeers)
     const removed = this.closestPeers.difference(newClosest)
     this.closestPeers = newClosest

package/src/routing-table/k-bucket.ts

@@ -300,7 +300,7 @@ export class KBucket {
       list.addWithKadId({ id: peer.peerId, multiaddrs: [] }, peer.kadId)
     }
 
-    yield * map(list.peers, info => info.id)
+    yield * map(list.peers, ({ peer }) => peer.id)
   }
 
   /**