@libp2p/kad-dht 14.0.2 → 14.1.0-a5cd8cfbe

package/src/kad-dht.ts

@@ -1,5 +1,6 @@
 import { NotFoundError, TypedEventEmitter, contentRoutingSymbol, peerDiscoverySymbol, peerRoutingSymbol, serviceCapabilities, serviceDependencies, start, stop } from '@libp2p/interface'
 import drain from 'it-drain'
+import createMortice from 'mortice'
 import pDefer from 'p-defer'
 import { PROTOCOL } from './constants.js'
 import { ContentFetching } from './content-fetching/index.js'
@@ -11,16 +12,18 @@ import { QueryManager } from './query/manager.js'
 import { QuerySelf } from './query-self.js'
 import { selectors as recordSelectors } from './record/selectors.js'
 import { validators as recordValidators } from './record/validators.js'
+import { Reprovider } from './reprovider.js'
 import { RoutingTable } from './routing-table/index.js'
 import { RoutingTableRefresh } from './routing-table/refresh.js'
 import { RPC } from './rpc/index.js'
 import { TopologyListener } from './topology-listener.js'
 import {
   multiaddrIsPublic,
-  removePrivateAddressesMapper
+  removePrivateAddressesMapper,
+  timeOperationGenerator
 } from './utils.js'
 import type { KadDHTComponents, KadDHTInit, Validators, Selectors, KadDHT as KadDHTInterface, QueryEvent, PeerInfoMapper } from './index.js'
-import type { ContentRouting, Logger, PeerDiscovery, PeerDiscoveryEvents, PeerId, PeerInfo, PeerRouting, RoutingOptions, Startable } from '@libp2p/interface'
+import type { ContentRouting, CounterGroup, Logger, MetricGroup, PeerDiscovery, PeerDiscoveryEvents, PeerId, PeerInfo, PeerRouting, RoutingOptions, Startable } from '@libp2p/interface'
 import type { CID } from 'multiformats/cid'
 
 /**
@@ -37,6 +40,10 @@ class DHTContentRouting implements ContentRouting {
     await drain(this.dht.provide(cid, options))
   }
 
+  async cancelReprovide (key: CID): Promise<void> {
+    await this.dht.cancelReprovide(key)
+  }
+
   async * findProviders (cid: CID, options: RoutingOptions = {}): AsyncGenerator<PeerInfo, void, undefined> {
     for await (const event of this.dht.findProviders(cid, options)) {
       if (event.name === 'PROVIDER') {
@@ -92,6 +99,15 @@ class DHTPeerRouting implements PeerRouting {
 export const DEFAULT_MAX_INBOUND_STREAMS = 32
 export const DEFAULT_MAX_OUTBOUND_STREAMS = 64
 
+export type Operation = 'GET_VALUE' | 'FIND_PROVIDERS' | 'FIND_PEER' | 'GET_CLOSEST_PEERS' | 'PROVIDE' | 'PUT_VALUE' | 'SELF_QUERY'
+
+export interface OperationMetrics {
+  queries?: MetricGroup<Operation>
+  errors?: CounterGroup<Operation>
+  queryTime?: MetricGroup<Operation>
+  errorTime?: MetricGroup<Operation>
+}
+
 /**
  * A DHT implementation modelled after Kademlia with S/Kademlia modifications.
  * Original implementation in go: https://github.com/libp2p/go-libp2p-kad-dht.
@@ -123,6 +139,7 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
   private readonly dhtContentRouting: DHTContentRouting
   private readonly dhtPeerRouting: DHTPeerRouting
   private readonly peerInfoMapper: PeerInfoMapper
+  private readonly reprovider: Reprovider
 
   /**
    * Create a new KadDHT
@@ -130,48 +147,52 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
   constructor (components: KadDHTComponents, init: KadDHTInit = {}) {
     super()
 
-    const {
-      kBucketSize,
-      clientMode,
-      validators,
-      selectors,
-      querySelfInterval,
-      protocol,
-      logPrefix,
-      maxInboundStreams,
-      maxOutboundStreams,
-      providers: providersInit
-    } = init
+    const logPrefix = init.logPrefix ?? 'libp2p:kad-dht'
+    const datastorePrefix = init.datastorePrefix ?? '/dht'
+    const metricsPrefix = init.metricsPrefix ?? 'libp2p_kad_dht'
 
-    const loggingPrefix = logPrefix ?? 'libp2p:kad-dht'
+    const operationMetrics: OperationMetrics = {
+      queries: components.metrics?.registerMetricGroup(`${metricsPrefix}_operations_total`, { label: 'operation' }),
+      errors: components.metrics?.registerCounterGroup(`${metricsPrefix}_operation_errors_total`, { label: 'operation' }),
+      queryTime: components.metrics?.registerMetricGroup(`${metricsPrefix}_operation_time_seconds`, { label: 'operation' }),
+      errorTime: components.metrics?.registerMetricGroup(`${metricsPrefix}_operation_error_time_seconds`, { label: 'operation' })
+    }
 
     this.running = false
     this.components = components
-    this.log = components.logger.forComponent(loggingPrefix)
-    this.protocol = protocol ?? PROTOCOL
-    this.kBucketSize = kBucketSize ?? 20
-    this.clientMode = clientMode ?? true
-    this.maxInboundStreams = maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS
-    this.maxOutboundStreams = maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS
+    this.log = components.logger.forComponent(logPrefix)
+    this.protocol = init.protocol ?? PROTOCOL
+    this.kBucketSize = init.kBucketSize ?? 20
+    this.clientMode = init.clientMode ?? true
+    this.maxInboundStreams = init.maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS
+    this.maxOutboundStreams = init.maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS
     this.peerInfoMapper = init.peerInfoMapper ?? removePrivateAddressesMapper
 
-    this.providers = new Providers(components, providersInit ?? {})
+    const providerLock = createMortice()
+
+    this.providers = new Providers(components, {
+      ...init.providers,
+      logPrefix,
+      datastorePrefix,
+      lock: providerLock
+    })
 
     this.validators = {
       ...recordValidators,
-      ...validators
+      ...init.validators
     }
     this.selectors = {
       ...recordSelectors,
-      ...selectors
+      ...init.selectors
     }
     this.network = new Network(components, {
       protocol: this.protocol,
-      logPrefix: loggingPrefix
+      logPrefix,
+      metricsPrefix
     })
 
     this.routingTable = new RoutingTable(components, {
-      kBucketSize,
+      kBucketSize: init.kBucketSize,
       pingOldContactTimeout: init.pingOldContactTimeout,
       pingOldContactConcurrency: init.pingOldContactConcurrency,
       pingOldContactMaxQueueSize: init.pingOldContactMaxQueueSize,
@@ -179,7 +200,8 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       pingNewContactConcurrency: init.pingNewContactConcurrency,
       pingNewContactMaxQueueSize: init.pingNewContactMaxQueueSize,
       protocol: this.protocol,
-      logPrefix: loggingPrefix,
+      logPrefix,
+      metricsPrefix,
       prefixLength: init.prefixLength,
       splitThreshold: init.kBucketSplitThreshold,
       network: this.network
@@ -198,7 +220,8 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
     this.queryManager = new QueryManager(components, {
       // Number of disjoint query paths to use - This is set to `kBucketSize/2` per the S/Kademlia paper
       disjointPaths: Math.ceil(this.kBucketSize / 2),
-      logPrefix: loggingPrefix,
+      logPrefix,
+      metricsPrefix,
       initialQuerySelfHasRun,
       routingTable: this.routingTable
     })
@@ -209,7 +232,7 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       network: this.network,
       validators: this.validators,
       queryManager: this.queryManager,
-      logPrefix: loggingPrefix
+      logPrefix
     })
     this.contentFetching = new ContentFetching(components, {
       validators: this.validators,
@@ -217,7 +240,7 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       peerRouting: this.peerRouting,
       queryManager: this.queryManager,
       network: this.network,
-      logPrefix: loggingPrefix
+      logPrefix
     })
     this.contentRouting = new KADDHTContentRouting(components, {
       network: this.network,
@@ -225,32 +248,43 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
       queryManager: this.queryManager,
       routingTable: this.routingTable,
       providers: this.providers,
-      logPrefix: loggingPrefix
+      logPrefix
     })
     this.routingTableRefresh = new RoutingTableRefresh(components, {
       peerRouting: this.peerRouting,
       routingTable: this.routingTable,
-      logPrefix: loggingPrefix
+      logPrefix
     })
     this.rpc = new RPC(components, {
       routingTable: this.routingTable,
       providers: this.providers,
       peerRouting: this.peerRouting,
       validators: this.validators,
-      logPrefix: loggingPrefix,
+      logPrefix,
+      metricsPrefix,
       peerInfoMapper: this.peerInfoMapper
     })
     this.topologyListener = new TopologyListener(components, {
       protocol: this.protocol,
-      logPrefix: loggingPrefix
+      logPrefix
     })
     this.querySelf = new QuerySelf(components, {
       peerRouting: this.peerRouting,
-      interval: querySelfInterval,
+      interval: init.querySelfInterval,
       initialInterval: init.initialQuerySelfInterval,
-      logPrefix: loggingPrefix,
+      logPrefix,
       initialQuerySelfHasRun,
-      routingTable: this.routingTable
+      routingTable: this.routingTable,
+      operationMetrics
+    })
+    this.reprovider = new Reprovider(components, {
+      ...init.reprovide,
+      logPrefix,
+      metricsPrefix,
+      datastorePrefix,
+      contentRouting: this.contentRouting,
+      lock: providerLock,
+      operationMetrics
     })
 
     // handle peers being discovered during processing of DHT messages
@@ -312,6 +346,13 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
           })
       })
     }
+
+    this.get = timeOperationGenerator(this.get.bind(this), operationMetrics, 'GET_VALUE')
+    this.findProviders = timeOperationGenerator(this.findProviders.bind(this), operationMetrics, 'FIND_PROVIDERS')
+    this.findPeer = timeOperationGenerator(this.findPeer.bind(this), operationMetrics, 'FIND_PEER')
+    this.getClosestPeers = timeOperationGenerator(this.getClosestPeers.bind(this), operationMetrics, 'GET_CLOSEST_PEERS')
+    this.provide = timeOperationGenerator(this.provide.bind(this), operationMetrics, 'PROVIDE')
+    this.put = timeOperationGenerator(this.put.bind(this), operationMetrics, 'PUT_VALUE')
   }
 
   readonly [Symbol.toStringTag] = '@libp2p/kad-dht'
@@ -403,6 +444,10 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
    * Start listening to incoming connections.
    */
   async start (): Promise<void> {
+    if (this.running) {
+      return
+    }
+
     this.running = true
 
     // Only respond to queries when not in client mode
@@ -410,11 +455,11 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
 
     await start(
       this.routingTable,
-      this.providers,
       this.queryManager,
       this.network,
       this.topologyListener,
-      this.routingTableRefresh
+      this.routingTableRefresh,
+      this.reprovider
     )
 
     // Query self after other components are configured
@@ -432,12 +477,12 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
 
     await stop(
       this.querySelf,
-      this.providers,
       this.queryManager,
       this.network,
       this.routingTable,
       this.routingTableRefresh,
-      this.topologyListener
+      this.topologyListener,
+      this.reprovider
     )
   }
 
@@ -464,6 +509,14 @@ export class KadDHT extends TypedEventEmitter<PeerDiscoveryEvents> implements Ka
     yield * this.contentRouting.provide(key, this.components.addressManager.getAddresses(), options)
   }
 
+  /**
+   * Provider records must be re-published every 24 hours - pass a previously
+   * provided CID here to not re-publish a record for it any more
+   */
+  async cancelReprovide (key: CID): Promise<void> {
+    await this.providers.removeProvider(key, this.components.peerId)
+  }
+
   /**
    * Search the dht for providers of the given CID
    */

package/src/network.ts

@@ -16,6 +16,7 @@ import type { AbortOptions, Logger, Stream, PeerId, PeerInfo, Startable, Routing
 export interface NetworkInit {
   protocol: string
   logPrefix: string
+  metricsPrefix: string
   timeout?: Omit<AdaptiveTimeoutInit, 'metricsName' | 'metrics'>
 }
 
@@ -43,19 +44,18 @@ export class Network extends TypedEventEmitter<NetworkEvents> implements Startab
   constructor (components: KadDHTComponents, init: NetworkInit) {
     super()
 
-    const { protocol } = init
     this.components = components
     this.log = components.logger.forComponent(`${init.logPrefix}:network`)
     this.running = false
-    this.protocol = protocol
+    this.protocol = init.protocol
     this.timeout = new AdaptiveTimeout({
       ...(init.timeout ?? {}),
       metrics: components.metrics,
-      metricName: `${init.logPrefix.replaceAll(':', '_')}_network_message_send_times_milliseconds`
+      metricName: `${init.metricsPrefix}_network_message_send_times_milliseconds`
     })
     this.metrics = {
-      operations: components.metrics?.registerCounterGroup(`${init.logPrefix.replaceAll(':', '_')}_outbound_rpc_requests_total`),
-      errors: components.metrics?.registerCounterGroup(`${init.logPrefix.replaceAll(':', '_')}_outbound_rpc_errors_total`)
+      operations: components.metrics?.registerCounterGroup(`${init.metricsPrefix}_outbound_rpc_requests_total`),
+      errors: components.metrics?.registerCounterGroup(`${init.metricsPrefix}_outbound_rpc_errors_total`)
     }
   }
 

package/src/peer-routing/index.ts

@@ -12,7 +12,7 @@ import {
   valueEvent
 } from '../query/events.js'
 import { verifyRecord } from '../record/validators.js'
-import * as utils from '../utils.js'
+import { convertBuffer, keyForPublicKey } from '../utils.js'
 import type { KadDHTComponents, DHTRecord, FinalPeerEvent, QueryEvent, Validators } from '../index.js'
 import type { Message } from '../message/dht.js'
 import type { Network } from '../network.js'
@@ -39,15 +39,13 @@ export class PeerRouting {
   private readonly peerId: PeerId
 
   constructor (components: KadDHTComponents, init: PeerRoutingInit) {
-    const { routingTable, network, validators, queryManager, logPrefix } = init
-
-    this.routingTable = routingTable
-    this.network = network
-    this.validators = validators
-    this.queryManager = queryManager
+    this.routingTable = init.routingTable
+    this.network = init.network
+    this.validators = init.validators
+    this.queryManager = init.queryManager
     this.peerStore = components.peerStore
     this.peerId = components.peerId
-    this.log = components.logger.forComponent(`${logPrefix}:peer-routing`)
+    this.log = components.logger.forComponent(`${init.logPrefix}:peer-routing`)
   }
 
   /**
@@ -108,7 +106,7 @@ export class PeerRouting {
    * Get the public key directly from a node
    */
   async * getPublicKeyFromNode (peer: PeerId, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
-    const pkKey = utils.keyForPublicKey(peer)
+    const pkKey = keyForPublicKey(peer)
 
     for await (const event of this._getValueSingle(peer, pkKey, options)) {
       yield event
@@ -205,7 +203,7 @@ export class PeerRouting {
    */
   async * getClosestPeers (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator<QueryEvent> {
     this.log('getClosestPeers to %b', key)
-    const kadId = await utils.convertBuffer(key)
+    const kadId = await convertBuffer(key)
     const tablePeers = this.routingTable.closestPeers(kadId)
     const self = this // eslint-disable-line @typescript-eslint/no-this-alias
 
@@ -289,7 +287,7 @@ export class PeerRouting {
    * than self
    */
   async getCloserPeersOffline (key: Uint8Array, closerThan: PeerId): Promise<PeerInfo[]> {
-    const id = await utils.convertBuffer(key)
+    const id = await convertBuffer(key)
     const ids = this.routingTable.closestPeers(id)
     const output: PeerInfo[] = []