@libp2p/kad-dht 13.1.2 → 14.0.0-0d326d102

package/src/routing-table/index.ts

@@ -1,19 +1,32 @@
-import { InvalidMessageError, TypedEventEmitter } from '@libp2p/interface'
-import { PeerSet } from '@libp2p/peer-collections'
+import { TypedEventEmitter, setMaxListeners, start, stop } from '@libp2p/interface'
+import { AdaptiveTimeout } from '@libp2p/utils/adaptive-timeout'
 import { PeerQueue } from '@libp2p/utils/peer-queue'
-import { pbStream } from 'it-protobuf-stream'
-import { Message, MessageType } from '../message/dht.js'
+import { anySignal } from 'any-signal'
+import parallel from 'it-parallel'
+import { EventTypes } from '../index.js'
+import { MessageType } from '../message/dht.js'
 import * as utils from '../utils.js'
-import { KBucket, isLeafBucket, type Bucket, type PingEventDetails } from './k-bucket.js'
-import type { ComponentLogger, CounterGroup, Logger, Metric, Metrics, PeerId, PeerStore, Startable, Stream } from '@libp2p/interface'
-import type { ConnectionManager } from '@libp2p/interface-internal'
+import { ClosestPeers } from './closest-peers.js'
+import { KBucket, isLeafBucket } from './k-bucket.js'
+import type { Bucket, LeafBucket, Peer } from './k-bucket.js'
+import type { Network } from '../network.js'
+import type { AbortOptions, ComponentLogger, CounterGroup, Logger, Metric, Metrics, PeerId, PeerStore, Startable, Stream } from '@libp2p/interface'
+import type { AdaptiveTimeoutInit } from '@libp2p/utils/adaptive-timeout'
 
-export const KAD_CLOSE_TAG_NAME = 'kad-close'
-export const KAD_CLOSE_TAG_VALUE = 50
 export const KBUCKET_SIZE = 20
-export const PREFIX_LENGTH = 32
-export const PING_TIMEOUT = 10000
-export const PING_CONCURRENCY = 10
+export const PREFIX_LENGTH = 8
+export const PING_NEW_CONTACT_TIMEOUT = 2000
+export const PING_NEW_CONTACT_CONCURRENCY = 20
+export const PING_NEW_CONTACT_MAX_QUEUE_SIZE = 100
+export const PING_OLD_CONTACT_COUNT = 3
+export const PING_OLD_CONTACT_TIMEOUT = 2000
+export const PING_OLD_CONTACT_CONCURRENCY = 20
+export const PING_OLD_CONTACT_MAX_QUEUE_SIZE = 100
+export const KAD_PEER_TAG_NAME = 'kad-peer'
+export const KAD_PEER_TAG_VALUE = 1
+export const LAST_PING_THRESHOLD = 600000
+export const POPULATE_FROM_DATASTORE_ON_START = true
+export const POPULATE_FROM_DATASTORE_LIMIT = 1000
 
 export interface RoutingTableInit {
   logPrefix: string
@@ -21,16 +34,28 @@ export interface RoutingTableInit {
   prefixLength?: number
   splitThreshold?: number
   kBucketSize?: number
-  pingTimeout?: number
-  pingConcurrency?: number
-  tagName?: string
-  tagValue?: number
+  pingNewContactTimeout?: AdaptiveTimeoutInit
+  pingNewContactConcurrency?: number
+  pingNewContactMaxQueueSize?: number
+  pingOldContactTimeout?: AdaptiveTimeoutInit
+  pingOldContactConcurrency?: number
+  pingOldContactMaxQueueSize?: number
+  numberOfOldContactsToPing?: number
+  peerTagName?: string
+  peerTagValue?: number
+  closeTagName?: string
+  closeTagValue?: number
+  network: Network
+  populateFromDatastoreOnStart?: boolean
+  populateFromDatastoreLimit?: number
+  lastPingThreshold?: number
+  closestPeerSetSize?: number
+  closestPeerSetRefreshInterval?: number
 }
 
 export interface RoutingTableComponents {
   peerId: PeerId
   peerStore: PeerStore
-  connectionManager: ConnectionManager
   metrics?: Metrics
   logger: ComponentLogger
 }
@@ -38,33 +63,37 @@ export interface RoutingTableComponents {
 export interface RoutingTableEvents {
   'peer:add': CustomEvent<PeerId>
   'peer:remove': CustomEvent<PeerId>
+  'peer:ping': CustomEvent<PeerId>
 }
 
 /**
- * A wrapper around `k-bucket`, to provide easy store and
- * retrieval for peers.
+ * A wrapper around `k-bucket`, to provide easy store and retrieval for peers.
  */
 export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implements Startable {
   public kBucketSize: number
-  public kb?: KBucket
-  public pingQueue: PeerQueue<boolean>
-
+  public kb: KBucket
+  public network: Network
+  private readonly closestPeerTagger: ClosestPeers
   private readonly log: Logger
   private readonly components: RoutingTableComponents
-  private readonly prefixLength: number
-  private readonly splitThreshold: number
-  private readonly pingTimeout: number
-  private readonly pingConcurrency: number
   private running: boolean
+  private readonly pingNewContactTimeout: AdaptiveTimeout
+  private readonly pingNewContactQueue: PeerQueue<boolean>
+  private readonly pingOldContactTimeout: AdaptiveTimeout
+  private readonly pingOldContactQueue: PeerQueue<boolean>
+  private readonly populateFromDatastoreOnStart: boolean
+  private readonly populateFromDatastoreLimit: number
   private readonly protocol: string
-  private readonly tagName: string
-  private readonly tagValue: number
+  private readonly peerTagName: string
+  private readonly peerTagValue: number
   private readonly metrics?: {
     routingTableSize: Metric
     routingTableKadBucketTotal: Metric
     routingTableKadBucketAverageOccupancy: Metric
     routingTableKadBucketMaxDepth: Metric
-    kadBucketEvents: CounterGroup<'ping' | 'ping_error' | 'peer_added' | 'peer_removed'>
+    routingTableKadBucketMinOccupancy: Metric
+    routingTableKadBucketMaxOccupancy: Metric
+    kadBucketEvents: CounterGroup<'ping_old_contact' | 'ping_old_contact_error' | 'ping_new_contact' | 'ping_new_contact_error' | 'peer_added' | 'peer_removed'>
   }
 
   constructor (components: RoutingTableComponents, init: RoutingTableInit) {
@@ -73,22 +102,61 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
     this.components = components
     this.log = components.logger.forComponent(`${init.logPrefix}:routing-table`)
     this.kBucketSize = init.kBucketSize ?? KBUCKET_SIZE
-    this.pingTimeout = init.pingTimeout ?? PING_TIMEOUT
-    this.pingConcurrency = init.pingConcurrency ?? PING_CONCURRENCY
     this.running = false
     this.protocol = init.protocol
-    this.tagName = init.tagName ?? KAD_CLOSE_TAG_NAME
-    this.tagValue = init.tagValue ?? KAD_CLOSE_TAG_VALUE
-    this.prefixLength = init.prefixLength ?? PREFIX_LENGTH
-    this.splitThreshold = init.splitThreshold ?? KBUCKET_SIZE
-
-    this.pingQueue = new PeerQueue({
-      concurrency: this.pingConcurrency,
-      metricName: `${init.logPrefix.replaceAll(':', '_')}_ping_queue`,
-      metrics: this.components.metrics
+    this.network = init.network
+    this.peerTagName = init.peerTagName ?? KAD_PEER_TAG_NAME
+    this.peerTagValue = init.peerTagValue ?? KAD_PEER_TAG_VALUE
+    this.pingOldContacts = this.pingOldContacts.bind(this)
+    this.verifyNewContact = this.verifyNewContact.bind(this)
+    this.peerAdded = this.peerAdded.bind(this)
+    this.peerRemoved = this.peerRemoved.bind(this)
+    this.populateFromDatastoreOnStart = init.populateFromDatastoreOnStart ?? POPULATE_FROM_DATASTORE_ON_START
+    this.populateFromDatastoreLimit = init.populateFromDatastoreLimit ?? POPULATE_FROM_DATASTORE_LIMIT
+
+    this.pingOldContactQueue = new PeerQueue({
+      concurrency: init.pingOldContactConcurrency ?? PING_OLD_CONTACT_CONCURRENCY,
+      metricName: `${init.logPrefix.replaceAll(':', '_')}_ping_old_contact_queue`,
+      metrics: this.components.metrics,
+      maxSize: init.pingOldContactMaxQueueSize ?? PING_OLD_CONTACT_MAX_QUEUE_SIZE
+    })
+    this.pingOldContactTimeout = new AdaptiveTimeout({
+      ...(init.pingOldContactTimeout ?? {}),
+      metrics: this.components.metrics,
+      metricName: `${init.logPrefix.replaceAll(':', '_')}_routing_table_ping_old_contact_time_milliseconds`
+    })
+
+    this.pingNewContactQueue = new PeerQueue({
+      concurrency: init.pingNewContactConcurrency ?? PING_NEW_CONTACT_CONCURRENCY,
+      metricName: `${init.logPrefix.replaceAll(':', '_')}_ping_new_contact_queue`,
+      metrics: this.components.metrics,
+      maxSize: init.pingNewContactMaxQueueSize ?? PING_NEW_CONTACT_MAX_QUEUE_SIZE
+    })
+    this.pingNewContactTimeout = new AdaptiveTimeout({
+      ...(init.pingNewContactTimeout ?? {}),
+      metrics: this.components.metrics,
+      metricName: `${init.logPrefix.replaceAll(':', '_')}_routing_table_ping_new_contact_time_milliseconds`
+    })
+
+    this.kb = new KBucket({
+      kBucketSize: init.kBucketSize,
+      prefixLength: init.prefixLength,
+      splitThreshold: init.splitThreshold,
+      numberOfOldContactsToPing: init.numberOfOldContactsToPing,
+      lastPingThreshold: init.lastPingThreshold,
+      ping: this.pingOldContacts,
+      verify: this.verifyNewContact,
+      onAdd: this.peerAdded,
+      onRemove: this.peerRemoved
     })
-    this.pingQueue.addEventListener('error', evt => {
-      this.log.error('error pinging peer', evt.detail)
+
+    this.closestPeerTagger = new ClosestPeers(this.components, {
+      logPrefix: init.logPrefix,
+      routingTable: this,
+      peerSetSize: init.closestPeerSetSize,
+      refreshInterval: init.closestPeerSetRefreshInterval,
+      closeTagName: init.closeTagName,
+      closeTagValue: init.closeTagValue
     })
 
     if (this.components.metrics != null) {
@@ -96,6 +164,8 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
         routingTableSize: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_size`),
         routingTableKadBucketTotal: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_kad_bucket_total`),
         routingTableKadBucketAverageOccupancy: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_kad_bucket_average_occupancy`),
+        routingTableKadBucketMinOccupancy: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_kad_bucket_min_occupancy`),
+        routingTableKadBucketMaxOccupancy: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_kad_bucket_max_occupancy`),
         routingTableKadBucketMaxDepth: this.components.metrics.registerMetric(`${init.logPrefix.replaceAll(':', '_')}_routing_table_kad_bucket_max_depth`),
         kadBucketEvents: this.components.metrics.registerCounterGroup(`${init.logPrefix.replaceAll(':', '_')}_kad_bucket_events_total`)
       }
@@ -109,106 +179,87 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
   async start (): Promise<void> {
     this.running = true
 
-    const kBuck = new KBucket({
-      localPeer: {
-        kadId: await utils.convertPeerId(this.components.peerId),
-        peerId: this.components.peerId
-      },
-      kBucketSize: this.kBucketSize,
-      prefixLength: this.prefixLength,
-      splitThreshold: this.splitThreshold,
-      numberOfNodesToPing: 1
-    })
-    this.kb = kBuck
-
-    // test whether to evict peers
-    kBuck.addEventListener('ping', (evt) => {
-      this.metrics?.kadBucketEvents.increment({ ping: true })
-
-      this._onPing(evt).catch(err => {
-        this.metrics?.kadBucketEvents.increment({ ping_error: true })
-        this.log.error('could not process k-bucket ping event', err)
-      })
-    })
+    await start(this.closestPeerTagger)
+    await this.kb.addSelfPeer(this.components.peerId)
+  }
 
-    let peerStorePeers = 0
+  async afterStart (): Promise<void> {
+    // do this async to not block startup but iterate serially to not overwhelm
+    // the ping queue
+    Promise.resolve().then(async () => {
+      if (!this.populateFromDatastoreOnStart) {
+        return
+      }
 
-    // add existing peers from the peer store to routing table
-    for (const peer of await this.components.peerStore.all()) {
-      if (peer.protocols.includes(this.protocol)) {
-        const id = await utils.convertPeerId(peer.id)
+      let peerStorePeers = 0
+
+      // add existing peers from the peer store to routing table
+      for (const peer of await this.components.peerStore.all({
+        filters: [(peer) => {
+          return peer.protocols.includes(this.protocol) && peer.tags.has(KAD_PEER_TAG_NAME)
+        }],
+        limit: this.populateFromDatastoreLimit
+      })) {
+        if (!this.running) {
+          // bail if we've been shut down
+          return
+        }
 
-        this.kb.add({ kadId: id, peerId: peer.id })
-        peerStorePeers++
+        try {
+          await this.add(peer.id)
+          peerStorePeers++
+        } catch (err) {
+          this.log('failed to add peer %p to routing table, removing kad-dht peer tags - %e')
+          await this.components.peerStore.merge(peer.id, {
+            tags: {
+              [this.peerTagName]: undefined
+            }
+          })
+        }
       }
-    }
-
-    this.log('added %d peer store peers to the routing table', peerStorePeers)
 
-    // tag kad-close peers
-    this._tagPeers(kBuck)
+      this.log('added %d peer store peers to the routing table', peerStorePeers)
+    })
+      .catch(err => {
+        this.log.error('error adding peer store peers to the routing table %e', err)
+      })
   }
 
   async stop (): Promise<void> {
     this.running = false
-    this.pingQueue.clear()
-    this.kb = undefined
+    await stop(this.closestPeerTagger)
+    this.pingOldContactQueue.abort()
+    this.pingNewContactQueue.abort()
   }
 
-  /**
-   * Keep track of our k-closest peers and tag them in the peer store as such
-   * - this will lower the chances that connections to them get closed when
-   * we reach connection limits
-   */
-  _tagPeers (kBuck: KBucket): void {
-    let kClosest = new PeerSet()
-
-    const updatePeerTags = utils.debounce(() => {
-      const newClosest = new PeerSet(
-        kBuck.closest(kBuck.localPeer.kadId, KBUCKET_SIZE)
-      )
-      const addedPeers = newClosest.difference(kClosest)
-      const removedPeers = kClosest.difference(newClosest)
-
-      Promise.resolve()
-        .then(async () => {
-          for (const peer of addedPeers) {
-            await this.components.peerStore.merge(peer, {
-              tags: {
-                [this.tagName]: {
-                  value: this.tagValue
-                }
-              }
-            })
-          }
-
-          for (const peer of removedPeers) {
-            await this.components.peerStore.merge(peer, {
-              tags: {
-                [this.tagName]: undefined
-              }
-            })
+  private async peerAdded (peer: Peer, bucket: LeafBucket): Promise<void> {
+    if (!this.components.peerId.equals(peer.peerId)) {
+      await this.components.peerStore.merge(peer.peerId, {
+        tags: {
+          [this.peerTagName]: {
+            value: this.peerTagValue
           }
-        })
-        .catch(err => {
-          this.log.error('Could not update peer tags', err)
-        })
-
-      kClosest = newClosest
-    })
+        }
+      })
+    }
 
-    kBuck.addEventListener('added', (evt) => {
-      updatePeerTags()
+    this.updateMetrics()
+    this.metrics?.kadBucketEvents.increment({ peer_added: true })
+    this.safeDispatchEvent('peer:add', { detail: peer.peerId })
+  }
 
-      this.metrics?.kadBucketEvents.increment({ peer_added: true })
-      this.safeDispatchEvent('peer:add', { detail: evt.detail.peerId })
-    })
-    kBuck.addEventListener('removed', (evt) => {
-      updatePeerTags()
+  private async peerRemoved (peer: Peer, bucket: LeafBucket): Promise<void> {
+    if (!this.components.peerId.equals(peer.peerId)) {
+      await this.components.peerStore.merge(peer.peerId, {
+        tags: {
+          [this.peerTagName]: undefined
+        }
+      })
+    }
 
-      this.metrics?.kadBucketEvents.increment({ peer_removed: true })
-      this.safeDispatchEvent('peer:remove', { detail: evt.detail.peerId })
-    })
+    this.updateMetrics()
+    this.metrics?.kadBucketEvents.increment({ peer_removed: true })
+    this.safeDispatchEvent('peer:remove', { detail: peer.peerId })
   }
 
   /**
@@ -221,82 +272,132 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
    * `oldContacts` will not be empty and is the list of contacts that
    * have not been contacted for the longest.
    */
-  async _onPing (evt: CustomEvent<PingEventDetails>): Promise<void> {
+  async * pingOldContacts (oldContacts: Peer[], options?: AbortOptions): AsyncGenerator<Peer> {
     if (!this.running) {
       return
     }
 
-    const {
-      oldContacts,
-      newContact
-    } = evt.detail
-
-    const results = await Promise.all(
-      oldContacts.map(async oldContact => {
-        // if a previous ping wants us to ping this contact, re-use the result
-        const pingJob = this.pingQueue.find(oldContact.peerId)
+    const jobs: Array<() => Promise<Peer | undefined>> = []
 
-        if (pingJob != null) {
-          return pingJob.join()
-        }
+    for (const oldContact of oldContacts) {
+      if (this.kb.get(oldContact.kadId) == null) {
+        this.log('asked to ping contact %p that was not in routing table', oldContact.peerId)
+        continue
+      }
 
-        return this.pingQueue.add(async () => {
-          let stream: Stream | undefined
+      this.metrics?.kadBucketEvents.increment({ ping_old_contact: true })
 
-          try {
-            const options = {
-              signal: AbortSignal.timeout(this.pingTimeout)
-            }
+      jobs.push(async () => {
+        // if a previous ping wants us to ping this contact, re-use the result
+        const existingJob = this.pingOldContactQueue.find(oldContact.peerId)
 
-            this.log('pinging old contact %p', oldContact.peerId)
-            const connection = await this.components.connectionManager.openConnection(oldContact.peerId, options)
-            stream = await connection.newStream(this.protocol, options)
+        if (existingJob != null) {
+          this.log('asked to ping contact %p was already being pinged', oldContact.peerId)
+          const result = await existingJob.join(options)
 
-            const pb = pbStream(stream)
-            await pb.write({
-              type: MessageType.PING
-            }, Message, options)
-            const response = await pb.read(Message, options)
+          if (!result) {
+            return oldContact
+          }
 
-            await pb.unwrap().close()
+          return
+        }
 
-            if (response.type !== MessageType.PING) {
-              throw new InvalidMessageError(`Incorrect message type received, expected PING got ${response.type}`)
-            }
+        const result = await this.pingOldContactQueue.add(async (options) => {
+          const signal = this.pingOldContactTimeout.getTimeoutSignal()
+          const signals = anySignal([signal, options?.signal])
+          setMaxListeners(Infinity, signal, signals)
 
+          try {
+            return await this.pingContact(oldContact, options)
+          } catch {
+            this.metrics?.kadBucketEvents.increment({ ping_old_contact_error: true })
             return true
-          } catch (err: any) {
-            if (this.running && this.kb != null) {
-              // only evict peers if we are still running, otherwise we evict
-              // when dialing is cancelled due to shutdown in progress
-              this.log.error('could not ping peer %p', oldContact.peerId, err)
-              this.log('evicting old contact after ping failed %p', oldContact.peerId)
-              this.kb.remove(oldContact.kadId)
-            }
-
-            stream?.abort(err)
-
-            return false
           } finally {
-            this.updateMetrics()
+            this.pingOldContactTimeout.cleanUp(signal)
+            signals.clear()
           }
         }, {
-          peerId: oldContact.peerId
+          peerId: oldContact.peerId,
+          signal: options?.signal
         })
+
+        if (!result) {
+          return oldContact
+        }
       })
-    )
+    }
+
+    for await (const peer of parallel(jobs)) {
+      if (peer != null) {
+        yield peer
+      }
+    }
+  }
+
+  async verifyNewContact (contact: Peer, options?: AbortOptions): Promise<boolean> {
+    const signal = this.pingNewContactTimeout.getTimeoutSignal()
+    const signals = anySignal([signal, options?.signal])
+    setMaxListeners(Infinity, signal, signals)
 
-    const responded = results
-      .filter(res => res)
-      .length
+    try {
+      const job = this.pingNewContactQueue.find(contact.peerId)
 
-    if (this.running && responded < oldContacts.length && this.kb != null) {
-      this.log('adding new contact %p', newContact.peerId)
-      this.kb.add(newContact)
+      if (job != null) {
+        this.log('joining existing ping to add new peer %p to routing table', contact.peerId)
+        return await job.join({
+          signal: signals
+        })
+      } else {
+        return await this.pingNewContactQueue.add(async (options) => {
+          this.metrics?.kadBucketEvents.increment({ ping_new_contact: true })
+
+          this.log('pinging new peer %p before adding to routing table', contact.peerId)
+          return this.pingContact(contact, options)
+        }, {
+          peerId: contact.peerId,
+          signal: signals
+        })
+      }
+    } catch (err) {
+      this.log.trace('tried to add peer %p but they were not online', contact.peerId)
+      this.metrics?.kadBucketEvents.increment({ ping_new_contact_error: true })
+
+      return false
+    } finally {
+      this.pingNewContactTimeout.cleanUp(signal)
+      signals.clear()
     }
   }
 
-  // -- Public Interface
+  async pingContact (contact: Peer, options?: AbortOptions): Promise<boolean> {
+    let stream: Stream | undefined
+
+    try {
+      this.log('pinging contact %p', contact.peerId)
+
+      for await (const event of this.network.sendRequest(contact.peerId, { type: MessageType.PING }, options)) {
+        if (event.type === EventTypes.PEER_RESPONSE) {
+          if (event.messageType === MessageType.PING) {
+            this.log('contact %p ping ok', contact.peerId)
+
+            this.safeDispatchEvent('peer:ping', {
+              detail: contact.peerId
+            })
+
+            return true
+          }
+
+          return false
+        }
+      }
+
+      return false
+    } catch (err: any) {
+      this.log('error pinging old contact %p - %e', contact.peerId, err)
+      stream?.abort(err)
+      return false
+    }
+  }
 
   /**
    * Amount of currently stored peers
@@ -313,8 +414,8 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
    * Find a specific peer by id
    */
   async find (peer: PeerId): Promise<PeerId | undefined> {
-    const key = await utils.convertPeerId(peer)
-    return this.kb?.get(key)?.peerId
+    const kadId = await utils.convertPeerId(peer)
+    return this.kb.get(kadId)?.peerId
   }
 
   /**
@@ -344,18 +445,12 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
   /**
    * Add or update the routing table with the given peer
    */
-  async add (peerId: PeerId): Promise<void> {
+  async add (peerId: PeerId, options?: AbortOptions): Promise<void> {
     if (this.kb == null) {
       throw new Error('RoutingTable is not started')
     }
 
-    const kadId = await utils.convertPeerId(peerId)
-
-    this.kb.add({ kadId, peerId })
-
-    this.log.trace('added %p with kad id %b', peerId, kadId)
-
-    this.updateMetrics()
+    await this.kb.add(peerId, options)
   }
 
   /**
@@ -366,11 +461,9 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
       throw new Error('RoutingTable is not started')
     }
 
-    const id = await utils.convertPeerId(peer)
-
-    this.kb.remove(id)
+    const kadId = await utils.convertPeerId(peer)
 
-    this.updateMetrics()
+    await this.kb.remove(kadId)
   }
 
   private updateMetrics (): void {
@@ -381,6 +474,8 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
     let size = 0
     let buckets = 0
     let maxDepth = 0
+    let minOccupancy = 20
+    let maxOccupancy = 0
 
     function count (bucket: Bucket): void {
       if (isLeafBucket(bucket)) {
@@ -390,6 +485,15 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
 
         buckets++
         size += bucket.peers.length
+
+        if (bucket.peers.length < minOccupancy) {
+          minOccupancy = bucket.peers.length
+        }
+
+        if (bucket.peers.length > maxOccupancy) {
+          maxOccupancy = bucket.peers.length
+        }
+
         return
       }
 
@@ -402,6 +506,8 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
     this.metrics.routingTableSize.update(size)
     this.metrics.routingTableKadBucketTotal.update(buckets)
     this.metrics.routingTableKadBucketAverageOccupancy.update(Math.round(size / buckets))
+    this.metrics.routingTableKadBucketMinOccupancy.update(minOccupancy)
+    this.metrics.routingTableKadBucketMaxOccupancy.update(maxOccupancy)
     this.metrics.routingTableKadBucketMaxDepth.update(maxDepth)
   }
 }