@libp2p/kad-dht 0.28.6

package/src/peer-routing/index.ts

@@ -0,0 +1,332 @@
+import errcode from 'err-code'
+import { verifyRecord } from '@libp2p/record/validators'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { Message, MESSAGE_TYPE } from '../message/index.js'
+import * as utils from '../utils.js'
+import {
+  queryErrorEvent,
+  finalPeerEvent,
+  valueEvent
+} from '../query/events.js'
+import { PeerDistanceList } from '../peer-list/peer-distance-list.js'
+import { Libp2pRecord } from '@libp2p/record'
+import { base58btc } from 'multiformats/bases/base58'
+import { logger } from '@libp2p/logger'
+import { keys } from '@libp2p/crypto'
+import { peerIdFromKeys } from '@libp2p/peer-id'
+import type { DHTRecord, QueryOptions, Validators } from '@libp2p/interfaces/dht'
+import type { RoutingTable } from '../routing-table/index.js'
+import type { PeerStore } from '@libp2p/interfaces/peer-store'
+import type { QueryManager } from '../query/manager.js'
+import type { Network } from '../network.js'
+import type { Logger } from '@libp2p/logger'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { QueryFunc } from '../query/types.js'
+import type { PeerData } from '@libp2p/interfaces/peer-data'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+
+export interface PeerRoutingOptions {
+  peerId: PeerId
+  routingTable: RoutingTable
+  peerStore: PeerStore
+  network: Network
+  validators: Validators
+  queryManager: QueryManager
+  lan: boolean
+}
+
+export class PeerRouting {
+  private readonly log: Logger
+  private readonly peerId: PeerId
+  private readonly routingTable: RoutingTable
+  private readonly peerStore: PeerStore
+  private readonly network: Network
+  private readonly validators: Validators
+  private readonly queryManager: QueryManager
+
+  constructor (options: PeerRoutingOptions) {
+    const { peerId, routingTable, peerStore, network, validators, queryManager, lan } = options
+
+    this.peerId = peerId
+    this.routingTable = routingTable
+    this.peerStore = peerStore
+    this.network = network
+    this.validators = validators
+    this.queryManager = queryManager
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:peer-routing`)
+  }
+
+  /**
+   * Look if we are connected to a peer with the given id.
+   * Returns its id and addresses, if found, otherwise `undefined`.
+   */
+  async findPeerLocal (peer: PeerId) {
+    let peerData
+    const p = await this.routingTable.find(peer)
+
+    if (p != null) {
+      this.log('findPeerLocal found %p in routing table', peer)
+
+      try {
+        peerData = await this.peerStore.get(p)
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          throw err
+        }
+      }
+    }
+
+    if (peerData == null) {
+      try {
+        peerData = await this.peerStore.get(peer)
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          throw err
+        }
+      }
+    }
+
+    if (peerData != null) {
+      this.log('findPeerLocal found %p in peer store', peer)
+
+      return {
+        id: peerData.id,
+        multiaddrs: peerData.addresses.map((address) => address.multiaddr),
+        protocols: []
+      }
+    }
+  }
+
+  /**
+   * Get a value via rpc call for the given parameters
+   */
+  async * _getValueSingle (peer: PeerId, key: Uint8Array, options: AbortOptions = {}) { // eslint-disable-line require-await
+    const msg = new Message(MESSAGE_TYPE.GET_VALUE, key, 0)
+    yield * this.network.sendRequest(peer, msg, options)
+  }
+
+  /**
+   * Get the public key directly from a node
+   */
+  async * getPublicKeyFromNode (peer: PeerId, options: AbortOptions = {}) {
+    const pkKey = utils.keyForPublicKey(peer)
+
+    for await (const event of this._getValueSingle(peer, pkKey, options)) {
+      yield event
+
+      if (event.name === 'PEER_RESPONSE' && event.record != null) {
+        const recPeer = await peerIdFromKeys(keys.marshalPublicKey({ bytes: event.record.value }))
+
+        // compare hashes of the pub key
+        if (!recPeer.equals(peer)) {
+          throw errcode(new Error('public key does not match id'), 'ERR_PUBLIC_KEY_DOES_NOT_MATCH_ID')
+        }
+
+        if (recPeer.publicKey == null) {
+          throw errcode(new Error('public key missing'), 'ERR_PUBLIC_KEY_MISSING')
+        }
+
+        yield valueEvent({ from: peer, value: recPeer.publicKey })
+      }
+    }
+
+    throw errcode(new Error(`Node not responding with its public key: ${peer.toString(base58btc)}`), 'ERR_INVALID_RECORD')
+  }
+
+  /**
+   * Search for a peer with the given ID
+   */
+  async * findPeer (id: PeerId, options: QueryOptions = {}) {
+    this.log('findPeer %p', id)
+
+    // Try to find locally
+    const pi = await this.findPeerLocal(id)
+
+    // already got it
+    if (pi != null) {
+      this.log('found local')
+      yield finalPeerEvent({
+        from: this.peerId,
+        peer: pi
+      })
+      return
+    }
+
+    const key = await utils.convertPeerId(id)
+    const peers = this.routingTable.closestPeers(key)
+
+    // sanity check
+    const match = peers.find((p) => p.equals(id))
+
+    if (match != null) {
+      try {
+        const peer = await this.peerStore.get(id)
+
+        this.log('found in peerStore')
+        yield finalPeerEvent({
+          from: this.peerId,
+          peer: {
+            id: peer.id,
+            multiaddrs: peer.addresses.map((address) => address.multiaddr),
+            protocols: []
+          }
+        })
+
+        return
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          throw err
+        }
+      }
+    }
+
+    const self = this // eslint-disable-line @typescript-eslint/no-this-alias
+
+    const findPeerQuery: QueryFunc = async function * ({ peer, signal }) {
+      const request = new Message(MESSAGE_TYPE.FIND_NODE, id.toBytes(), 0)
+
+      for await (const event of self.network.sendRequest(peer, request, { signal })) {
+        yield event
+
+        if (event.name === 'PEER_RESPONSE') {
+          const match = event.closer.find((p) => p.id.equals(id))
+
+          // found the peer
+          if (match != null) {
+            yield finalPeerEvent({ from: event.from, peer: match })
+          }
+        }
+      }
+    }
+
+    let foundPeer = false
+
+    for await (const event of this.queryManager.run(id.toBytes(), peers, findPeerQuery, options)) {
+      if (event.name === 'FINAL_PEER') {
+        foundPeer = true
+      }
+
+      yield event
+    }
+
+    if (!foundPeer) {
+      yield queryErrorEvent({ from: this.peerId, error: errcode(new Error('Not found'), 'ERR_NOT_FOUND') })
+    }
+  }
+
+  /**
+   * Kademlia 'node lookup' operation on a key, which could be a the
+   * bytes from a multihash or a peer ID
+   */
+  async * getClosestPeers (key: Uint8Array, options: QueryOptions = {}) {
+    this.log('getClosestPeers to %b', key)
+    const id = await utils.convertBuffer(key)
+    const tablePeers = this.routingTable.closestPeers(id)
+    const self = this // eslint-disable-line @typescript-eslint/no-this-alias
+
+    const peers = new PeerDistanceList(id, this.routingTable.kBucketSize)
+    await Promise.all(tablePeers.map(async peer => await peers.add(peer)))
+
+    const getCloserPeersQuery: QueryFunc = async function * ({ peer, signal }) {
+      self.log('closerPeersSingle %s from %p', uint8ArrayToString(key, 'base32'), peer)
+      const request = new Message(MESSAGE_TYPE.FIND_NODE, key, 0)
+
+      yield * self.network.sendRequest(peer, request, { signal })
+    }
+
+    for await (const event of this.queryManager.run(key, tablePeers, getCloserPeersQuery, options)) {
+      yield event
+
+      if (event.name === 'PEER_RESPONSE') {
+        await Promise.all(event.closer.map(async peerData => await peers.add(peerData.id)))
+      }
+    }
+
+    this.log('found %d peers close to %b', peers.length, key)
+
+    for (const peer of peers.peers) {
+      yield finalPeerEvent({
+        from: this.peerId,
+        peer: {
+          id: peer,
+          multiaddrs: (await (this.peerStore.addressBook.get(peer)) ?? []).map(addr => addr.multiaddr),
+          protocols: []
+        }
+      })
+    }
+  }
+
+  /**
+   * Query a particular peer for the value for the given key.
+   * It will either return the value or a list of closer peers.
+   *
+   * Note: The peerStore is updated with new addresses found for the given peer.
+   */
+  async * getValueOrPeers (peer: PeerId, key: Uint8Array, options: AbortOptions = {}) {
+    for await (const event of this._getValueSingle(peer, key, options)) {
+      if (event.name === 'PEER_RESPONSE') {
+        if (event.record != null) {
+          // We have a record
+          try {
+            await this._verifyRecordOnline(event.record)
+          } catch (err: any) {
+            const errMsg = 'invalid record received, discarded'
+            this.log(errMsg)
+
+            yield queryErrorEvent({ from: event.from, error: errcode(new Error(errMsg), 'ERR_INVALID_RECORD') })
+            continue
+          }
+        }
+      }
+
+      yield event
+    }
+  }
+
+  /**
+   * Verify a record, fetching missing public keys from the network.
+   * Throws an error if the record is invalid.
+   */
+  async _verifyRecordOnline (record: DHTRecord) {
+    await verifyRecord(this.validators, new Libp2pRecord(record.key, record.value, record.timeReceived))
+  }
+
+  /**
+   * Get the nearest peers to the given query, but if closer
+   * than self
+   */
+  async getCloserPeersOffline (key: Uint8Array, closerThan: PeerId) {
+    const id = await utils.convertBuffer(key)
+    const ids = this.routingTable.closestPeers(id)
+    const output: PeerData[] = []
+
+    for (const peerId of ids) {
+      if (peerId.equals(closerThan)) {
+        continue
+      }
+
+      try {
+        const addresses = await this.peerStore.addressBook.get(peerId)
+        const protocols = await this.peerStore.protoBook.get(peerId)
+
+        output.push({
+          id: peerId,
+          multiaddrs: addresses.map((address) => address.multiaddr),
+          protocols
+        })
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          throw err
+        }
+      }
+    }
+
+    if (output.length > 0) {
+      this.log('getCloserPeersOffline found %d peer(s) closer to %b than %p', output.length, key, closerThan)
+    } else {
+      this.log('getCloserPeersOffline could not find peer closer to %b than %p', key, closerThan)
+    }
+
+    return output
+  }
+}

package/src/providers.ts

@@ -0,0 +1,278 @@
+import cache from 'hashlru'
+import varint from 'varint'
+import { Key } from 'interface-datastore/key'
+import Queue from 'p-queue'
+import {
+  PROVIDERS_CLEANUP_INTERVAL,
+  PROVIDERS_VALIDITY,
+  PROVIDERS_LRU_CACHE_SIZE,
+  PROVIDER_KEY_PREFIX
+} from './constants.js'
+import { logger } from '@libp2p/logger'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { peerIdFromString } from '@libp2p/peer-id'
+import type { Datastore } from 'interface-datastore'
+import type { Startable } from '@libp2p/interfaces'
+import type { CID } from 'multiformats'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+
+const log = logger('libp2p:kad-dht:providers')
+
+export interface ProvidersOptions {
+  datastore: Datastore
+  cacheSize?: number
+  /**
+   * How often invalid records are cleaned. (in seconds)
+   */
+  cleanupInterval?: number
+  /**
+   * How long is a provider valid for. (in seconds)
+   */
+  provideValidity?: number
+}
+
+/**
+ * This class manages known providers.
+ * A provider is a peer that we know to have the content for a given CID.
+ *
+ * Every `cleanupInterval` providers are checked if they
+ * are still valid, i.e. younger than the `provideValidity`.
+ * If they are not, they are deleted.
+ *
+ * To ensure the list survives restarts of the daemon,
+ * providers are stored in the datastore, but to ensure
+ * access is fast there is an LRU cache in front of that.
+ */
+export class Providers implements Startable {
+  private readonly datastore: Datastore
+  private readonly cache: ReturnType<typeof cache>
+  private readonly cleanupInterval: number
+  private readonly provideValidity: number
+  private readonly syncQueue: Queue
+  private started: boolean
+  private cleaner?: NodeJS.Timer
+
+  constructor (options: ProvidersOptions) {
+    const { datastore, cacheSize, cleanupInterval, provideValidity } = options
+
+    this.datastore = datastore
+    this.cleanupInterval = cleanupInterval ?? PROVIDERS_CLEANUP_INTERVAL
+    this.provideValidity = provideValidity ?? PROVIDERS_VALIDITY
+    this.cache = cache(cacheSize ?? PROVIDERS_LRU_CACHE_SIZE)
+    this.syncQueue = new Queue({ concurrency: 1 })
+    this.started = false
+  }
+
+  isStarted () {
+    return this.started
+  }
+
+  /**
+   * Start the provider cleanup service
+   */
+  async start () {
+    if (this.started) {
+      return
+    }
+
+    this.started = true
+
+    this.cleaner = setInterval(
+      () => {
+        this._cleanup().catch(err => {
+          log.error(err)
+        })
+      },
+      this.cleanupInterval
+    )
+  }
+
+  /**
+   * Release any resources.
+   */
+  async stop () {
+    this.started = false
+
+    if (this.cleaner != null) {
+      clearInterval(this.cleaner)
+      this.cleaner = undefined
+    }
+  }
+
+  /**
+   * Check all providers if they are still valid, and if not delete them
+   */
+  async _cleanup () {
+    return await this.syncQueue.add(async () => {
+      const start = Date.now()
+
+      let count = 0
+      let deleteCount = 0
+      const deleted = new Map<string, Set<string>>()
+      const batch = this.datastore.batch()
+
+      // Get all provider entries from the datastore
+      const query = this.datastore.query({ prefix: PROVIDER_KEY_PREFIX })
+
+      for await (const entry of query) {
+        try {
+          // Add a delete to the batch for each expired entry
+          const { cid, peerId } = parseProviderKey(entry.key)
+          const time = readTime(entry.value).getTime()
+          const now = Date.now()
+          const delta = now - time
+          const expired = delta > this.provideValidity
+
+          log('comparing: %d - %d = %d > %d %s', now, time, delta, this.provideValidity, expired ? '(expired)' : '')
+
+          if (expired) {
+            deleteCount++
+            batch.delete(entry.key)
+            const peers = deleted.get(cid) ?? new Set<string>()
+            peers.add(peerId)
+            deleted.set(cid, peers)
+          }
+          count++
+        } catch (err: any) {
+          log.error(err.message)
+        }
+      }
+
+      // Commit the deletes to the datastore
+      if (deleted.size > 0) {
+        log('deleting %d / %d entries', deleteCount, count)
+        await batch.commit()
+      } else {
+        log('nothing to delete')
+      }
+
+      // Clear expired entries from the cache
+      for (const [cid, peers] of deleted) {
+        const key = makeProviderKey(cid)
+        const provs = this.cache.get(key)
+
+        if (provs != null) {
+          for (const peerId of peers) {
+            provs.delete(peerId)
+          }
+
+          if (provs.size === 0) {
+            this.cache.remove(key)
+          } else {
+            this.cache.set(key, provs)
+          }
+        }
+      }
+
+      log('Cleanup successful (%dms)', Date.now() - start)
+    })
+  }
+
+  /**
+   * Get the currently known provider peer ids for a given CID
+   */
+  async _getProvidersMap (cid: CID) {
+    const cacheKey = makeProviderKey(cid)
+    let provs: Map<string, Date> = this.cache.get(cacheKey)
+
+    if (provs == null) {
+      provs = await loadProviders(this.datastore, cid)
+      this.cache.set(cacheKey, provs)
+    }
+
+    return provs
+  }
+
+  /**
+   * Add a new provider for the given CID
+   */
+  async addProvider (cid: CID, provider: PeerId) {
+    return await this.syncQueue.add(async () => {
+      log('%p provides %s', provider, cid)
+      const provs = await this._getProvidersMap(cid)
+
+      log('loaded %s provs', provs.size)
+      const now = new Date()
+      provs.set(provider.toString(), now)
+
+      const dsKey = makeProviderKey(cid)
+      this.cache.set(dsKey, provs)
+
+      await writeProviderEntry(this.datastore, cid, provider, now)
+    })
+  }
+
+  /**
+   * Get a list of providers for the given CID
+   */
+  async getProviders (cid: CID): Promise<PeerId[]> {
+    return await this.syncQueue.add(async () => {
+      log('get providers for %s', cid)
+      const provs = await this._getProvidersMap(cid)
+
+      return [...provs.keys()].map(peerIdStr => {
+        return peerIdFromString(peerIdStr)
+      })
+    })
+  }
+}
+
+/**
+ * Encode the given key its matching datastore key
+ */
+function makeProviderKey (cid: CID | string) {
+  const cidStr = typeof cid === 'string' ? cid : uint8ArrayToString(cid.multihash.bytes, 'base32')
+
+  return `${PROVIDER_KEY_PREFIX}/${cidStr}`
+}
+
+/**
+ * Write a provider into the given store
+ */
+async function writeProviderEntry (store: Datastore, cid: CID, peer: PeerId, time: Date) { // eslint-disable-line require-await
+  const dsKey = [
+    makeProviderKey(cid),
+    '/',
+    peer.toString()
+  ].join('')
+
+  const key = new Key(dsKey)
+  const buffer = Uint8Array.from(varint.encode(time.getTime()))
+
+  return await store.put(key, buffer)
+}
+
+/**
+ * Parse the CID and provider peer id from the key
+ */
+function parseProviderKey (key: Key) {
+  const parts = key.toString().split('/')
+
+  if (parts.length !== 5) {
+    throw new Error(`incorrectly formatted provider entry key in datastore: ${key.toString()}`)
+  }
+
+  return {
+    cid: parts[3],
+    peerId: parts[4]
+  }
+}
+
+/**
+ * Load providers for the given CID from the store
+ */
+async function loadProviders (store: Datastore, cid: CID) {
+  const providers = new Map<string, Date>()
+  const query = store.query({ prefix: makeProviderKey(cid) })
+
+  for await (const entry of query) {
+    const { peerId } = parseProviderKey(entry.key)
+    providers.set(peerId, readTime(entry.value))
+  }
+
+  return providers
+}
+
+function readTime (buf: Uint8Array) {
+  return new Date(varint.decode(buf))
+}