@libp2p/kad-dht 0.28.6

package/src/rpc/handlers/get-providers.ts

@@ -0,0 +1,95 @@
+import { CID } from 'multiformats/cid'
+import errcode from 'err-code'
+import { Message } from '../../message/index.js'
+import {
+  removePrivateAddresses,
+  removePublicAddresses
+} from '../../utils.js'
+import { logger } from '@libp2p/logger'
+import type { DHTMessageHandler } from '../index.js'
+import type { Providers } from '../../providers.js'
+import type { AddressBook } from '@libp2p/interfaces/peer-store'
+import type { PeerRouting } from '../../peer-routing/index.js'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { PeerData } from '@libp2p/interfaces/peer-data'
+
+const log = logger('libp2p:kad-dht:rpc:handlers:get-providers')
+
+export interface GetProvidersHandlerOptions {
+  peerRouting: PeerRouting
+  providers: Providers
+  addressBook: AddressBook
+  lan: boolean
+}
+
+export class GetProvidersHandler implements DHTMessageHandler {
+  private readonly peerRouting: PeerRouting
+  private readonly providers: Providers
+  private readonly addressBook: AddressBook
+  private readonly lan: boolean
+
+  constructor (options: GetProvidersHandlerOptions) {
+    const { peerRouting, providers, addressBook, lan } = options
+
+    this.peerRouting = peerRouting
+    this.providers = providers
+    this.addressBook = addressBook
+    this.lan = Boolean(lan)
+  }
+
+  async handle (peerId: PeerId, msg: Message) {
+    let cid
+    try {
+      cid = CID.decode(msg.key)
+    } catch (err: any) {
+      throw errcode(new Error('Invalid CID'), 'ERR_INVALID_CID')
+    }
+
+    log('%p asking for providers for %s', peerId, cid)
+
+    const [peers, closer] = await Promise.all([
+      this.providers.getProviders(cid),
+      this.peerRouting.getCloserPeersOffline(msg.key, peerId)
+    ])
+
+    const providerPeers = await this._getPeers(peers)
+    const closerPeers = await this._getPeers(closer.map(({ id }) => id))
+    const response = new Message(msg.type, msg.key, msg.clusterLevel)
+
+    if (providerPeers.length > 0) {
+      response.providerPeers = providerPeers
+    }
+
+    if (closerPeers.length > 0) {
+      response.closerPeers = closerPeers
+    }
+
+    log('got %s providers %s closerPeers', providerPeers.length, closerPeers.length)
+    return response
+  }
+
+  async _getAddresses (peerId: PeerId) {
+    const addrs = await this.addressBook.get(peerId)
+
+    return addrs.map(address => address.multiaddr)
+  }
+
+  async _getPeers (peerIds: PeerId[]) {
+    const output: PeerData[] = []
+    const addrFilter = this.lan ? removePublicAddresses : removePrivateAddresses
+
+    for (const peerId of peerIds) {
+      const peer = addrFilter({
+        id: peerId,
+        multiaddrs: await this._getAddresses(peerId),
+        protocols: []
+      })
+
+      if (peer.multiaddrs.length > 0) {
+        output.push(peer)
+      }
+    }
+
+    return output
+  }
+}

package/src/rpc/handlers/get-value.ts

@@ -0,0 +1,130 @@
+import { Libp2pRecord } from '@libp2p/record'
+import errcode from 'err-code'
+import { Message, MESSAGE_TYPE } from '../../message/index.js'
+import {
+  MAX_RECORD_AGE
+} from '../../constants.js'
+import { bufferToRecordKey, isPublicKeyKey, fromPublicKeyKey } from '../../utils.js'
+import { logger } from '@libp2p/logger'
+import type { DHTMessageHandler } from '../index.js'
+import type { Datastore } from 'interface-datastore'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { KeyBook } from '@libp2p/interfaces/peer-store'
+import type { PeerRouting } from '../../peer-routing/index.js'
+
+const log = logger('libp2p:kad-dht:rpc:handlers:get-value')
+
+export interface GetValueHandlerOptions {
+  keyBook: KeyBook
+  peerRouting: PeerRouting
+  datastore: Datastore
+}
+
+export class GetValueHandler implements DHTMessageHandler {
+  private readonly keyBook: KeyBook
+  private readonly peerRouting: PeerRouting
+  private readonly datastore: Datastore
+
+  constructor (options: GetValueHandlerOptions) {
+    const { keyBook, peerRouting, datastore } = options
+
+    this.keyBook = keyBook
+    this.peerRouting = peerRouting
+    this.datastore = datastore
+  }
+
+  async handle (peerId: PeerId, msg: Message) {
+    const key = msg.key
+
+    log('%p asked for key %b', peerId, key)
+
+    if (key == null || key.length === 0) {
+      throw errcode(new Error('Invalid key'), 'ERR_INVALID_KEY')
+    }
+
+    const response = new Message(MESSAGE_TYPE.GET_VALUE, key, msg.clusterLevel)
+
+    if (isPublicKeyKey(key)) {
+      log('is public key')
+      const idFromKey = fromPublicKeyKey(key)
+      let pubKey: Uint8Array | undefined
+
+      try {
+        const key = await this.keyBook.get(idFromKey)
+
+        if (key == null) {
+          throw errcode(new Error('No public key found in key book'), 'ERR_NOT_FOUND')
+        }
+
+        pubKey = key
+      } catch (err: any) {
+        if (err.code !== 'ERR_NOT_FOUND') {
+          throw err
+        }
+      }
+
+      if (pubKey != null) {
+        log('returning found public key')
+        response.record = new Libp2pRecord(key, pubKey)
+        return response
+      }
+    }
+
+    const [record, closer] = await Promise.all([
+      this._checkLocalDatastore(key),
+      this.peerRouting.getCloserPeersOffline(msg.key, peerId)
+    ])
+
+    if (record != null) {
+      log('had record for %b in local datastore', key)
+      response.record = record
+    }
+
+    if (closer.length > 0) {
+      log('had %s closer peers in routing table', closer.length)
+      response.closerPeers = closer
+    }
+
+    return response
+  }
+
+  /**
+   * Try to fetch a given record by from the local datastore.
+   * Returns the record iff it is still valid, meaning
+   * - it was either authored by this node, or
+   * - it was received less than `MAX_RECORD_AGE` ago.
+   */
+  async _checkLocalDatastore (key: Uint8Array) {
+    log('checkLocalDatastore looking for %b', key)
+    const dsKey = bufferToRecordKey(key)
+
+    // Fetch value from ds
+    let rawRecord
+    try {
+      rawRecord = await this.datastore.get(dsKey)
+    } catch (err: any) {
+      if (err.code === 'ERR_NOT_FOUND') {
+        return undefined
+      }
+      throw err
+    }
+
+    // Create record from the returned bytes
+    const record = Libp2pRecord.deserialize(rawRecord)
+
+    if (record == null) {
+      throw errcode(new Error('Invalid record'), 'ERR_INVALID_RECORD')
+    }
+
+    // Check validity: compare time received with max record age
+    if (record.timeReceived == null ||
+      Date.now() - record.timeReceived.getTime() > MAX_RECORD_AGE) {
+      // If record is bad delete it and return
+      await this.datastore.delete(dsKey)
+      return undefined
+    }
+
+    // Record is valid
+    return record
+  }
+}

package/src/rpc/handlers/ping.ts

@@ -0,0 +1,13 @@
+import { logger } from '@libp2p/logger'
+import type { Message } from '../../message/index.js'
+import type { DHTMessageHandler } from '../index.js'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+
+const log = logger('libp2p:kad-dht:rpc:handlers:ping')
+
+export class PingHandler implements DHTMessageHandler {
+  async handle (peerId: PeerId, msg: Message) {
+    log('ping from %p', peerId)
+    return msg
+  }
+}

package/src/rpc/handlers/put-value.ts

@@ -0,0 +1,58 @@
+import { bufferToRecordKey } from '../../utils.js'
+import errcode from 'err-code'
+import { verifyRecord } from '@libp2p/record/validators'
+import { Logger, logger } from '@libp2p/logger'
+import type { DHTMessageHandler } from '../index.js'
+import type { Validators } from '@libp2p/interfaces/dht'
+import type { Datastore } from 'interface-datastore'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { Message } from '../../message/index.js'
+import { base58btc } from 'multiformats/bases/base58'
+
+export interface PutValueHandlerOptions {
+  peerId: PeerId
+  validators: Validators
+  datastore: Datastore
+}
+
+export class PutValueHandler implements DHTMessageHandler {
+  private readonly validators: Validators
+  private readonly datastore: Datastore
+  private readonly log: Logger
+
+  constructor (options: PutValueHandlerOptions) {
+    const { validators, datastore, peerId } = options
+
+    this.log = logger('libp2p:kad-dht:rpc:handlers:put-value:' + peerId.toString())
+
+    this.validators = validators
+    this.datastore = datastore
+  }
+
+  async handle (peerId: PeerId, msg: Message) {
+    const key = msg.key
+    this.log('%p asked us to store value for key %b', peerId, key)
+
+    const record = msg.record
+
+    if (record == null) {
+      const errMsg = `Empty record from: ${peerId.toString(base58btc)}`
+
+      this.log.error(errMsg)
+      throw errcode(new Error(errMsg), 'ERR_EMPTY_RECORD')
+    }
+
+    try {
+      await verifyRecord(this.validators, record)
+
+      record.timeReceived = new Date()
+      const recordKey = bufferToRecordKey(record.key)
+      await this.datastore.put(recordKey, record.serialize())
+      this.log('put record for %b into datastore under key %k', key, recordKey)
+    } catch (err: any) {
+      this.log('did not put record for key %b into datastore %o', key, err)
+    }
+
+    return msg
+  }
+}

package/src/rpc/index.ts

@@ -0,0 +1,118 @@
+import { pipe } from 'it-pipe'
+import * as lp from 'it-length-prefixed'
+import { Logger, logger } from '@libp2p/logger'
+import type { RoutingTable } from '../routing-table'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import { Message, MESSAGE_TYPE, MESSAGE_TYPE_LOOKUP } from '../message/index.js'
+import { AddProviderHandler } from './handlers/add-provider.js'
+import { FindNodeHandler } from './handlers/find-node.js'
+import { GetProvidersHandler } from './handlers/get-providers.js'
+import { GetValueHandler } from './handlers/get-value.js'
+import { PingHandler } from './handlers/ping.js'
+import { PutValueHandler } from './handlers/put-value.js'
+import type { IncomingStreamData } from '@libp2p/interfaces/registrar'
+import type { KeyBook, AddressBook } from '@libp2p/interfaces/peer-store'
+import type { Providers } from '../providers'
+import type { PeerRouting } from '../peer-routing'
+import type { Datastore } from 'interface-datastore'
+import type { Validators } from '@libp2p/interfaces/dht'
+
+export interface DHTMessageHandler {
+  handle: (peerId: PeerId, msg: Message) => Promise<Message | undefined>
+}
+
+export interface RPCOptions {
+  peerId: PeerId
+  routingTable: RoutingTable
+  keyBook: KeyBook
+  addressBook: AddressBook
+  providers: Providers
+  peerRouting: PeerRouting
+  datastore: Datastore
+  validators: Validators
+  lan: boolean
+}
+
+export class RPC {
+  private readonly handlers: Record<number, DHTMessageHandler>
+  private readonly routingTable: RoutingTable
+  private readonly log: Logger
+
+  constructor (options: RPCOptions) {
+    const { keyBook, addressBook, providers, peerRouting, datastore, validators, lan, peerId } = options
+
+    this.log = logger('libp2p:kad-dht:rpc:' + peerId.toString())
+
+    this.routingTable = options.routingTable
+    this.handlers = {
+      [MESSAGE_TYPE.GET_VALUE]: new GetValueHandler({ keyBook, peerRouting, datastore }),
+      [MESSAGE_TYPE.PUT_VALUE]: new PutValueHandler({ peerId, validators, datastore }),
+      [MESSAGE_TYPE.FIND_NODE]: new FindNodeHandler({ peerRouting, lan }),
+      [MESSAGE_TYPE.ADD_PROVIDER]: new AddProviderHandler({ providers }),
+      [MESSAGE_TYPE.GET_PROVIDERS]: new GetProvidersHandler({ peerRouting, providers, addressBook, lan }),
+      [MESSAGE_TYPE.PING]: new PingHandler()
+    }
+  }
+
+  /**
+   * Process incoming DHT messages
+   */
+  async handleMessage (peerId: PeerId, msg: Message) {
+    try {
+      await this.routingTable.add(peerId)
+    } catch (err: any) {
+      this.log.error('Failed to update the kbucket store', err)
+    }
+
+    // get handler & execute it
+    const handler = this.handlers[msg.type]
+
+    if (handler == null) {
+      this.log.error(`no handler found for message type: ${msg.type}`)
+      return
+    }
+
+    return await handler.handle(peerId, msg)
+  }
+
+  /**
+   * Handle incoming streams on the dht protocol
+   */
+  onIncomingStream (evt: CustomEvent<IncomingStreamData>) {
+    Promise.resolve().then(async () => {
+      const { stream, connection } = evt.detail
+      const peerId = connection.remotePeer
+
+      try {
+        await this.routingTable.add(peerId)
+      } catch (err: any) {
+        this.log.error(err)
+      }
+
+      const self = this // eslint-disable-line @typescript-eslint/no-this-alias
+
+      await pipe(
+        stream.source,
+        lp.decode(),
+        source => (async function * () {
+          for await (const msg of source) {
+            // handle the message
+            const desMessage = Message.deserialize(msg.slice())
+            self.log('incoming %s from %p', MESSAGE_TYPE_LOOKUP[desMessage.type], peerId)
+            const res = await self.handleMessage(peerId, desMessage)
+
+            // Not all handlers will return a response
+            if (res != null) {
+              yield res.serialize()
+            }
+          }
+        })(),
+        lp.encode(),
+        stream.sink
+      )
+    })
+      .catch(err => {
+        this.log.error(err)
+      })
+  }
+}

package/src/topology-listener.ts

@@ -0,0 +1,78 @@
+import { Topology } from '@libp2p/topology'
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces'
+import { logger } from '@libp2p/logger'
+import type { Registrar } from '@libp2p/interfaces/registrar'
+import type { Logger } from '@libp2p/logger'
+import type { Startable } from '@libp2p/interfaces'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+
+export interface TopologyListenerOptions {
+  registrar: Registrar
+  protocol: string
+  lan: boolean
+}
+
+export interface TopologyListenerEvents {
+  'peer': CustomEvent<PeerId>
+}
+
+/**
+ * Receives notifications of new peers joining the network that support the DHT protocol
+ */
+export class TopologyListener extends EventEmitter<TopologyListenerEvents> implements Startable {
+  private readonly log: Logger
+  private readonly registrar: Registrar
+  private readonly protocol: string
+  private running: boolean
+  private registrarId?: string
+
+  constructor (options: TopologyListenerOptions) {
+    super()
+
+    const { registrar, protocol, lan } = options
+
+    this.log = logger(`libp2p:kad-dht:topology-listener:${lan ? 'lan' : 'wan'}`)
+    this.running = false
+    this.registrar = registrar
+    this.protocol = protocol
+  }
+
+  isStarted () {
+    return this.running
+  }
+
+  /**
+   * Start the network
+   */
+  async start () {
+    if (this.running) {
+      return
+    }
+
+    this.running = true
+
+    // register protocol with topology
+    const topology = new Topology({
+      onConnect: (peerId) => {
+        this.log('observed peer %p with protocol %s', this.protocol, peerId)
+        this.dispatchEvent(new CustomEvent('peer', {
+          detail: peerId
+        }))
+      }
+    })
+    this.registrarId = await this.registrar.register(this.protocol, topology)
+  }
+
+  /**
+   * Stop all network activity
+   */
+  stop () {
+    this.running = false
+
+    // unregister protocol and handlers
+    if (this.registrarId != null) {
+      this.registrar.unregister(this.registrarId)
+      this.registrarId = undefined
+    }
+  }
+}

package/src/utils.ts

@@ -0,0 +1,108 @@
+import { sha256 } from 'multiformats/hashes/sha2'
+import { Key } from 'interface-datastore/key'
+import { Libp2pRecord } from '@libp2p/record'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import isPrivateIp from 'private-ip'
+import type { PeerData } from '@libp2p/interfaces/peer-data'
+import { peerIdFromBytes } from '@libp2p/peer-id'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import { RECORD_KEY_PREFIX } from './constants.js'
+
+// const IPNS_PREFIX = uint8ArrayFromString('/ipns/')
+const PK_PREFIX = uint8ArrayFromString('/pk/')
+
+export function removePrivateAddresses (peer: PeerData): PeerData {
+  return {
+    ...peer,
+    multiaddrs: peer.multiaddrs.filter(multiaddr => {
+      const [[type, addr]] = multiaddr.stringTuples()
+
+      if (type !== 4 && type !== 6) {
+        return false
+      }
+
+      // @ts-expect-error types are wrong https://github.com/frenchbread/private-ip/issues/18
+      return !isPrivateIp(addr)
+    })
+  }
+}
+
+export function removePublicAddresses (peer: PeerData): PeerData {
+  return {
+    ...peer,
+    multiaddrs: peer.multiaddrs.filter(multiaddr => {
+      const [[type, addr]] = multiaddr.stringTuples()
+
+      if (type !== 4 && type !== 6) {
+        return false
+      }
+
+      // @ts-expect-error types are wrong https://github.com/frenchbread/private-ip/issues/18
+      return isPrivateIp(addr)
+    })
+  }
+}
+
+/**
+ * Creates a DHT ID by hashing a given Uint8Array
+ */
+export async function convertBuffer (buf: Uint8Array) {
+  const multihash = await sha256.digest(buf)
+
+  return multihash.digest
+}
+
+/**
+ * Creates a DHT ID by hashing a Peer ID
+ */
+export async function convertPeerId (peerId: PeerId) {
+  return await convertBuffer(peerId.toBytes())
+}
+
+/**
+ * Convert a Uint8Array to their SHA2-256 hash
+ */
+export function bufferToKey (buf: Uint8Array) {
+  return new Key('/' + uint8ArrayToString(buf, 'base32'), false)
+}
+
+/**
+ * Convert a Uint8Array to their SHA2-256 hash
+ */
+export function bufferToRecordKey (buf: Uint8Array) {
+  return new Key(`${RECORD_KEY_PREFIX}/${uint8ArrayToString(buf, 'base32')}`, false)
+}
+
+/**
+ * Generate the key for a public key.
+ */
+export function keyForPublicKey (peer: PeerId) {
+  return uint8ArrayConcat([
+    PK_PREFIX,
+    peer.toBytes()
+  ])
+}
+
+export function isPublicKeyKey (key: Uint8Array) {
+  return uint8ArrayToString(key.slice(0, 4)) === '/pk/'
+}
+
+export function isIPNSKey (key: Uint8Array) {
+  return uint8ArrayToString(key.slice(0, 4)) === '/ipns/'
+}
+
+export function fromPublicKeyKey (key: Uint8Array) {
+  return peerIdFromBytes(key.slice(4))
+}
+
+/**
+ * Create a new put record, encodes and signs it if enabled
+ */
+export function createPutRecord (key: Uint8Array, value: Uint8Array) {
+  const timeReceived = new Date()
+  const rec = new Libp2pRecord(key, value, timeReceived)
+
+  return rec.serialize()
+}