@libp2p/kad-dht 0.28.6

package/src/content-fetching/index.ts

@@ -0,0 +1,276 @@
+import errcode from 'err-code'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import { Libp2pRecord } from '@libp2p/record'
+import { verifyRecord } from '@libp2p/record/validators'
+import { bestRecord } from '@libp2p/record/selectors'
+import parallel from 'it-parallel'
+import map from 'it-map'
+import {
+  valueEvent,
+  queryErrorEvent
+} from '../query/events.js'
+import { Message, MESSAGE_TYPE } from '../message/index.js'
+import { pipe } from 'it-pipe'
+import {
+  ALPHA
+} from '../constants.js'
+import { createPutRecord, convertBuffer, bufferToRecordKey } from '../utils.js'
+import { logger } from '@libp2p/logger'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { Validators, Selectors, ValueEvent, QueryOptions } from '@libp2p/interfaces/dht'
+import type { Datastore } from 'interface-datastore'
+import type { PeerRouting } from '../peer-routing/index.js'
+import type { QueryManager } from '../query/manager.js'
+import type { RoutingTable } from '../routing-table/index.js'
+import type { Network } from '../network.js'
+import type { Logger } from '@libp2p/logger'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { QueryFunc } from '../query/types.js'
+
+export interface ContentFetchingOptions {
+  peerId: PeerId
+  datastore: Datastore
+  validators: Validators
+  selectors: Selectors
+  peerRouting: PeerRouting
+  queryManager: QueryManager
+  routingTable: RoutingTable
+  network: Network
+  lan: boolean
+}
+
+export class ContentFetching {
+  private readonly log: Logger
+  private readonly peerId: PeerId
+  private readonly datastore: Datastore
+  private readonly validators: Validators
+  private readonly selectors: Selectors
+  private readonly peerRouting: PeerRouting
+  private readonly queryManager: QueryManager
+  private readonly routingTable: RoutingTable
+  private readonly network: Network
+
+  constructor (options: ContentFetchingOptions) {
+    const { peerId, datastore, validators, selectors, peerRouting, queryManager, routingTable, network, lan } = options
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:content-fetching:${peerId.toString()}`)
+    this.peerId = peerId
+    this.datastore = datastore
+    this.validators = validators
+    this.selectors = selectors
+    this.peerRouting = peerRouting
+    this.queryManager = queryManager
+    this.routingTable = routingTable
+    this.network = network
+  }
+
+  async putLocal (key: Uint8Array, rec: Uint8Array) { // eslint-disable-line require-await
+    const dsKey = bufferToRecordKey(key)
+    await this.datastore.put(dsKey, rec)
+  }
+
+  /**
+   * Attempt to retrieve the value for the given key from
+   * the local datastore
+   */
+  async getLocal (key: Uint8Array) {
+    this.log('getLocal %b', key)
+
+    const dsKey = bufferToRecordKey(key)
+
+    this.log('fetching record for key %k', dsKey)
+
+    const raw = await this.datastore.get(dsKey)
+    this.log('found %k in local datastore', dsKey)
+
+    const rec = Libp2pRecord.deserialize(raw)
+
+    await verifyRecord(this.validators, rec)
+
+    return rec
+  }
+
+  /**
+   * Send the best record found to any peers that have an out of date record
+   */
+  async * sendCorrectionRecord (key: Uint8Array, vals: ValueEvent[], best: Uint8Array, options: AbortOptions = {}) {
+    this.log('sendCorrection for %b', key)
+    const fixupRec = await createPutRecord(key, best)
+
+    for (const { value, from } of vals) {
+      // no need to do anything
+      if (uint8ArrayEquals(value, best)) {
+        this.log('record was ok')
+        continue
+      }
+
+      // correct ourself
+      if (this.peerId.equals(from)) {
+        try {
+          const dsKey = bufferToRecordKey(key)
+          this.log(`Storing corrected record for key ${dsKey.toString()}`)
+          await this.datastore.put(dsKey, fixupRec)
+        } catch (err: any) {
+          this.log.error('Failed error correcting self', err)
+        }
+
+        continue
+      }
+
+      // send correction
+      let sentCorrection = false
+      const request = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0)
+      request.record = Libp2pRecord.deserialize(fixupRec)
+
+      for await (const event of this.network.sendRequest(from, request, options)) {
+        if (event.name === 'PEER_RESPONSE' && (event.record != null) && uint8ArrayEquals(event.record.value, Libp2pRecord.deserialize(fixupRec).value)) {
+          sentCorrection = true
+        }
+
+        yield event
+      }
+
+      if (!sentCorrection) {
+        yield queryErrorEvent({ from, error: errcode(new Error('value not put correctly'), 'ERR_PUT_VALUE_INVALID') })
+      }
+
+      this.log.error('Failed error correcting entry')
+    }
+  }
+
+  /**
+   * Store the given key/value pair in the DHT
+   */
+  async * put (key: Uint8Array, value: Uint8Array, options: AbortOptions = {}) {
+    this.log('put key %b value %b', key, value)
+
+    // create record in the dht format
+    const record = await createPutRecord(key, value)
+
+    // store the record locally
+    const dsKey = bufferToRecordKey(key)
+    this.log(`storing record for key ${dsKey.toString()}`)
+    await this.datastore.put(dsKey, record)
+
+    // put record to the closest peers
+    yield * pipe(
+      this.peerRouting.getClosestPeers(key, { signal: options.signal }),
+      (source) => map(source, (event) => {
+        return async () => {
+          if (event.name !== 'FINAL_PEER') {
+            return [event]
+          }
+
+          const events = []
+
+          const msg = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0)
+          msg.record = Libp2pRecord.deserialize(record)
+
+          for await (const putEvent of this.network.sendRequest(event.peer.id, msg, options)) {
+            events.push(putEvent)
+
+            if (putEvent.name !== 'PEER_RESPONSE') {
+              continue
+            }
+
+            if (!(putEvent.record != null && uint8ArrayEquals(putEvent.record.value, Libp2pRecord.deserialize(record).value))) {
+              events.push(queryErrorEvent({ from: event.peer.id, error: errcode(new Error('value not put correctly'), 'ERR_PUT_VALUE_INVALID') }))
+            }
+          }
+
+          return events
+        }
+      }),
+      (source) => parallel(source, {
+        ordered: false,
+        concurrency: ALPHA
+      }),
+      async function * (source) {
+        for await (const events of source) {
+          yield * events
+        }
+      }
+    )
+  }
+
+  /**
+   * Get the value to the given key
+   */
+  async * get (key: Uint8Array, options: QueryOptions = {}) {
+    this.log('get %b', key)
+
+    const vals: ValueEvent[] = []
+
+    for await (const event of this.getMany(key, options)) {
+      if (event.name === 'VALUE') {
+        vals.push(event)
+      }
+
+      yield event
+    }
+
+    if (vals.length === 0) {
+      return
+    }
+
+    const records = vals.map((v) => v.value)
+    let i = 0
+
+    try {
+      i = bestRecord(this.selectors, key, records)
+    } catch (err: any) {
+      // Assume the first record if no selector available
+      if (err.code !== 'ERR_NO_SELECTOR_FUNCTION_FOR_RECORD_KEY') {
+        throw err
+      }
+    }
+
+    const best = records[i]
+    this.log('GetValue %b %b', key, best)
+
+    if (best == null) {
+      throw errcode(new Error('best value was not found'), 'ERR_NOT_FOUND')
+    }
+
+    yield * this.sendCorrectionRecord(key, vals, best, options)
+
+    yield vals[i]
+  }
+
+  /**
+   * Get the `n` values to the given key without sorting
+   */
+  async * getMany (key: Uint8Array, options: QueryOptions = {}) {
+    this.log('getMany values for %b', key)
+
+    try {
+      const localRec = await this.getLocal(key)
+
+      yield valueEvent({
+        value: localRec.value,
+        from: this.peerId
+      })
+    } catch (err: any) {
+      this.log('error getting local value for %b', key, err)
+    }
+
+    const id = await convertBuffer(key)
+    const rtp = this.routingTable.closestPeers(id)
+
+    this.log('found %d peers in routing table', rtp.length)
+
+    const self = this // eslint-disable-line @typescript-eslint/no-this-alias
+
+    const getValueQuery: QueryFunc = async function * ({ peer, signal }) {
+      for await (const event of self.peerRouting.getValueOrPeers(peer, key, { signal })) {
+        yield event
+
+        if (event.name === 'PEER_RESPONSE' && (event.record != null)) {
+          yield valueEvent({ from: peer, value: event.record.value })
+        }
+      }
+    }
+
+    // we have peers, lets send the actual query to them
+    yield * this.queryManager.run(key, rtp, getValueQuery, options)
+  }
+}

package/src/content-routing/index.ts

@@ -0,0 +1,202 @@
+import { Message, MESSAGE_TYPE } from '../message/index.js'
+import parallel from 'it-parallel'
+import map from 'it-map'
+import { convertBuffer } from '../utils.js'
+import { ALPHA } from '../constants.js'
+import { pipe } from 'it-pipe'
+import {
+  queryErrorEvent,
+  peerResponseEvent,
+  providerEvent
+} from '../query/events.js'
+import { logger } from '@libp2p/logger'
+import type { PeerId } from '@libp2p/interfaces/peer-id'
+import type { QueryEvent, QueryOptions } from '@libp2p/interfaces/dht'
+import type { PeerRouting } from '../peer-routing/index.js'
+import type { QueryManager } from '../query/manager.js'
+import type { RoutingTable } from '../routing-table/index.js'
+import type { Network } from '../network.js'
+import type { Logger } from '@libp2p/logger'
+import type { Providers } from '../providers.js'
+import type { PeerStore } from '@libp2p/interfaces/peer-store'
+import type { QueryFunc } from '../query/types.js'
+import type { CID } from 'multiformats/cid'
+import type { AbortOptions } from '@libp2p/interfaces'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import type { PeerData } from '@libp2p/interfaces/peer-data'
+import { base58btc } from 'multiformats/bases/base58'
+
+export interface ContentRoutingOptions {
+  peerId: PeerId
+  network: Network
+  peerRouting: PeerRouting
+  queryManager: QueryManager
+  routingTable: RoutingTable
+  providers: Providers
+  peerStore: PeerStore
+  lan: boolean
+}
+
+export class ContentRouting {
+  private readonly log: Logger
+  private readonly peerId: PeerId
+  private readonly network: Network
+  private readonly peerRouting: PeerRouting
+  private readonly queryManager: QueryManager
+  private readonly routingTable: RoutingTable
+  private readonly providers: Providers
+  private readonly peerStore: PeerStore
+
+  constructor (options: ContentRoutingOptions) {
+    const { peerId, network, peerRouting, queryManager, routingTable, providers, peerStore, lan } = options
+
+    this.log = logger(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:content-routing`)
+    this.peerId = peerId
+    this.network = network
+    this.peerRouting = peerRouting
+    this.queryManager = queryManager
+    this.routingTable = routingTable
+    this.providers = providers
+    this.peerStore = peerStore
+  }
+
+  /**
+   * Announce to the network that we can provide the value for a given key and
+   * are contactable on the given multiaddrs
+   */
+  async * provide (key: CID, multiaddrs: Multiaddr[], options: AbortOptions = {}) {
+    this.log('provide %s', key)
+
+    // Add peer as provider
+    await this.providers.addProvider(key, this.peerId)
+
+    const msg = new Message(MESSAGE_TYPE.ADD_PROVIDER, key.bytes, 0)
+    msg.providerPeers = [{
+      id: this.peerId,
+      multiaddrs,
+      protocols: []
+    }]
+
+    let sent = 0
+
+    const maybeNotifyPeer = (event: QueryEvent) => {
+      return async () => {
+        if (event.name !== 'FINAL_PEER') {
+          return [event]
+        }
+
+        const events = []
+
+        this.log('putProvider %s to %p', key, event.peer.id)
+
+        try {
+          this.log('sending provider record for %s to %p', key, event.peer.id)
+
+          for await (const sendEvent of this.network.sendMessage(event.peer.id, msg, options)) {
+            if (sendEvent.name === 'PEER_RESPONSE') {
+              this.log('sent provider record for %s to %p', key, event.peer.id)
+              sent++
+            }
+
+            events.push(sendEvent)
+          }
+        } catch (err: any) {
+          this.log.error('error sending provide record to peer %p', event.peer.id, err)
+          events.push(queryErrorEvent({ from: event.peer.id, error: err }))
+        }
+
+        return events
+      }
+    }
+
+    // Notify closest peers
+    yield * pipe(
+      this.peerRouting.getClosestPeers(key.multihash.bytes, options),
+      (source) => map(source, (event) => maybeNotifyPeer(event)),
+      (source) => parallel(source, {
+        ordered: false,
+        concurrency: ALPHA
+      }),
+      async function * (source) {
+        for await (const events of source) {
+          yield * events
+        }
+      }
+    )
+
+    this.log('sent provider records to %d peers', sent)
+  }
+
+  /**
+   * Search the dht for up to `K` providers of the given CID.
+   */
+  async * findProviders (key: CID, options: QueryOptions) {
+    const toFind = this.routingTable.kBucketSize
+    const target = key.multihash.bytes
+    const id = await convertBuffer(target)
+    const self = this // eslint-disable-line @typescript-eslint/no-this-alias
+
+    this.log('findProviders %c', key)
+
+    const provs = await this.providers.getProviders(key)
+
+    // yield values if we have some, also slice because maybe we got lucky and already have too many?
+    if (provs.length > 0) {
+      const providers: PeerData[] = []
+
+      for (const peerId of provs.slice(0, toFind)) {
+        providers.push({
+          id: peerId,
+          multiaddrs: ((await this.peerStore.addressBook.get(peerId)) ?? []).map(address => address.multiaddr),
+          protocols: []
+        })
+      }
+
+      yield peerResponseEvent({ from: this.peerId, messageType: MESSAGE_TYPE.GET_PROVIDERS, providers })
+      yield providerEvent({ from: this.peerId, providers: providers })
+    }
+
+    // All done
+    if (provs.length >= toFind) {
+      return
+    }
+
+    /**
+     * The query function to use on this particular disjoint path
+     */
+    const findProvidersQuery: QueryFunc = async function * ({ peer, signal }) {
+      const request = new Message(MESSAGE_TYPE.GET_PROVIDERS, target, 0)
+
+      yield * self.network.sendRequest(peer, request, { signal })
+    }
+
+    const providers = new Set(provs.map(p => p.toString(base58btc)))
+
+    for await (const event of this.queryManager.run(target, this.routingTable.closestPeers(id), findProvidersQuery, options)) {
+      yield event
+
+      if (event.name === 'PEER_RESPONSE') {
+        this.log('Found %d provider entries for %c and %d closer peers', event.providers.length, key, event.closer.length)
+
+        const newProviders = []
+
+        for (const peer of event.providers) {
+          if (providers.has(peer.id.toString(base58btc))) {
+            continue
+          }
+
+          providers.add(peer.id.toString(base58btc))
+          newProviders.push(peer)
+        }
+
+        if (newProviders.length > 0) {
+          yield providerEvent({ from: event.from, providers: newProviders })
+        }
+
+        if (providers.size === toFind) {
+          return
+        }
+      }
+    }
+  }
+}