libp2p 0.36.0 → 0.36.2-1473044

package/src/connection-manager/latency-monitor.ts

@@ -0,0 +1,319 @@
+/**
+ * This code is based on `latency-monitor` (https://github.com/mlucool/latency-monitor) by `mlucool` (https://github.com/mlucool), available under Apache License 2.0 (https://github.com/mlucool/latency-monitor/blob/master/LICENSE)
+ */
+
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces'
+import { VisibilityChangeEmitter } from './visibility-change-emitter.js'
+import { logger } from '@libp2p/logger'
+
+const log = logger('libp2p:connection-manager:latency-monitor')
+
+export interface LatencyMonitorEvents {
+  'data': CustomEvent<SummaryObject>
+}
+
+export interface LatencyMonitorInit {
+  /**
+   * How often to add a latency check event (ms)
+   */
+  latencyCheckIntervalMs?: number
+
+  /**
+   * How often to summarize latency check events. null or 0 disables event firing
+   */
+  dataEmitIntervalMs?: number
+
+  /**
+   * What cb-style async function to use
+   */
+  asyncTestFn?: (cb: () => void) => void
+
+  /**
+   * What percent (+/-) of latencyCheckIntervalMs should we randomly use? This helps avoid alignment to other events.
+   */
+  latencyRandomPercentage?: number
+}
+
+export interface SummaryObject {
+  /**
+   * How many events were called
+   */
+  events: number
+
+  /**
+   * What was the min time for a cb to be called
+   */
+  minMs: number
+
+  /**
+   * What was the max time for a cb to be called
+   */
+  maxMs: number
+
+  /**
+   * What was the average time for a cb to be called
+   */
+  avgMs: number
+
+  /**
+   * How long this interval was in ms
+   */
+  lengthMs: number
+}
+
+interface LatencyData {
+  startTime: number
+  events: number
+  minMs: number
+  maxMs: number
+  totalMs: number
+}
+
+/**
+ * A class to monitor latency of any async function which works in a browser or node. This works by periodically calling
+ * the asyncTestFn and timing how long it takes the callback to be called. It can also periodically emit stats about this.
+ * This can be disabled and stats can be pulled via setting dataEmitIntervalMs = 0.
+ *
+ * @extends {EventEmitter}
+ *
+ * The default implementation is an event loop latency monitor. This works by firing periodic events into the event loop
+ * and timing how long it takes to get back.
+ *
+ * @example
+ * const monitor = new LatencyMonitor();
+ * monitor.on('data', (summary) => console.log('Event Loop Latency: %O', summary));
+ *
+ * @example
+ * const monitor = new LatencyMonitor({latencyCheckIntervalMs: 1000, dataEmitIntervalMs: 60000, asyncTestFn:ping});
+ * monitor.on('data', (summary) => console.log('Ping Pong Latency: %O', summary));
+ */
+export class LatencyMonitor extends EventEmitter<LatencyMonitorEvents> {
+  private readonly latencyCheckIntervalMs: number
+  private readonly latencyRandomPercentage: number
+  private readonly latencyCheckMultiply: number
+  private readonly latencyCheckSubtract: number
+  private readonly dataEmitIntervalMs?: number
+  private readonly asyncTestFn?: (cb: () => void) => void
+
+  private readonly now: (num?: any) => any
+  private readonly getDeltaMS: (num: number) => number
+  private visibilityChangeEmitter?: VisibilityChangeEmitter
+  private latencyData: LatencyData
+  private checkLatencyID?: NodeJS.Timeout
+  private emitIntervalID?: NodeJS.Timeout
+
+  constructor (init: LatencyMonitorInit = {}) {
+    super()
+
+    const { latencyCheckIntervalMs, dataEmitIntervalMs, asyncTestFn, latencyRandomPercentage } = init
+
+    // 0 isn't valid here, so its ok to use ||
+    this.latencyCheckIntervalMs = latencyCheckIntervalMs ?? 500 // 0.5s
+    this.latencyRandomPercentage = latencyRandomPercentage ?? 10
+    this.latencyCheckMultiply = 2 * (this.latencyRandomPercentage / 100.0) * this.latencyCheckIntervalMs
+    this.latencyCheckSubtract = this.latencyCheckMultiply / 2
+
+    this.dataEmitIntervalMs = (dataEmitIntervalMs === null || dataEmitIntervalMs === 0)
+      ? undefined
+      : dataEmitIntervalMs ?? 5 * 1000 // 5s
+    log('latencyCheckIntervalMs: %s dataEmitIntervalMs: %s',
+      this.latencyCheckIntervalMs, this.dataEmitIntervalMs)
+    if (this.dataEmitIntervalMs != null) {
+      log('Expecting ~%s events per summary', this.latencyCheckIntervalMs / this.dataEmitIntervalMs)
+    } else {
+      log('Not emitting summaries')
+    }
+
+    this.asyncTestFn = asyncTestFn // If there is no asyncFn, we measure latency
+
+    // If process: use high resolution timer
+    if (globalThis.process?.hrtime != null) {
+      log('Using process.hrtime for timing')
+      this.now = globalThis.process.hrtime // eslint-disable-line no-undef
+      this.getDeltaMS = (startTime) => {
+        const hrtime = this.now(startTime)
+        return (hrtime[0] * 1000) + (hrtime[1] / 1000000)
+      }
+      // Let's try for a timer that only monotonically increases
+    } else if (typeof window !== 'undefined' && window.performance?.now != null) {
+      log('Using performance.now for timing')
+      this.now = window.performance.now.bind(window.performance)
+      this.getDeltaMS = (startTime) => Math.round(this.now() - startTime)
+    } else {
+      log('Using Date.now for timing')
+      this.now = Date.now
+      this.getDeltaMS = (startTime) => this.now() - startTime
+    }
+
+    this.latencyData = this.initLatencyData()
+  }
+
+  start () {
+    // We check for isBrowser because of browsers set max rates of timeouts when a page is hidden,
+    // so we fall back to another library
+    // See: http://stackoverflow.com/questions/6032429/chrome-timeouts-interval-suspended-in-background-tabs
+    if (isBrowser()) {
+      this.visibilityChangeEmitter = new VisibilityChangeEmitter()
+
+      this.visibilityChangeEmitter.addEventListener('visibilityChange', (evt) => {
+        const { detail: pageInFocus } = evt
+
+        if (pageInFocus) {
+          this._startTimers()
+        } else {
+          this._emitSummary()
+          this._stopTimers()
+        }
+      })
+    }
+
+    if (this.visibilityChangeEmitter?.isVisible() === true) {
+      this._startTimers()
+    }
+  }
+
+  stop () {
+    this._stopTimers()
+  }
+
+  /**
+   * Start internal timers
+   *
+   * @private
+   */
+  _startTimers () {
+    // Timer already started, ignore this
+    if (this.checkLatencyID != null) {
+      return
+    }
+
+    this.checkLatency()
+
+    if (this.dataEmitIntervalMs != null) {
+      this.emitIntervalID = setInterval(() => this._emitSummary(), this.dataEmitIntervalMs)
+      if (typeof this.emitIntervalID.unref === 'function') {
+        this.emitIntervalID.unref() // Doesn't block exit
+      }
+    }
+  }
+
+  /**
+   * Stop internal timers
+   *
+   * @private
+   */
+  _stopTimers () {
+    if (this.checkLatencyID != null) {
+      clearTimeout(this.checkLatencyID)
+      this.checkLatencyID = undefined
+    }
+    if (this.emitIntervalID != null) {
+      clearInterval(this.emitIntervalID)
+      this.emitIntervalID = undefined
+    }
+  }
+
+  /**
+   * Emit summary only if there were events. It might not have any events if it was forced via a page hidden/show
+   *
+   * @private
+   */
+  _emitSummary () {
+    const summary = this.getSummary()
+    if (summary.events > 0) {
+      this.dispatchEvent(new CustomEvent<SummaryObject>('data', {
+        detail: summary
+      }))
+    }
+  }
+
+  /**
+   * Calling this function will end the collection period. If a timing event was already fired and somewhere in the queue,
+   * it will not count for this time period
+   */
+  getSummary (): SummaryObject {
+    // We might want to adjust for the number of expected events
+    // Example: first 1 event it comes back, then such a long blocker that the next emit check comes
+    // Then this fires - looks like no latency!!
+    const latency = {
+      events: this.latencyData.events,
+      minMs: this.latencyData.minMs,
+      maxMs: this.latencyData.maxMs,
+      avgMs: this.latencyData.events > 0
+        ? this.latencyData.totalMs / this.latencyData.events
+        : Number.POSITIVE_INFINITY,
+      lengthMs: this.getDeltaMS(this.latencyData.startTime)
+    }
+    this.latencyData = this.initLatencyData() // Clear
+
+    log.trace('Summary: %O', latency)
+    return latency
+  }
+
+  /**
+   * Randomly calls an async fn every roughly latencyCheckIntervalMs (plus some randomness). If no async fn is found,
+   * it will simply report on event loop latency.
+   */
+  checkLatency () {
+    // Randomness is needed to avoid alignment by accident to regular things in the event loop
+    const randomness = (Math.random() * this.latencyCheckMultiply) - this.latencyCheckSubtract
+
+    // We use this to ensure that in case some overlap somehow, we don't take the wrong startTime/offset
+    const localData = {
+      deltaOffset: Math.ceil(this.latencyCheckIntervalMs + randomness),
+      startTime: this.now()
+    }
+
+    const cb = () => {
+      // We are already stopped, ignore this datapoint
+      if (this.checkLatencyID == null) {
+        return
+      }
+      const deltaMS = this.getDeltaMS(localData.startTime) - localData.deltaOffset
+      this.checkLatency() // Start again ASAP
+
+      // Add the data point. If this gets complex, refactor it
+      this.latencyData.events++
+      this.latencyData.minMs = Math.min(this.latencyData.minMs, deltaMS)
+      this.latencyData.maxMs = Math.max(this.latencyData.maxMs, deltaMS)
+      this.latencyData.totalMs += deltaMS
+      log.trace('MS: %s Data: %O', deltaMS, this.latencyData)
+    }
+    log.trace('localData: %O', localData)
+
+    this.checkLatencyID = setTimeout(() => {
+      // This gets rid of including event loop
+      if (this.asyncTestFn != null) {
+        // Clear timing related things
+        localData.deltaOffset = 0
+        localData.startTime = this.now()
+        this.asyncTestFn(cb)
+      } else {
+        // setTimeout is not more accurate than 1ms, so this will ensure positive numbers. Add 1 to emitted data to remove.
+        // This is not the best, but for now it'll be just fine. This isn't meant to be sub ms accurate.
+        localData.deltaOffset -= 1
+        // If there is no function to test, we mean check latency which is a special case that is really cb => cb()
+        // We avoid that for the few extra function all overheads. Also, we want to keep the timers different
+        cb()
+      }
+    }, localData.deltaOffset)
+
+    if (typeof this.checkLatencyID.unref === 'function') {
+      this.checkLatencyID.unref() // Doesn't block exit
+    }
+  }
+
+  initLatencyData (): LatencyData {
+    return {
+      startTime: this.now(),
+      minMs: Number.POSITIVE_INFINITY,
+      maxMs: Number.NEGATIVE_INFINITY,
+      events: 0,
+      totalMs: 0
+    }
+  }
+}
+
+function isBrowser () {
+  return typeof globalThis.window !== 'undefined'
+}

package/src/connection-manager/{visibility-change-emitter.js → visibility-change-emitter.ts}

@@ -1,14 +1,17 @@
-// @ts-nocheck
-/* global document */
-
 /**
  * This code is based on `latency-monitor` (https://github.com/mlucool/latency-monitor) by `mlucool` (https://github.com/mlucool), available under Apache License 2.0 (https://github.com/mlucool/latency-monitor/blob/master/LICENSE)
  */
-'use strict'
 
-const { EventEmitter } = require('events')
+import { CustomEvent, EventEmitter } from '@libp2p/interfaces'
+import { logger } from '@libp2p/logger'
+
+const log = logger('libp2p:connection-manager:latency-monitor:visibility-change-emitter')
+
+interface VisibilityChangeEmitterEvents {
+  'visibilityChange': CustomEvent<boolean>
+}
 
-const debug = require('debug')('latency-monitor:VisibilityChangeEmitter')
+type Hidden = 'hidden' | 'mozHidden' | 'msHidden' | 'webkitHidden'
 
 /**
  * Listen to page visibility change events (i.e. when the page is focused / blurred) by an event emitter.
@@ -32,20 +35,20 @@ const debug = require('debug')('latency-monitor:VisibilityChangeEmitter')
  *     // To access the visibility state directly, call:
  *     console.log('Am I focused now? ' + myVisibilityEmitter.isVisible());
  */
-class VisibilityChangeEmitter extends EventEmitter {
-  /**
-   * Creates a VisibilityChangeEmitter
-   *
-   * @class
-   */
+export class VisibilityChangeEmitter extends EventEmitter<VisibilityChangeEmitterEvents> {
+  private hidden: Hidden
+  private visibilityChange: string
+
   constructor () {
     super()
-    if (typeof document === 'undefined') {
-      debug('This is not a browser, no "document" found. Stopping.')
-      return
+
+    this.hidden = 'hidden'
+    this.visibilityChange = 'visibilityChange'
+
+    if (globalThis.document != null) {
+      this._initializeVisibilityVarNames()
+      this._addVisibilityChangeListener()
     }
-    this._initializeVisibilityVarNames()
-    this._addVisibilityChangeListener()
   }
 
   /**
@@ -58,23 +61,28 @@ class VisibilityChangeEmitter extends EventEmitter {
    * @private
    */
   _initializeVisibilityVarNames () {
-    let hidden
-    let visibilityChange
-    if (typeof document.hidden !== 'undefined') { // Opera 12.10 and Firefox 18 and later support
+    let hidden: Hidden = 'hidden'
+    let visibilityChange = 'visibilitychange'
+
+    if (typeof globalThis.document.hidden !== 'undefined') { // Opera 12.10 and Firefox 18 and later support
       hidden = 'hidden'
       visibilityChange = 'visibilitychange'
-    } else if (typeof document.mozHidden !== 'undefined') {
+      // @ts-expect-error mozHidden is a non-standard field name
+    } else if (typeof globalThis.document.mozHidden !== 'undefined') {
       hidden = 'mozHidden'
       visibilityChange = 'mozvisibilitychange'
-    } else if (typeof document.msHidden !== 'undefined') {
+      // @ts-expect-error msHidden is a non-standard field name
+    } else if (typeof globalThis.document.msHidden !== 'undefined') {
       hidden = 'msHidden'
       visibilityChange = 'msvisibilitychange'
-    } else if (typeof document.webkitHidden !== 'undefined') {
+      // @ts-expect-error webkitHidden is a non-standard field name
+    } else if (typeof globalThis.document.webkitHidden !== 'undefined') {
       hidden = 'webkitHidden'
       visibilityChange = 'webkitvisibilitychange'
     }
-    this._hidden = hidden
-    this._visibilityChange = visibilityChange
+
+    this.hidden = hidden
+    this.visibilityChange = visibilityChange
   }
 
   /**
@@ -84,27 +92,27 @@ class VisibilityChangeEmitter extends EventEmitter {
    * @private
    */
   _addVisibilityChangeListener () {
-    if (typeof document.addEventListener === 'undefined' ||
-            typeof document[this._hidden] === 'undefined') {
-      debug('Checking page visibility requires a browser that supports the Page Visibility API.')
+    // @ts-expect-error cannot index document object with string key
+    if (typeof globalThis.document.addEventListener === 'undefined' || typeof document[this.hidden] === 'undefined') {
+      log('Checking page visibility requires a browser that supports the Page Visibility API.')
     } else {
       // Handle page visibility change
-      document.addEventListener(this._visibilityChange, this._handleVisibilityChange.bind(this), false)
+      globalThis.document.addEventListener(this.visibilityChange, this._handleVisibilityChange.bind(this), false)
     }
   }
 
   /**
    * The function returns ```true``` if the page is visible or ```false``` if the page is not visible and
    * ```undefined``` if the page visibility API is not supported by the browser.
-   *
-   * @returns {boolean | void} whether the page is now visible or not (undefined is unknown)
    */
   isVisible () {
-    if (this._hidden === undefined || document[this._hidden] === undefined) {
+    // @ts-expect-error cannot index document object with string key
+    if (this.hidden === undefined || document[this.hidden] === undefined) {
       return undefined
     }
 
-    return !document[this._hidden]
+    // @ts-expect-error cannot index document object with string key
+    return document[this.hidden] == null
   }
 
   /**
@@ -115,11 +123,13 @@ class VisibilityChangeEmitter extends EventEmitter {
    * @private
    */
   _handleVisibilityChange () {
-    const visible = !document[this._hidden]
-    debug(visible ? 'Page Visible' : 'Page Hidden')
+    // @ts-expect-error cannot index document object with string key
+    const visible = globalThis.document[this.hidden] === false
+    log(visible ? 'Page Visible' : 'Page Hidden')
+
     // Emit the event
-    this.emit('visibilityChange', visible)
+    this.dispatchEvent(new CustomEvent<boolean>('visibilityChange', {
+      detail: visible
+    }))
   }
 }
-
-module.exports = VisibilityChangeEmitter

package/src/constants.ts

@@ -0,0 +1,31 @@
+
+/**
+ * How long in ms a dial attempt is allowed to take
+ */
+export const DIAL_TIMEOUT = 30e3
+
+/**
+ * Maximum allowed concurrent dials
+ */
+export const MAX_PARALLEL_DIALS = 100
+
+/**
+ * Allowed parallel dials per DialRequest
+ */
+export const MAX_PER_PEER_DIALS = 4
+
+/**
+ * Maximum number of allowed addresses to attempt to dial
+ */
+export const MAX_ADDRS_TO_DIAL = 25
+
+export const METRICS = {
+  computeThrottleMaxQueueSize: 1000,
+  computeThrottleTimeout: 2000,
+  movingAverageIntervals: [
+    60 * 1000, // 1 minute
+    5 * 60 * 1000, // 5 minutes
+    15 * 60 * 1000 // 15 minutes
+  ],
+  maxOldPeersRetention: 50
+}

package/src/content-routing/index.ts

@@ -0,0 +1,143 @@
+import errCode from 'err-code'
+import { messages, codes } from '../errors.js'
+import {
+  storeAddresses,
+  uniquePeers,
+  requirePeers
+} from './utils.js'
+import drain from 'it-drain'
+import merge from 'it-merge'
+import { pipe } from 'it-pipe'
+import type { ContentRouting } from '@libp2p/interfaces/content-routing'
+import type { AbortOptions, Startable } from '@libp2p/interfaces'
+import type { CID } from 'multiformats/cid'
+import type { Components } from '@libp2p/interfaces/components'
+
+export interface CompoundContentRoutingInit {
+  routers: ContentRouting[]
+}
+
+export class CompoundContentRouting implements ContentRouting, Startable {
+  private readonly routers: ContentRouting[]
+  private started: boolean
+  private readonly components: Components
+
+  constructor (components: Components, init: CompoundContentRoutingInit) {
+    this.routers = init.routers ?? []
+    this.started = false
+    this.components = components
+  }
+
+  isStarted () {
+    return this.started
+  }
+
+  async start () {
+    this.started = true
+  }
+
+  async stop () {
+    this.started = false
+  }
+
+  /**
+   * Iterates over all content routers in parallel to find providers of the given key
+   */
+  async * findProviders (key: CID, options: AbortOptions = {}) {
+    if (this.routers.length === 0) {
+      throw errCode(new Error('No content this.routers available'), codes.ERR_NO_ROUTERS_AVAILABLE)
+    }
+
+    yield * pipe(
+      merge(
+        ...this.routers.map(router => router.findProviders(key, options))
+      ),
+      (source) => storeAddresses(source, this.components.getPeerStore()),
+      (source) => uniquePeers(source),
+      (source) => requirePeers(source)
+    )
+  }
+
+  /**
+   * Iterates over all content routers in parallel to notify it is
+   * a provider of the given key
+   */
+  async provide (key: CID, options: AbortOptions = {}) {
+    if (this.routers.length === 0) {
+      throw errCode(new Error('No content routers available'), codes.ERR_NO_ROUTERS_AVAILABLE)
+    }
+
+    await Promise.all(this.routers.map(async (router) => await router.provide(key, options)))
+  }
+
+  /**
+   * Store the given key/value pair in the available content routings
+   */
+  async put (key: Uint8Array, value: Uint8Array, options?: AbortOptions) {
+    if (!this.isStarted()) {
+      throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
+    }
+
+    const dht = this.components.getDHT()
+
+    if (dht != null) {
+      await drain(dht.put(key, value, options))
+    }
+  }
+
+  /**
+   * Get the value to the given key.
+   * Times out after 1 minute by default.
+   */
+  async get (key: Uint8Array, options?: AbortOptions): Promise<Uint8Array> {
+    if (!this.isStarted()) {
+      throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
+    }
+
+    const dht = this.components.getDHT()
+
+    if (dht != null) {
+      for await (const event of dht.get(key, options)) {
+        if (event.name === 'VALUE') {
+          return event.value
+        }
+      }
+    }
+
+    throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND)
+  }
+
+  /**
+   * Get the `n` values to the given key without sorting
+   */
+  async * getMany (key: Uint8Array, nVals: number, options: AbortOptions) { // eslint-disable-line require-await
+    if (!this.isStarted()) {
+      throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED)
+    }
+
+    if (nVals == null || nVals === 0) {
+      return
+    }
+
+    let gotValues = 0
+    const dht = this.components.getDHT()
+
+    if (dht != null) {
+      for await (const event of dht.get(key, options)) {
+        if (event.name === 'VALUE') {
+          yield { from: event.from, val: event.value }
+
+          gotValues++
+
+          if (gotValues === nVals) {
+            break
+          }
+        }
+      }
+    }
+
+    if (gotValues === 0) {
+      throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND)
+    }
+  }
+}

package/src/content-routing/utils.ts

@@ -0,0 +1,54 @@
+import errCode from 'err-code'
+import filter from 'it-filter'
+import map from 'it-map'
+import type { Source } from 'it-stream-types'
+import type { PeerInfo } from '@libp2p/interfaces/peer-info'
+import type { PeerStore } from '@libp2p/interfaces/peer-store'
+
+/**
+ * Store the multiaddrs from every peer in the passed peer store
+ */
+export async function * storeAddresses (source: Source<PeerInfo>, peerStore: PeerStore) {
+  yield * map(source, async (peer) => {
+    // ensure we have the addresses for a given peer
+    await peerStore.addressBook.add(peer.id, peer.multiaddrs)
+
+    return peer
+  })
+}
+
+/**
+ * Filter peers by unique peer id
+ */
+export function uniquePeers (source: Source<PeerInfo>) {
+  /** @type Set<string> */
+  const seen = new Set()
+
+  return filter(source, (peer) => {
+    // dedupe by peer id
+    if (seen.has(peer.id.toString())) {
+      return false
+    }
+
+    seen.add(peer.id.toString())
+
+    return true
+  })
+}
+
+/**
+ * Require at least `min` peers to be yielded from `source`
+ */
+export async function * requirePeers (source: Source<PeerInfo>, min: number = 1) {
+  let seen = 0
+
+  for await (const peer of source) {
+    seen++
+
+    yield peer
+  }
+
+  if (seen < min) {
+    throw errCode(new Error('not found'), 'NOT_FOUND')
+  }
+}