libp2p 0.40.0 → 0.41.0-19e96cc

package/src/metrics/index.ts

@@ -1,311 +0,0 @@
-import { pipe } from 'it-pipe'
-import each from 'it-foreach'
-import LRU from 'hashlru'
-import { METRICS as defaultOptions } from '../constants.js'
-import { DefaultStats, StatsInit } from './stats.js'
-import type { ComponentMetricsUpdate, Metrics, Stats, TrackedMetric, TrackStreamOptions } from '@libp2p/interface-metrics'
-import type { PeerId } from '@libp2p/interface-peer-id'
-import type { Startable } from '@libp2p/interfaces/startable'
-
-const initialCounters: ['dataReceived', 'dataSent'] = [
-  'dataReceived',
-  'dataSent'
-]
-
-const directionToEvent = {
-  in: 'dataReceived',
-  out: 'dataSent'
-}
-
-export interface OnMessageOptions {
-  remotePeer: PeerId
-  protocol?: string
-  direction: 'in' | 'out'
-  dataLength: number
-}
-
-export interface MetricsInit {
-  enabled: boolean
-  computeThrottleMaxQueueSize: number
-  computeThrottleTimeout: number
-  movingAverageIntervals: number[]
-  maxOldPeersRetention: number
-}
-
-export class DefaultMetrics implements Metrics, Startable {
-  public globalStats: DefaultStats
-
-  private readonly enabled: boolean
-  private readonly peerStats: Map<string, DefaultStats>
-  private readonly protocolStats: Map<string, DefaultStats>
-  private readonly oldPeers: ReturnType<typeof LRU>
-  private running: boolean
-  private readonly systems: Map<string, Map<string, Map<string, TrackedMetric>>>
-  private readonly statsInit: StatsInit
-
-  constructor (init: MetricsInit) {
-    this.enabled = init.enabled
-    this.statsInit = {
-      ...init,
-      initialCounters
-    }
-    this.globalStats = new DefaultStats(this.statsInit)
-    this.peerStats = new Map()
-    this.protocolStats = new Map()
-    this.oldPeers = LRU(init.maxOldPeersRetention ?? defaultOptions.maxOldPeersRetention)
-    this.running = false
-    this._onMessage = this._onMessage.bind(this)
-    this.systems = new Map()
-  }
-
-  isStarted () {
-    return this.running
-  }
-
-  /**
-   * Must be called for stats to saved. Any data pushed for tracking
-   * will be ignored.
-   */
-  async start () {
-    if (!this.enabled) {
-      return
-    }
-
-    this.running = true
-  }
-
-  /**
-   * Stops all averages timers and prevents new data from being tracked.
-   * Once `stop` is called, `start` must be called to resume stats tracking.
-   */
-  async stop () {
-    if (!this.running) {
-      return
-    }
-
-    this.running = false
-    this.globalStats.stop()
-
-    for (const stats of this.peerStats.values()) {
-      stats.stop()
-    }
-
-    for (const stats of this.protocolStats.values()) {
-      stats.stop()
-    }
-  }
-
-  /**
-   * Gets the global `Stats` object
-   */
-  getGlobal () {
-    return this.globalStats
-  }
-
-  /**
-   * Returns a list of `PeerId` strings currently being tracked
-   */
-  getPeers () {
-    return Array.from(this.peerStats.keys())
-  }
-
-  getComponentMetrics () {
-    return this.systems
-  }
-
-  updateComponentMetric (update: ComponentMetricsUpdate) {
-    const { system = 'libp2p', component, metric, value, label, help } = update
-
-    if (!this.systems.has(system)) {
-      this.systems.set(system, new Map())
-    }
-
-    const systemMetrics = this.systems.get(system)
-
-    if (systemMetrics == null) {
-      throw new Error('Unknown metric system')
-    }
-
-    if (!systemMetrics.has(component)) {
-      systemMetrics.set(component, new Map())
-    }
-
-    const componentMetrics = systemMetrics.get(component)
-
-    if (componentMetrics == null) {
-      throw new Error('Unknown metric component')
-    }
-
-    componentMetrics.set(metric, {
-      label,
-      help,
-      calculate: typeof value !== 'function' ? () => value : value
-    })
-  }
-
-  /**
-   * Returns the `Stats` object for the given `PeerId` whether it
-   * is a live peer, or in the disconnected peer LRU cache.
-   */
-  forPeer (peerId: PeerId): Stats | undefined {
-    const idString = peerId.toString()
-    return this.peerStats.get(idString) ?? this.oldPeers.get(idString)
-  }
-
-  /**
-   * Returns a list of all protocol strings currently being tracked
-   */
-  getProtocols (): string[] {
-    return Array.from(this.protocolStats.keys())
-  }
-
-  /**
-   * Returns the `Stats` object for the given `protocol`
-   */
-  forProtocol (protocol: string): Stats | undefined {
-    return this.protocolStats.get(protocol)
-  }
-
-  /**
-   * Should be called when all connections to a given peer
-   * have closed. The `Stats` collection for the peer will
-   * be stopped and moved to an LRU for temporary retention.
-   */
-  onPeerDisconnected (peerId: PeerId) {
-    const idString = peerId.toString()
-    const peerStats = this.peerStats.get(idString)
-
-    if (peerStats != null) {
-      peerStats.stop()
-
-      this.peerStats.delete(idString)
-      this.oldPeers.set(idString, peerStats)
-    }
-  }
-
-  /**
-   * Takes the metadata for a message and tracks it in the
-   * appropriate categories. If the protocol is present, protocol
-   * stats will also be tracked.
-   */
-  _onMessage (opts: OnMessageOptions) {
-    if (!this.running) {
-      return
-    }
-
-    const { remotePeer, protocol, direction, dataLength } = opts
-
-    const key = directionToEvent[direction]
-
-    let peerStats = this.forPeer(remotePeer)
-    if (peerStats == null) {
-      const stats = new DefaultStats(this.statsInit)
-      this.peerStats.set(remotePeer.toString(), stats)
-      peerStats = stats
-    }
-
-    // Peer and global stats
-    peerStats.push(key, dataLength)
-    this.globalStats.push(key, dataLength)
-
-    // Protocol specific stats
-    if (protocol != null) {
-      let protocolStats = this.forProtocol(protocol)
-
-      if (protocolStats == null) {
-        const stats = new DefaultStats(this.statsInit)
-        this.protocolStats.set(protocol, stats)
-        protocolStats = stats
-      }
-
-      protocolStats.push(key, dataLength)
-    }
-  }
-
-  /**
-   * Replaces the `PeerId` string with the given `peerId`.
-   * If stats are already being tracked for the given `peerId`, the
-   * placeholder stats will be merged with the existing stats.
-   *
-   * @param {PeerId} placeholder - A peerId string
-   * @param {PeerId} peerId
-   * @returns {void}
-   */
-  updatePlaceholder (placeholder: PeerId, peerId: PeerId) {
-    if (!this.running) {
-      return
-    }
-
-    const placeholderString = placeholder.toString()
-    const placeholderStats = this.peerStats.get(placeholderString) ?? this.oldPeers.get(placeholderString)
-    const peerIdString = peerId.toString()
-    const existingStats = this.peerStats.get(peerIdString) ?? this.oldPeers.get(peerIdString)
-    let mergedStats = placeholderStats
-
-    // If we already have stats, merge the two
-    if (existingStats != null) {
-      // If existing, merge
-      mergedStats = mergeStats(existingStats, mergedStats)
-      // Attempt to delete from the old peers list just in case it was tracked there
-      this.oldPeers.remove(peerIdString)
-    }
-
-    this.peerStats.delete(placeholder.toString())
-    this.peerStats.set(peerIdString, mergedStats)
-    mergedStats.start()
-  }
-
-  /**
-   * Tracks data running through a given Duplex Iterable `stream`. If
-   * the `peerId` is not provided, a placeholder string will be created and
-   * returned. This allows lazy tracking of a peer when the peer is not yet known.
-   * When the `PeerId` is known, `Metrics.updatePlaceholder` should be called
-   * with the placeholder string returned from here, and the known `PeerId`.
-   */
-  trackStream (opts: TrackStreamOptions): void {
-    const { stream, remotePeer, protocol } = opts
-
-    if (!this.running) {
-      return
-    }
-
-    const source = stream.source
-    stream.source = each(source, chunk => this._onMessage({
-      remotePeer,
-      protocol,
-      direction: 'in',
-      dataLength: chunk.byteLength
-    }))
-
-    const sink = stream.sink
-    stream.sink = async source => {
-      return await pipe(
-        source,
-        (source) => each(source, chunk => {
-          this._onMessage({
-            remotePeer,
-            protocol,
-            direction: 'out',
-            dataLength: chunk.byteLength
-          })
-        }),
-        sink
-      )
-    }
-  }
-}
-
-/**
- * Merges `other` into `target`. `target` will be modified
- * and returned
- */
-function mergeStats (target: DefaultStats, other: DefaultStats) {
-  target.stop()
-  other.stop()
-
-  // Merge queues
-  target.queue = [...target.queue, ...other.queue]
-
-  // TODO: how to merge moving averages?
-  return target
-}

package/src/metrics/moving-average.ts

@@ -1,53 +0,0 @@
-import type { MovingAverage } from '@libp2p/interface-metrics'
-
-export class DefaultMovingAverage {
-  public movingAverage: number
-  public variance: number
-  public deviation: number
-  public forecast: number
-  private readonly timespan: number
-  private previousTime?: number
-
-  constructor (timespan: number) {
-    if (typeof timespan !== 'number') {
-      throw new Error('must provide a timespan to the moving average constructor')
-    }
-
-    if (timespan <= 0) {
-      throw new Error('must provide a timespan > 0 to the moving average constructor')
-    }
-
-    this.timespan = timespan
-    this.movingAverage = 0
-    this.variance = 0
-    this.deviation = 0
-    this.forecast = 0
-  }
-
-  alpha (t: number, pt: number) {
-    return 1 - (Math.exp(-(t - pt) / this.timespan))
-  }
-
-  push (time: number, value: number) {
-    if (this.previousTime != null) {
-      // calculate moving average
-      const a = this.alpha(time, this.previousTime)
-      const diff = value - this.movingAverage
-      const incr = a * diff
-      this.movingAverage = a * value + (1 - a) * this.movingAverage
-      // calculate variance & deviation
-      this.variance = (1 - a) * (this.variance + diff * incr)
-      this.deviation = Math.sqrt(this.variance)
-      // calculate forecast
-      this.forecast = this.movingAverage + a * diff
-    } else {
-      this.movingAverage = value
-    }
-
-    this.previousTime = time
-  }
-}
-
-export function createMovingAverage (timespan: number): MovingAverage {
-  return new DefaultMovingAverage(timespan)
-}

package/src/metrics/stats.ts

@@ -1,238 +0,0 @@
-import { CustomEvent, EventEmitter } from '@libp2p/interfaces/events'
-import { createMovingAverage } from './moving-average.js'
-// @ts-expect-error no types
-import retimer from 'retimer'
-import type { MovingAverages, Stats, TransferStats } from '@libp2p/interface-metrics'
-
-export interface StatsEvents {
-  'update': CustomEvent<TransferStats>
-}
-
-export interface StatsInit {
-  enabled: boolean
-  initialCounters: ['dataReceived', 'dataSent']
-  movingAverageIntervals: number[]
-  computeThrottleMaxQueueSize: number
-  computeThrottleTimeout: number
-}
-
-export class DefaultStats extends EventEmitter<StatsEvents> implements Stats {
-  private readonly enabled: boolean
-  public queue: Array<[string, number, number]>
-  private stats: TransferStats
-  private frequencyLastTime: number
-  private frequencyAccumulators: Record<string, number>
-  private movingAverages: MovingAverages
-  private timeout?: any
-  private readonly computeThrottleMaxQueueSize: number
-  private readonly computeThrottleTimeout: number
-  private readonly movingAverageIntervals: number[]
-
-  /**
-   * A queue based manager for stat processing
-   */
-  constructor (init: StatsInit) {
-    super()
-
-    this.enabled = init.enabled
-    this.queue = []
-    this.stats = {
-      dataReceived: 0n,
-      dataSent: 0n
-    }
-    this.frequencyLastTime = Date.now()
-    this.frequencyAccumulators = {}
-    this.movingAverages = {
-      dataReceived: [],
-      dataSent: []
-    }
-    this.computeThrottleMaxQueueSize = init.computeThrottleMaxQueueSize
-    this.computeThrottleTimeout = init.computeThrottleTimeout
-
-    this._update = this._update.bind(this)
-
-    this.movingAverageIntervals = init.movingAverageIntervals
-
-    for (let i = 0; i < init.initialCounters.length; i++) {
-      const key = init.initialCounters[i]
-      this.stats[key] = 0n
-      this.movingAverages[key] = []
-
-      for (let k = 0; k < this.movingAverageIntervals.length; k++) {
-        const interval = this.movingAverageIntervals[k]
-        const ma = this.movingAverages[key][interval] = createMovingAverage(interval)
-        ma.push(this.frequencyLastTime, 0)
-      }
-    }
-  }
-
-  /**
-   * Initializes the internal timer if there are items in the queue. This
-   * should only need to be called if `Stats.stop` was previously called, as
-   * `Stats.push` will also start the processing
-   */
-  start () {
-    if (!this.enabled) {
-      return
-    }
-
-    if (this.queue.length > 0) {
-      this._resetComputeTimeout()
-    }
-  }
-
-  /**
-   * Stops processing and computing of stats by clearing the internal
-   * timer
-   */
-  stop () {
-    if (this.timeout != null) {
-      this.timeout.clear()
-      this.timeout = null
-    }
-  }
-
-  /**
-   * Returns a clone of the current stats.
-   */
-  getSnapshot () {
-    return Object.assign({}, this.stats)
-  }
-
-  /**
-   * Returns a clone of the internal movingAverages
-   */
-  getMovingAverages (): MovingAverages {
-    return Object.assign({}, this.movingAverages)
-  }
-
-  /**
-   * Pushes the given operation data to the queue, along with the
-   * current Timestamp, then resets the update timer.
-   */
-  push (counter: string, inc: number) {
-    this.queue.push([counter, inc, Date.now()])
-    this._resetComputeTimeout()
-  }
-
-  /**
-   * Resets the timeout for triggering updates.
-   */
-  _resetComputeTimeout () {
-    this.timeout = retimer(this._update, this._nextTimeout())
-  }
-
-  /**
-   * Calculates and returns the timeout for the next update based on
-   * the urgency of the update.
-   */
-  _nextTimeout () {
-    // calculate the need for an update, depending on the queue length
-    const urgency = this.queue.length / this.computeThrottleMaxQueueSize
-    const timeout = Math.max(this.computeThrottleTimeout * (1 - urgency), 0)
-    return timeout
-  }
-
-  /**
-   * If there are items in the queue, they will will be processed and
-   * the frequency for all items will be updated based on the Timestamp
-   * of the last item in the queue. The `update` event will also be emitted
-   * with the latest stats.
-   *
-   * If there are no items in the queue, no action is taken.
-   */
-  _update () {
-    this.timeout = null
-    if (this.queue.length > 0) {
-      let last: [string, number, number] = ['', 0, 0]
-
-      for (last of this.queue) {
-        this._applyOp(last)
-      }
-
-      this.queue = []
-
-      if (last.length > 2 && last[0] !== '') {
-        this._updateFrequency(last[2]) // contains timestamp of last op
-      }
-
-      this.dispatchEvent(new CustomEvent<TransferStats>('update', {
-        detail: this.stats
-      }))
-    }
-  }
-
-  /**
-   * For each key in the stats, the frequency and moving averages
-   * will be updated via Stats._updateFrequencyFor based on the time
-   * difference between calls to this method.
-   */
-  _updateFrequency (latestTime: number) {
-    const timeDiff = latestTime - this.frequencyLastTime
-
-    this._updateFrequencyFor('dataReceived', timeDiff, latestTime)
-    this._updateFrequencyFor('dataSent', timeDiff, latestTime)
-
-    this.frequencyLastTime = latestTime
-  }
-
-  /**
-   * Updates the `movingAverages` for the given `key` and also
-   * resets the `frequencyAccumulator` for the `key`.
-   */
-  _updateFrequencyFor (key: 'dataReceived' | 'dataSent', timeDiffMS: number, latestTime: number) {
-    const count = this.frequencyAccumulators[key] ?? 0
-    this.frequencyAccumulators[key] = 0
-    // if `timeDiff` is zero, `hz` becomes Infinity, so we fallback to 1ms
-    const safeTimeDiff = timeDiffMS ?? 1
-    const hz = (count / safeTimeDiff) * 1000
-
-    let movingAverages = this.movingAverages[key]
-    if (movingAverages == null) {
-      movingAverages = this.movingAverages[key] = []
-    }
-
-    const intervals = this.movingAverageIntervals
-
-    for (let i = 0; i < intervals.length; i++) {
-      const movingAverageInterval = intervals[i]
-      let movingAverage = movingAverages[movingAverageInterval]
-      if (movingAverage == null) {
-        movingAverage = movingAverages[movingAverageInterval] = createMovingAverage(movingAverageInterval)
-      }
-      movingAverage.push(latestTime, hz)
-    }
-  }
-
-  /**
-   * For the given operation, `op`, the stats and `frequencyAccumulator`
-   * will be updated or initialized if they don't already exist.
-   */
-  _applyOp (op: [string, number, number]) {
-    const key = op[0]
-    const inc = op[1]
-
-    if (typeof inc !== 'number') {
-      throw new Error('invalid increment number')
-    }
-
-    let n: bigint
-
-    if (!Object.prototype.hasOwnProperty.call(this.stats, key)) {
-      // @ts-expect-error cannot index type with key
-      n = this.stats[key] = 0n
-    } else {
-      // @ts-expect-error cannot index type with key
-      n = this.stats[key]
-    }
-
-    // @ts-expect-error cannot index type with key
-    this.stats[key] = n + BigInt(inc)
-
-    if (this.frequencyAccumulators[key] == null) {
-      this.frequencyAccumulators[key] = 0
-    }
-
-    this.frequencyAccumulators[key] += inc
-  }
-}