@libp2p/gossipsub 14.1.1-6059227cb

package/src/score/scoreMetrics.ts

@@ -0,0 +1,215 @@
+import type { PeerScoreParams } from './peer-score-params.js'
+import type { PeerStats } from './peer-stats.js'
+
+type TopicLabel = string
+type TopicStr = string
+type TopicStrToLabel = Map<TopicStr, TopicLabel>
+
+export interface TopicScoreWeights<T> {
+  p1w: T
+  p2w: T
+  p3w: T
+  p3bw: T
+  p4w: T
+}
+export interface ScoreWeights<T> {
+  byTopic: Map<TopicLabel, TopicScoreWeights<T>>
+  p5w: T
+  p6w: T
+  p7w: T
+  score: T
+}
+
+export function computeScoreWeights (
+  peer: string,
+  pstats: PeerStats,
+  params: PeerScoreParams,
+  peerIPs: Map<string, Set<string>>,
+  topicStrToLabel: TopicStrToLabel
+): ScoreWeights<number> {
+  let score = 0
+
+  const byTopic = new Map<TopicLabel, TopicScoreWeights<number>>()
+
+  // topic stores
+  Object.entries(pstats.topics).forEach(([topic, tstats]) => {
+    // the topic parameters
+    // Aggregate by known topicLabel or throw to 'unknown'. This prevent too high cardinality
+    const topicLabel = topicStrToLabel.get(topic) ?? 'unknown'
+    const topicParams = params.topics[topic]
+    if (topicParams === undefined) {
+      // we are not scoring this topic
+      return
+    }
+
+    let topicScores = byTopic.get(topicLabel)
+    if (topicScores == null) {
+      topicScores = {
+        p1w: 0,
+        p2w: 0,
+        p3w: 0,
+        p3bw: 0,
+        p4w: 0
+      }
+      byTopic.set(topicLabel, topicScores)
+    }
+
+    let p1w = 0
+    let p2w = 0
+    let p3w = 0
+    let p3bw = 0
+    let p4w = 0
+
+    // P1: time in Mesh
+    if (tstats.inMesh) {
+      const p1 = Math.max(tstats.meshTime / topicParams.timeInMeshQuantum, topicParams.timeInMeshCap)
+      p1w += p1 * topicParams.timeInMeshWeight
+    }
+
+    // P2: first message deliveries
+    let p2 = tstats.firstMessageDeliveries
+    if (p2 > topicParams.firstMessageDeliveriesCap) {
+      p2 = topicParams.firstMessageDeliveriesCap
+    }
+    p2w += p2 * topicParams.firstMessageDeliveriesWeight
+
+    // P3: mesh message deliveries
+    if (
+      tstats.meshMessageDeliveriesActive &&
+      tstats.meshMessageDeliveries < topicParams.meshMessageDeliveriesThreshold
+    ) {
+      const deficit = topicParams.meshMessageDeliveriesThreshold - tstats.meshMessageDeliveries
+      const p3 = deficit * deficit
+      p3w += p3 * topicParams.meshMessageDeliveriesWeight
+    }
+
+    // P3b:
+    // NOTE: the weight of P3b is negative (validated in validateTopicScoreParams) so this detracts
+    const p3b = tstats.meshFailurePenalty
+    p3bw += p3b * topicParams.meshFailurePenaltyWeight
+
+    // P4: invalid messages
+    // NOTE: the weight of P4 is negative (validated in validateTopicScoreParams) so this detracts
+    const p4 = tstats.invalidMessageDeliveries * tstats.invalidMessageDeliveries
+    p4w += p4 * topicParams.invalidMessageDeliveriesWeight
+
+    // update score, mixing with topic weight
+    score += (p1w + p2w + p3w + p3bw + p4w) * topicParams.topicWeight
+
+    topicScores.p1w += p1w
+    topicScores.p2w += p2w
+    topicScores.p3w += p3w
+    topicScores.p3bw += p3bw
+    topicScores.p4w += p4w
+  })
+
+  // apply the topic score cap, if any
+  if (params.topicScoreCap > 0 && score > params.topicScoreCap) {
+    score = params.topicScoreCap
+
+    // Proportionally apply cap to all individual contributions
+    const capF = params.topicScoreCap / score
+    for (const ws of byTopic.values()) {
+      ws.p1w *= capF
+      ws.p2w *= capF
+      ws.p3w *= capF
+      ws.p3bw *= capF
+      ws.p4w *= capF
+    }
+  }
+
+  let p5w = 0
+  let p6w = 0
+  let p7w = 0
+
+  // P5: application-specific score
+  const p5 = params.appSpecificScore(peer)
+  p5w += p5 * params.appSpecificWeight
+
+  // P6: IP colocation factor
+  pstats.knownIPs.forEach((ip) => {
+    if (params.IPColocationFactorWhitelist.has(ip)) {
+      return
+    }
+
+    // P6 has a cliff (IPColocationFactorThreshold)
+    // It's only applied if at least that many peers are connected to us from that source IP addr.
+    // It is quadratic, and the weight is negative (validated in validatePeerScoreParams)
+    const peersInIP = peerIPs.get(ip)
+    const numPeersInIP = (peersInIP != null) ? peersInIP.size : 0
+    if (numPeersInIP > params.IPColocationFactorThreshold) {
+      const surplus = numPeersInIP - params.IPColocationFactorThreshold
+      const p6 = surplus * surplus
+      p6w += p6 * params.IPColocationFactorWeight
+    }
+  })
+
+  // P7: behavioural pattern penalty
+  const p7 = pstats.behaviourPenalty * pstats.behaviourPenalty
+  p7w += p7 * params.behaviourPenaltyWeight
+
+  score += p5w + p6w + p7w
+
+  return {
+    byTopic,
+    p5w,
+    p6w,
+    p7w,
+    score
+  }
+}
+
+export function computeAllPeersScoreWeights (
+  peerIdStrs: Iterable<string>,
+  peerStats: Map<string, PeerStats>,
+  params: PeerScoreParams,
+  peerIPs: Map<string, Set<string>>,
+  topicStrToLabel: TopicStrToLabel
+): ScoreWeights<number[]> {
+  const sw: ScoreWeights<number[]> = {
+    byTopic: new Map(),
+    p5w: [],
+    p6w: [],
+    p7w: [],
+    score: []
+  }
+
+  for (const peerIdStr of peerIdStrs) {
+    const pstats = peerStats.get(peerIdStr)
+    if (pstats != null) {
+      const swPeer = computeScoreWeights(peerIdStr, pstats, params, peerIPs, topicStrToLabel)
+
+      for (const [topic, swPeerTopic] of swPeer.byTopic) {
+        let swTopic = sw.byTopic.get(topic)
+        if (swTopic == null) {
+          swTopic = {
+            p1w: [],
+            p2w: [],
+            p3w: [],
+            p3bw: [],
+            p4w: []
+          }
+          sw.byTopic.set(topic, swTopic)
+        }
+
+        swTopic.p1w.push(swPeerTopic.p1w)
+        swTopic.p2w.push(swPeerTopic.p2w)
+        swTopic.p3w.push(swPeerTopic.p3w)
+        swTopic.p3bw.push(swPeerTopic.p3bw)
+        swTopic.p4w.push(swPeerTopic.p4w)
+      }
+
+      sw.p5w.push(swPeer.p5w)
+      sw.p6w.push(swPeer.p6w)
+      sw.p7w.push(swPeer.p7w)
+      sw.score.push(swPeer.score)
+    } else {
+      sw.p5w.push(0)
+      sw.p6w.push(0)
+      sw.p7w.push(0)
+      sw.score.push(0)
+    }
+  }
+
+  return sw
+}

package/src/stream.ts

@@ -0,0 +1,79 @@
+import { pipe } from '@libp2p/utils'
+import { encode, decode } from 'it-length-prefixed'
+import type { AbortOptions, Stream } from '@libp2p/interface'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+interface OutboundStreamOpts {
+  /** Max size in bytes for pushable buffer. If full, will throw on .push */
+  maxBufferSize?: number
+}
+
+interface InboundStreamOpts {
+  /** Max size in bytes for reading messages from the stream */
+  maxDataLength?: number
+}
+
+export class OutboundStream {
+  constructor (private readonly rawStream: Stream, errCallback: (e: Error) => void, opts: OutboundStreamOpts) {
+    if (opts.maxBufferSize != null) {
+      rawStream.maxWriteBufferLength = opts.maxBufferSize
+    }
+
+    rawStream.addEventListener('close', (evt) => {
+      if (evt.error != null) {
+        errCallback(evt.error)
+      }
+    })
+  }
+
+  get protocol (): string {
+    return this.rawStream.protocol
+  }
+
+  async push (data: Uint8Array): Promise<void> {
+    return this.pushPrefixed(encode.single(data))
+  }
+
+  /**
+   * Same to push() but this is prefixed data so no need to encode length prefixed again
+   */
+  pushPrefixed (data: Uint8ArrayList): void {
+    // TODO: backpressure
+    this.rawStream.send(data)
+  }
+
+  async close (options?: AbortOptions): Promise<void> {
+    await this.rawStream.close(options)
+      .catch(err => {
+        this.rawStream.abort(err)
+      })
+  }
+}
+
+export class InboundStream {
+  public readonly source: AsyncIterable<Uint8ArrayList>
+
+  private readonly rawStream: Stream
+  private readonly closeController: AbortController
+
+  constructor (rawStream: Stream, opts: InboundStreamOpts = {}) {
+    this.rawStream = rawStream
+    this.closeController = new AbortController()
+
+    this.closeController.signal.addEventListener('abort', () => {
+      rawStream.close()
+        .catch(err => {
+          rawStream.abort(err)
+        })
+    })
+
+    this.source = pipe(
+      this.rawStream,
+      (source) => decode(source, opts)
+    )
+  }
+
+  async close (): Promise<void> {
+    this.closeController.abort()
+  }
+}

package/src/tracer.ts

@@ -0,0 +1,177 @@
+import { RejectReason } from './types.js'
+import type { Metrics } from './metrics.js'
+import type { MsgIdStr, MsgIdToStrFn, PeerIdStr } from './types.js'
+
+/**
+ * IWantTracer is an internal tracer that tracks IWANT requests in order to penalize
+ * peers who don't follow up on IWANT requests after an IHAVE advertisement.
+ * The tracking of promises is probabilistic to avoid using too much memory.
+ *
+ * Note: Do not confuse these 'promises' with JS Promise objects.
+ * These 'promises' are merely expectations of a peer's behavior.
+ */
+export class IWantTracer {
+  /**
+   * Promises to deliver a message
+   * Map per message id, per peer, promise expiration time
+   */
+  private readonly promises = new Map<MsgIdStr, Map<PeerIdStr, number>>()
+  /**
+   * First request time by msgId. Used for metrics to track expire times.
+   * Necessary to know if peers are actually breaking promises or simply sending them a bit later
+   */
+  private readonly requestMsByMsg = new Map<MsgIdStr, number>()
+  private readonly requestMsByMsgExpire: number
+
+  constructor (
+    private readonly gossipsubIWantFollowupMs: number,
+    private readonly msgIdToStrFn: MsgIdToStrFn,
+    private readonly metrics: Metrics | null
+  ) {
+    this.requestMsByMsgExpire = 10 * gossipsubIWantFollowupMs
+  }
+
+  get size (): number {
+    return this.promises.size
+  }
+
+  get requestMsByMsgSize (): number {
+    return this.requestMsByMsg.size
+  }
+
+  /**
+   * Track a promise to deliver a message from a list of msgIds we are requesting
+   */
+  addPromise (from: PeerIdStr, msgIds: Uint8Array[]): void {
+    // pick msgId randomly from the list
+    const ix = Math.floor(Math.random() * msgIds.length)
+    const msgId = msgIds[ix]
+    const msgIdStr = this.msgIdToStrFn(msgId)
+
+    let expireByPeer = this.promises.get(msgIdStr)
+    if (expireByPeer == null) {
+      expireByPeer = new Map()
+      this.promises.set(msgIdStr, expireByPeer)
+    }
+
+    const now = Date.now()
+
+    // If a promise for this message id and peer already exists we don't update the expiry
+    if (!expireByPeer.has(from)) {
+      expireByPeer.set(from, now + this.gossipsubIWantFollowupMs)
+
+      if (this.metrics != null) {
+        this.metrics.iwantPromiseStarted.inc(1)
+        if (!this.requestMsByMsg.has(msgIdStr)) {
+          this.requestMsByMsg.set(msgIdStr, now)
+        }
+      }
+    }
+  }
+
+  /**
+   * Returns the number of broken promises for each peer who didn't follow up on an IWANT request.
+   *
+   * This should be called not too often relative to the expire times, since it iterates over the whole data.
+   */
+  getBrokenPromises (): Map<PeerIdStr, number> {
+    const now = Date.now()
+    const result = new Map<PeerIdStr, number>()
+
+    let brokenPromises = 0
+
+    this.promises.forEach((expireByPeer, msgId) => {
+      expireByPeer.forEach((expire, p) => {
+        // the promise has been broken
+        if (expire < now) {
+          // add 1 to result
+          result.set(p, (result.get(p) ?? 0) + 1)
+          // delete from tracked promises
+          expireByPeer.delete(p)
+          // for metrics
+          brokenPromises++
+        }
+      })
+      // clean up empty promises for a msgId
+      if (expireByPeer.size === 0) {
+        this.promises.delete(msgId)
+      }
+    })
+
+    this.metrics?.iwantPromiseBroken.inc(brokenPromises)
+
+    return result
+  }
+
+  /**
+   * Someone delivered a message, stop tracking promises for it
+   */
+  deliverMessage (msgIdStr: MsgIdStr, isDuplicate = false): void {
+    this.trackMessage(msgIdStr)
+
+    const expireByPeer = this.promises.get(msgIdStr)
+
+    // Expired promise, check requestMsByMsg
+    if (expireByPeer != null) {
+      this.promises.delete(msgIdStr)
+
+      if (this.metrics != null) {
+        this.metrics.iwantPromiseResolved.inc(1)
+        if (isDuplicate) { this.metrics.iwantPromiseResolvedFromDuplicate.inc(1) }
+        this.metrics.iwantPromiseResolvedPeers.inc(expireByPeer.size)
+      }
+    }
+  }
+
+  /**
+   * A message got rejected, so we can stop tracking promises and let the score penalty apply from invalid message delivery,
+   * unless its an obviously invalid message.
+   */
+  rejectMessage (msgIdStr: MsgIdStr, reason: RejectReason): void {
+    this.trackMessage(msgIdStr)
+
+    // A message got rejected, so we can stop tracking promises and let the score penalty apply.
+    // With the expection of obvious invalid messages
+    switch (reason) {
+      case RejectReason.Error:
+        return
+      default:
+        break
+    }
+
+    this.promises.delete(msgIdStr)
+  }
+
+  clear (): void {
+    this.promises.clear()
+  }
+
+  prune (): void {
+    const maxMs = Date.now() - this.requestMsByMsgExpire
+    let count = 0
+
+    for (const [k, v] of this.requestMsByMsg.entries()) {
+      if (v < maxMs) {
+        // messages that stay too long in the requestMsByMsg map, delete
+        this.requestMsByMsg.delete(k)
+        count++
+      } else {
+        // recent messages, keep them
+        // sort by insertion order
+        break
+      }
+    }
+
+    this.metrics?.iwantMessagePruned.inc(count)
+  }
+
+  private trackMessage (msgIdStr: MsgIdStr): void {
+    if (this.metrics != null) {
+      const requestMs = this.requestMsByMsg.get(msgIdStr)
+      if (requestMs !== undefined) {
+        this.metrics.iwantPromiseDeliveryTime.observe((Date.now() - requestMs) / 1000)
+        this.requestMsByMsg.delete(msgIdStr)
+      }
+    }
+  }
+}

package/src/types.ts

@@ -0,0 +1,178 @@
+import { TopicValidatorResult } from './index.ts'
+import type { Message } from './index.ts'
+import type { RPC } from './message/rpc.js'
+import type { PrivateKey, PeerId } from '@libp2p/interface'
+import type { Multiaddr } from '@multiformats/multiaddr'
+
+export type MsgIdStr = string
+export type PeerIdStr = string
+export type TopicStr = string
+export type IPStr = string
+
+export interface AddrInfo {
+  id: PeerId
+  addrs: Multiaddr[]
+}
+
+/**
+ * Compute a local non-spec'ed msg-id for faster de-duplication of seen messages.
+ * Used exclusively for a local seen_cache
+ */
+export interface FastMsgIdFn { (msg: RPC.Message): string | number }
+
+/**
+ * By default, gossipsub only provide a browser friendly function to convert Uint8Array message id to string.
+ * Application could use this option to provide a more efficient function.
+ */
+export interface MsgIdToStrFn { (msgId: Uint8Array): string }
+
+/**
+ * Compute spec'ed msg-id. Used for IHAVE / IWANT messages
+ */
+export interface MsgIdFn {
+  (msg: Message): Promise<Uint8Array> | Uint8Array
+}
+
+export interface DataTransform {
+  /**
+   * Takes the data published by peers on a topic and transforms the data.
+   * Should be the reverse of outboundTransform(). Example:
+   * - `inboundTransform()`: decompress snappy payload
+   * - `outboundTransform()`: compress snappy payload
+   */
+  inboundTransform(topic: TopicStr, data: Uint8Array): Uint8Array
+
+  /**
+   * Takes the data to be published (a topic and associated data) transforms the data. The
+   * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers.
+   */
+  outboundTransform(topic: TopicStr, data: Uint8Array): Uint8Array
+}
+
+export enum SignaturePolicy {
+  /**
+   * On the producing side:
+   * - Build messages with the signature, key (from may be enough for certain inlineable public key types), from and seqno fields.
+   *
+   * On the consuming side:
+   * - Enforce the fields to be present, reject otherwise.
+   * - Propagate only if the fields are valid and signature can be verified, reject otherwise.
+   */
+  StrictSign = 'StrictSign',
+  /**
+   * On the producing side:
+   * - Build messages without the signature, key, from and seqno fields.
+   * - The corresponding protobuf key-value pairs are absent from the marshalled message, not just empty.
+   *
+   * On the consuming side:
+   * - Enforce the fields to be absent, reject otherwise.
+   * - Propagate only if the fields are absent, reject otherwise.
+   * - A message_id function will not be able to use the above fields, and should instead rely on the data field. A commonplace strategy is to calculate a hash.
+   */
+  StrictNoSign = 'StrictNoSign'
+}
+
+export interface PublishOpts {
+  /**
+   * Do not throw `PublishError.NoPeersSubscribedToTopic` error if there are no
+   * peers listening on the topic.
+   *
+   * N.B. if you sent this option to true, and you publish a message on a topic
+   * with no peers listening on that topic, no other network node will ever
+   * receive the message.
+   */
+  allowPublishToZeroTopicPeers?: boolean
+  ignoreDuplicatePublishError?: boolean
+  /** serialize message once and send to all peers without control messages */
+  batchPublish?: boolean
+}
+
+export enum PublishConfigType {
+  Signing,
+  Anonymous
+}
+
+export type PublishConfig =
+  | {
+    type: PublishConfigType.Signing
+    author: PeerId
+    key: Uint8Array
+    privateKey: PrivateKey
+  }
+  | { type: PublishConfigType.Anonymous }
+
+export type RejectReasonObj =
+  | { reason: RejectReason.Error, error: ValidateError }
+  | { reason: Exclude<RejectReason, RejectReason.Error> }
+
+export enum RejectReason {
+  /**
+   * The message failed the configured validation during decoding.
+   * SelfOrigin is considered a ValidationError
+   */
+  Error = 'error',
+  /**
+   * Custom validator fn reported status IGNORE.
+   */
+  Ignore = 'ignore',
+  /**
+   * Custom validator fn reported status REJECT.
+   */
+  Reject = 'reject',
+  /**
+   * The peer that sent the message OR the source from field is blacklisted.
+   * Causes messages to be ignored, not penalized, neither do score record creation.
+   */
+  Blacklisted = 'blacklisted'
+}
+
+export enum ValidateError {
+  /// The message has an invalid signature,
+  InvalidSignature = 'invalid_signature',
+  /// The sequence number was the incorrect size
+  InvalidSeqno = 'invalid_seqno',
+  /// The PeerId was invalid
+  InvalidPeerId = 'invalid_peerid',
+  /// Signature existed when validation has been sent to
+  /// [`crate::behaviour::MessageAuthenticity::Anonymous`].
+  SignaturePresent = 'signature_present',
+  /// Sequence number existed when validation has been sent to
+  /// [`crate::behaviour::MessageAuthenticity::Anonymous`].
+  SeqnoPresent = 'seqno_present',
+  /// Message source existed when validation has been sent to
+  /// [`crate::behaviour::MessageAuthenticity::Anonymous`].
+  FromPresent = 'from_present',
+  /// The data transformation failed.
+  TransformFailed = 'transform_failed'
+}
+
+export enum MessageStatus {
+  duplicate = 'duplicate',
+  invalid = 'invalid',
+  valid = 'valid'
+}
+
+/**
+ * Store both Uint8Array and string message id so that we don't have to convert data between the two.
+ * See https://github.com/ChainSafe/js-libp2p-gossipsub/pull/274
+ */
+export interface MessageId {
+  msgId: Uint8Array
+  msgIdStr: MsgIdStr
+}
+
+/**
+ * Typesafe conversion of MessageAcceptance -> RejectReason. TS ensures all values covered
+ */
+export function rejectReasonFromAcceptance (
+  acceptance: Exclude<TopicValidatorResult, TopicValidatorResult.Accept>
+): RejectReason.Ignore | RejectReason.Reject {
+  switch (acceptance) {
+    case TopicValidatorResult.Ignore:
+      return RejectReason.Ignore
+    case TopicValidatorResult.Reject:
+      return RejectReason.Reject
+    default:
+      throw new Error('Unreachable')
+  }
+}