@libp2p/gossipsub 14.1.1-6059227cb

package/src/utils/buildRawMessage.ts

@@ -0,0 +1,174 @@
+import { randomBytes } from '@libp2p/crypto'
+import { publicKeyFromProtobuf } from '@libp2p/crypto/keys'
+import { peerIdFromMultihash } from '@libp2p/peer-id'
+import * as Digest from 'multiformats/hashes/digest'
+import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { StrictSign, StrictNoSign } from '../index.ts'
+import { RPC } from '../message/rpc.js'
+import { PublishConfigType, ValidateError } from '../types.js'
+import type { Message } from '../index.ts'
+import type { PublishConfig, TopicStr } from '../types.js'
+import type { PublicKey, PeerId } from '@libp2p/interface'
+
+export const SignPrefix = uint8ArrayFromString('libp2p-pubsub:')
+
+export interface RawMessageAndMessage {
+  raw: RPC.Message
+  msg: Message
+}
+
+export async function buildRawMessage (
+  publishConfig: PublishConfig,
+  topic: TopicStr,
+  originalData: Uint8Array,
+  transformedData: Uint8Array
+): Promise<RawMessageAndMessage> {
+  switch (publishConfig.type) {
+    case PublishConfigType.Signing: {
+      const rpcMsg: RPC.Message = {
+        from: publishConfig.author.toMultihash().bytes,
+        data: transformedData,
+        seqno: randomBytes(8),
+        topic,
+        signature: undefined, // Exclude signature field for signing
+        key: undefined // Exclude key field for signing
+      }
+
+      // Get the message in bytes, and prepend with the pubsub prefix
+      // the signature is over the bytes "libp2p-pubsub:<protobuf-message>"
+      const bytes = uint8ArrayConcat([SignPrefix, RPC.Message.encode(rpcMsg)])
+
+      rpcMsg.signature = await publishConfig.privateKey.sign(bytes)
+      rpcMsg.key = publishConfig.key
+
+      const msg: Message = {
+        type: 'signed',
+        from: publishConfig.author,
+        data: originalData,
+        sequenceNumber: BigInt(`0x${uint8ArrayToString(rpcMsg.seqno ?? new Uint8Array(0), 'base16')}`),
+        topic,
+        signature: rpcMsg.signature,
+        key: publicKeyFromProtobuf(rpcMsg.key)
+      }
+      return {
+        raw: rpcMsg,
+        msg
+      }
+    }
+
+    case PublishConfigType.Anonymous: {
+      return {
+        raw: {
+          from: undefined,
+          data: transformedData,
+          seqno: undefined,
+          topic,
+          signature: undefined,
+          key: undefined
+        },
+        msg: {
+          type: 'unsigned',
+          data: originalData,
+          topic
+        }
+      }
+    }
+
+    default:
+      throw new Error('Unreachable')
+  }
+}
+
+export type ValidationResult = { valid: true, message: Message } | { valid: false, error: ValidateError }
+
+export async function validateToRawMessage (
+  signaturePolicy: typeof StrictNoSign | typeof StrictSign,
+  msg: RPC.Message
+): Promise<ValidationResult> {
+  // If strict-sign, verify all
+  // If anonymous (no-sign), ensure no preven
+
+  switch (signaturePolicy) {
+    case StrictNoSign:
+      if (msg.signature != null) { return { valid: false, error: ValidateError.SignaturePresent } }
+      if (msg.seqno != null) { return { valid: false, error: ValidateError.SeqnoPresent } }
+      if (msg.key != null) { return { valid: false, error: ValidateError.FromPresent } }
+
+      return { valid: true, message: { type: 'unsigned', topic: msg.topic, data: msg.data ?? new Uint8Array(0) } }
+
+    case StrictSign: {
+      // Verify seqno
+      if (msg.seqno == null) { return { valid: false, error: ValidateError.InvalidSeqno } }
+      if (msg.seqno.length !== 8) {
+        return { valid: false, error: ValidateError.InvalidSeqno }
+      }
+
+      if (msg.signature == null) { return { valid: false, error: ValidateError.InvalidSignature } }
+      if (msg.from == null) { return { valid: false, error: ValidateError.InvalidPeerId } }
+
+      let fromPeerId: PeerId
+      try {
+        // TODO: Fix PeerId types
+        fromPeerId = peerIdFromMultihash(Digest.decode(msg.from))
+      } catch (e) {
+        return { valid: false, error: ValidateError.InvalidPeerId }
+      }
+
+      // - check from defined
+      // - transform source to PeerId
+      // - parse signature
+      // - get .key, else from source
+      // - check key == source if present
+      // - verify sig
+
+      let publicKey: PublicKey
+      if (msg.key != null) {
+        publicKey = publicKeyFromProtobuf(msg.key)
+        // TODO: Should `fromPeerId.pubKey` be optional?
+        if (fromPeerId.publicKey !== undefined && !publicKey.equals(fromPeerId.publicKey)) {
+          return { valid: false, error: ValidateError.InvalidPeerId }
+        }
+      } else {
+        if (fromPeerId.publicKey == null) {
+          return { valid: false, error: ValidateError.InvalidPeerId }
+        }
+        publicKey = fromPeerId.publicKey
+      }
+
+      const rpcMsgPreSign: RPC.Message = {
+        from: msg.from,
+        data: msg.data,
+        seqno: msg.seqno,
+        topic: msg.topic,
+        signature: undefined, // Exclude signature field for signing
+        key: undefined // Exclude key field for signing
+      }
+
+      // Get the message in bytes, and prepend with the pubsub prefix
+      // the signature is over the bytes "libp2p-pubsub:<protobuf-message>"
+      const bytes = uint8ArrayConcat([SignPrefix, RPC.Message.encode(rpcMsgPreSign)])
+
+      if (!(await publicKey.verify(bytes, msg.signature))) {
+        return { valid: false, error: ValidateError.InvalidSignature }
+      }
+
+      return {
+        valid: true,
+        message: {
+          type: 'signed',
+          from: fromPeerId,
+          data: msg.data ?? new Uint8Array(0),
+          sequenceNumber: BigInt(`0x${uint8ArrayToString(msg.seqno, 'base16')}`),
+          topic: msg.topic,
+          signature: msg.signature,
+          key: msg.key != null ? publicKeyFromProtobuf(msg.key) : publicKey
+        }
+      }
+    }
+
+    default:
+      throw new Error('Unreachable')
+  }
+}

package/src/utils/create-gossip-rpc.ts

@@ -0,0 +1,34 @@
+import type { RPC } from '../message/rpc.js'
+
+/**
+ * Create a gossipsub RPC object
+ */
+export function createGossipRpc (messages: RPC.Message[] = [], control?: Partial<RPC.ControlMessage>): RPC {
+  return {
+    subscriptions: [],
+    messages,
+    control: control !== undefined
+      ? {
+          graft: control.graft ?? [],
+          prune: control.prune ?? [],
+          ihave: control.ihave ?? [],
+          iwant: control.iwant ?? [],
+          idontwant: control.idontwant ?? []
+        }
+      : undefined
+  }
+}
+
+export function ensureControl (rpc: RPC): Required<RPC> {
+  if (rpc.control === undefined) {
+    rpc.control = {
+      graft: [],
+      prune: [],
+      ihave: [],
+      iwant: [],
+      idontwant: []
+    }
+  }
+
+  return rpc as Required<RPC>
+}

package/src/utils/index.ts

@@ -0,0 +1,3 @@
+export * from './shuffle.js'
+export * from './messageIdToString.js'
+export { getPublishConfigFromPeerId } from './publishConfig.js'

package/src/utils/messageIdToString.ts

@@ -0,0 +1,8 @@
+import { toString } from 'uint8arrays/to-string'
+
+/**
+ * Browser friendly function to convert Uint8Array message id to base64 string.
+ */
+export function messageIdToString (msgId: Uint8Array): string {
+  return toString(msgId, 'base64')
+}

package/src/utils/msgIdFn.ts

@@ -0,0 +1,24 @@
+import { msgId } from '@libp2p/pubsub/utils'
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { Message } from '../index.js'
+
+/**
+ * Generate a message id, based on the `key` and `seqno`
+ */
+export function msgIdFnStrictSign (msg: Message): Uint8Array {
+  if (msg.type !== 'signed') {
+    throw new Error('expected signed message type')
+  }
+  // Should never happen
+  if (msg.sequenceNumber == null) { throw Error('missing seqno field') }
+
+  // TODO: Should use .from here or key?
+  return msgId(msg.from.publicKey ?? msg.key, msg.sequenceNumber)
+}
+
+/**
+ * Generate a message id, based on message `data`
+ */
+export async function msgIdFnStrictNoSign (msg: Message): Promise<Uint8Array> {
+  return sha256.encode(msg.data)
+}

package/src/utils/multiaddr.ts

@@ -0,0 +1,19 @@
+import { getNetConfig, isNetworkAddress } from '@libp2p/utils'
+import type { Multiaddr } from '@multiformats/multiaddr'
+
+export function multiaddrToIPStr (multiaddr: Multiaddr): string | null {
+  if (isNetworkAddress(multiaddr)) {
+    const config = getNetConfig(multiaddr)
+
+    switch (config.type) {
+      case 'ip4':
+      case 'ip6':
+
+        return config.host
+      default:
+        break
+    }
+  }
+
+  return null
+}

package/src/utils/publishConfig.ts

@@ -0,0 +1,33 @@
+import { publicKeyToProtobuf } from '@libp2p/crypto/keys'
+import { StrictSign, StrictNoSign } from '../index.ts'
+import { PublishConfigType } from '../types.js'
+import type { PublishConfig } from '../types.js'
+import type { PeerId, PrivateKey } from '@libp2p/interface'
+
+/**
+ * Prepare a PublishConfig object from a PeerId.
+ */
+export function getPublishConfigFromPeerId (
+  signaturePolicy: typeof StrictSign | typeof StrictNoSign,
+  peerId: PeerId,
+  privateKey: PrivateKey
+): PublishConfig {
+  switch (signaturePolicy) {
+    case StrictSign: {
+      return {
+        type: PublishConfigType.Signing,
+        author: peerId,
+        key: publicKeyToProtobuf(privateKey.publicKey),
+        privateKey
+      }
+    }
+
+    case StrictNoSign:
+      return {
+        type: PublishConfigType.Anonymous
+      }
+
+    default:
+      throw new Error(`Unknown signature policy "${signaturePolicy}"`)
+  }
+}

package/src/utils/set.ts

@@ -0,0 +1,43 @@
+/**
+ * Exclude up to `ineed` items from a set if item meets condition `cond`
+ */
+export function removeItemsFromSet<T> (
+  superSet: Set<T>,
+  ineed: number,
+  cond: (peer: T) => boolean = () => true
+): Set<T> {
+  const subset = new Set<T>()
+  if (ineed <= 0) { return subset }
+
+  for (const id of superSet) {
+    if (subset.size >= ineed) { break }
+    if (cond(id)) {
+      subset.add(id)
+      superSet.delete(id)
+    }
+  }
+
+  return subset
+}
+
+/**
+ * Exclude up to `ineed` items from a set
+ */
+export function removeFirstNItemsFromSet<T> (superSet: Set<T>, ineed: number): Set<T> {
+  return removeItemsFromSet(superSet, ineed, () => true)
+}
+
+export class MapDef<K, V> extends Map<K, V> {
+  constructor (private readonly getDefault: () => V) {
+    super()
+  }
+
+  getOrDefault (key: K): V {
+    let value = super.get(key)
+    if (value === undefined) {
+      value = this.getDefault()
+      this.set(key, value)
+    }
+    return value
+  }
+}

package/src/utils/shuffle.ts

@@ -0,0 +1,21 @@
+/**
+ * Pseudo-randomly shuffles an array
+ *
+ * Mutates the input array
+ */
+export function shuffle<T> (arr: T[]): T[] {
+  if (arr.length <= 1) {
+    return arr
+  }
+  const randInt = (): number => {
+    return Math.floor(Math.random() * Math.floor(arr.length))
+  }
+
+  for (let i = 0; i < arr.length; i++) {
+    const j = randInt()
+    const tmp = arr[i]
+    arr[i] = arr[j]
+    arr[j] = tmp
+  }
+  return arr
+}

package/src/utils/time-cache.ts

@@ -0,0 +1,71 @@
+interface SimpleTimeCacheOpts {
+  validityMs: number
+}
+
+interface CacheValue<T> {
+  value: T
+  validUntilMs: number
+}
+
+/**
+ * This is similar to https://github.com/daviddias/time-cache/blob/master/src/index.js
+ * for our own need, we don't use lodash throttle to improve performance.
+ * This gives 4x - 5x performance gain compared to npm TimeCache
+ */
+export class SimpleTimeCache<T> {
+  private readonly entries = new Map<string | number, CacheValue<T>>()
+  private readonly validityMs: number
+
+  constructor (opts: SimpleTimeCacheOpts) {
+    this.validityMs = opts.validityMs
+
+    // allow negative validityMs so that this does not cache anything, spec test compliance.spec.js
+    // sends duplicate messages and expect peer to receive all. Application likely uses positive validityMs
+  }
+
+  get size (): number {
+    return this.entries.size
+  }
+
+  /** Returns true if there was a key collision and the entry is dropped */
+  put (key: string | number, value: T): boolean {
+    if (this.entries.has(key)) {
+      // Key collisions break insertion order in the entries cache, which break prune logic.
+      // prune relies on each iterated entry to have strictly ascending validUntilMs, else it
+      // won't prune expired entries and SimpleTimeCache will grow unexpectedly.
+      // As of Oct 2022 NodeJS v16, inserting the same key twice with different value does not
+      // change the key position in the iterator stream. A unit test asserts this behaviour.
+      return true
+    }
+
+    this.entries.set(key, { value, validUntilMs: Date.now() + this.validityMs })
+    return false
+  }
+
+  prune (): void {
+    const now = Date.now()
+
+    for (const [k, v] of this.entries.entries()) {
+      if (v.validUntilMs < now) {
+        this.entries.delete(k)
+      } else {
+        // Entries are inserted with strictly ascending validUntilMs.
+        // Stop early to save iterations
+        break
+      }
+    }
+  }
+
+  has (key: string): boolean {
+    return this.entries.has(key)
+  }
+
+  get (key: string | number): T | undefined {
+    const value = this.entries.get(key)
+    return (value != null) && value.validUntilMs >= Date.now() ? value.value : undefined
+  }
+
+  clear (): void {
+    this.entries.clear()
+  }
+}