@libp2p/utils 6.7.2 → 7.0.0-55b7e5fea

package/src/length-prefixed-decoder.ts

@@ -0,0 +1,98 @@
+import { InvalidParametersError } from '@libp2p/interface'
+import * as varint from 'uint8-varint'
+import { Uint8ArrayList } from 'uint8arraylist'
+import { InvalidMessageLengthError } from './stream-utils.ts'
+
+const DEFAULT_MAX_BUFFER_SIZE = 1024 * 1024 * 4
+const DEFAULT_MAX_DATA_LENGTH = 1024 * 1024 * 4
+
+export interface LengthPrefixedDecoderInit {
+  /**
+   * How large the internal buffer is allowed to grow - attempting to store more
+   * data than this will throw
+   */
+  maxBufferSize?: number
+
+  /**
+   * Throw an error if the message that would be read from the buffer is larger
+   * than this value
+   */
+  maxDataLength?: number
+
+  /**
+   * Read a varint from the buffer
+   */
+  lengthDecoder?(data: Uint8ArrayList | Uint8Array): number
+
+  /**
+   * Return how many bytes it takes to encode the passed value
+   */
+  encodingLength?(length: number): number
+}
+
+/**
+ * Decode length-prefixed data from a buffer
+ */
+export class LengthPrefixedDecoder {
+  private readonly buffer: Uint8ArrayList
+  private readonly maxBufferSize: number
+  private readonly lengthDecoder: (data: Uint8ArrayList | Uint8Array) => number
+  private readonly maxDataLength: number
+  private readonly encodingLength: (length: number) => number
+
+  constructor (init: LengthPrefixedDecoderInit = {}) {
+    this.buffer = new Uint8ArrayList()
+    this.maxBufferSize = init.maxBufferSize ?? DEFAULT_MAX_BUFFER_SIZE
+    this.maxDataLength = init.maxDataLength ?? DEFAULT_MAX_DATA_LENGTH
+    this.lengthDecoder = init.lengthDecoder ?? varint.decode
+    this.encodingLength = init.encodingLength ?? varint.encodingLength
+  }
+
+  /**
+   * Decodes length-prefixed data
+   */
+  * decode (buf: Uint8Array | Uint8ArrayList): Generator<Uint8ArrayList> {
+    this.buffer.append(buf)
+
+    if (this.buffer.byteLength > this.maxBufferSize) {
+      throw new InvalidParametersError(`Buffer length limit exceeded - ${this.buffer.byteLength}/${this.maxBufferSize}`)
+    }
+
+    // Loop to consume as many bytes from the buffer as possible
+    // Eg: when a single chunk contains several frames
+    while (true) {
+      let dataLength: number
+
+      try {
+        dataLength = this.lengthDecoder(this.buffer)
+      } catch (err) {
+        if (err instanceof RangeError) {
+          // ignore errors where we don't have enough data to read the length
+          // prefix
+          break
+        }
+
+        throw err
+      }
+
+      if (dataLength < 0 || dataLength > this.maxDataLength) {
+        throw new InvalidMessageLengthError('Invalid message length')
+      }
+
+      const lengthLength = this.encodingLength(dataLength)
+      const chunkLength = lengthLength + dataLength
+
+      if (this.buffer.byteLength >= chunkLength) {
+        const buf = this.buffer.sublist(lengthLength, chunkLength)
+
+        this.buffer.consume(chunkLength)
+
+        if (buf.byteLength > 0) {
+          yield buf
+        }
+      } else {
+        break
+      }
+    }
+  }
+}

package/src/message-queue.ts

@@ -0,0 +1,156 @@
+import { StreamMessageEvent } from '@libp2p/interface'
+import delay from 'delay'
+import { TypedEventEmitter } from 'main-event'
+import { raceSignal } from 'race-signal'
+import { isUint8ArrayList, Uint8ArrayList } from 'uint8arraylist'
+import { Queue } from './queue/index.js'
+import type { AbortOptions, Logger } from '@libp2p/interface'
+
+const DEFAULT_CHUNK_SIZE = 1024 * 64
+
+export interface MessageQueueEvents {
+  /**
+   * Message data
+   */
+  message: StreamMessageEvent
+
+  /**
+   * Emitted when the queue is empty
+   */
+  drain: Event
+
+  /**
+   * The remote closed the connection abruptly
+   */
+  reset: Event
+}
+
+export interface MessageQueueInit {
+  /**
+   * How much delay there should be between each message send in ms (note that
+   * even 0 introduces a small delay)
+   *
+   * @default 0
+   */
+  delay?: number
+
+  /**
+   * How many messages to hold in the send queue before applying backpressure to
+   * the sender
+   */
+  capacity?: number
+
+  /**
+   * Data messages larger than this size will be chunked into smaller messages.
+   *
+   * Defaults to the maximum TCP package size.
+   *
+   * @default 65_536
+   */
+  chunkSize?: number
+}
+
+interface MessageQueueJobOptions extends AbortOptions {
+  evt: Event
+}
+
+/**
+ * Accepts events to emit after a short delay, and with a configurable maximum
+ * queue capacity after which the send method will return false to let us
+ * simulate write backpressure.
+ */
+export class MessageQueue<Events> extends TypedEventEmitter<Events & MessageQueueEvents> {
+  public needsDrain: boolean
+
+  private queue: Queue<void, MessageQueueJobOptions>
+  private capacity: number
+  private delay: number
+  private log: Logger
+  private chunkSize: number
+
+  constructor (init: MessageQueueInit & { log: Logger }) {
+    super()
+
+    this.needsDrain = false
+    this.queue = new Queue({
+      concurrency: 1
+    })
+    this.capacity = init.capacity ?? 5
+    this.delay = init.delay ?? 0
+    this.log = init.log
+    this.chunkSize = init.chunkSize ?? DEFAULT_CHUNK_SIZE
+
+    this.queue.addEventListener('idle', () => {
+      if (this.needsDrain) {
+        this.log('network send queue drained')
+        this.needsDrain = false
+        this.safeDispatchEvent('drain')
+      } else {
+        this.log('network send queue idle')
+      }
+    })
+  }
+
+  send (evt: Event): boolean {
+    if (isMessageEvent(evt)) {
+      // chunk outgoing messages to match TCP packet sizes
+      const data = new Uint8ArrayList(evt.data)
+
+      while (data.byteLength > 0) {
+        const end = Math.min(this.chunkSize, data.byteLength)
+        const chunk = data.sublist(0, end)
+        data.consume(chunk.byteLength)
+
+        const chunkEvent = new StreamMessageEvent(chunk)
+
+        this.queue.add(async (opts) => {
+          if (this.delay > 0) {
+            await raceSignal(delay(this.delay), opts.signal)
+          }
+
+          this.dispatchEvent(opts.evt)
+        }, {
+          evt: chunkEvent
+        })
+      }
+    } else {
+      this.queue.add(async (opts) => {
+        if (this.delay > 0) {
+          await raceSignal(delay(this.delay), opts.signal)
+        }
+
+        this.dispatchEvent(opts.evt)
+      }, {
+        evt
+      })
+    }
+
+    if (this.queue.size >= this.capacity) {
+      this.log('network send queue full')
+      this.needsDrain = true
+      return false
+    }
+
+    return true
+  }
+
+  pause (): void {
+    this.queue.pause()
+  }
+
+  resume (): void {
+    this.queue.resume()
+  }
+
+  onIdle (): Promise<void> {
+    return this.queue.onIdle()
+  }
+
+  size (): number {
+    return this.queue.size
+  }
+}
+
+function isMessageEvent (evt?: any): evt is StreamMessageEvent {
+  return evt?.data instanceof Uint8Array || isUint8ArrayList(evt?.data)
+}

package/src/mock-muxer.ts

@@ -0,0 +1,304 @@
+import * as cborg from 'cborg'
+import * as lp from 'it-length-prefixed'
+import { pushable } from 'it-pushable'
+import { Uint8ArrayList } from 'uint8arraylist'
+import { AbstractStreamMuxer } from './abstract-stream-muxer.ts'
+import { AbstractStream } from './abstract-stream.ts'
+import { Queue } from './queue/index.js'
+import type { SendResult } from './abstract-message-stream.ts'
+import type { AbstractStreamInit } from './abstract-stream.ts'
+import type { AbortOptions, MessageStreamDirection, CreateStreamOptions, StreamMuxerFactory, StreamMuxer, MultiaddrConnection, StreamMuxerOptions } from '@libp2p/interface'
+import type { Pushable } from 'it-pushable'
+import type { SupportedEncodings } from 'uint8arrays/from-string'
+
+interface DataMessage {
+  id: string
+  type: 'data'
+  chunk: Uint8Array
+}
+
+interface ResetMessage {
+  id: string
+  type: 'reset'
+}
+
+interface CloseWriteMessage {
+  id: string
+  type: 'closeWrite'
+}
+
+interface CloseReadMessage {
+  id: string
+  type: 'closeRead'
+}
+
+interface CreateMessage {
+  id: string
+  type: 'create'
+  protocol?: string
+}
+
+interface PauseMessage {
+  id: string
+  type: 'pause'
+}
+
+interface ResumeMessage {
+  id: string
+  type: 'resume'
+}
+
+type StreamMessage = DataMessage | ResetMessage | CloseWriteMessage | CloseReadMessage | CreateMessage | PauseMessage | ResumeMessage
+
+export interface MockMuxedStreamInit extends AbstractStreamInit {
+  sendMessage(message: StreamMessage): boolean
+  encoding: SupportedEncodings
+}
+
+class MockMuxedStream extends AbstractStream {
+  private readonly sendMessage: (message: StreamMessage) => boolean
+  private dataQueue: Queue
+  private encoding: SupportedEncodings
+
+  constructor (init: MockMuxedStreamInit) {
+    super(init)
+
+    this.sendMessage = init.sendMessage
+    this.encoding = init.encoding
+    this.dataQueue = new Queue({
+      concurrency: 1
+    })
+
+    if (this.direction === 'outbound') {
+      this.sendMessage({
+        id: this.id,
+        type: 'create',
+        protocol: this.protocol
+      })
+    }
+  }
+
+  sendData (data: Uint8ArrayList): SendResult {
+    const canSendMore = this.sendMessage({
+      id: this.id,
+      type: 'data',
+      chunk: data.subarray()
+    })
+
+    return {
+      sentBytes: data.byteLength,
+      canSendMore
+    }
+  }
+
+  sendReset (): void {
+    this.sendMessage({
+      id: this.id,
+      type: 'reset'
+    })
+  }
+
+  async sendCloseWrite (options?: AbortOptions): Promise<void> {
+    this.sendMessage({
+      id: this.id,
+      type: 'closeWrite'
+    })
+
+    options?.signal?.throwIfAborted()
+  }
+
+  async sendCloseRead (options?: AbortOptions): Promise<void> {
+    this.sendMessage({
+      id: this.id,
+      type: 'closeRead'
+    })
+
+    options?.signal?.throwIfAborted()
+  }
+
+  sendPause (): void {
+    this.sendMessage({
+      id: this.id,
+      type: 'pause'
+    })
+  }
+
+  sendResume (): void {
+    this.sendMessage({
+      id: this.id,
+      type: 'resume'
+    })
+  }
+
+  onRemotePaused (): void {
+    this.dataQueue.pause()
+  }
+
+  onRemoteResumed (): void {
+    this.dataQueue.resume()
+  }
+}
+
+interface MockMuxerInit extends StreamMuxerOptions {
+  /**
+   * How long the input queue can grow
+   */
+  maxInputQueueSize?: number
+
+  /**
+   * How to encode data message
+   *
+   * @default base64
+   */
+  encoding?: SupportedEncodings
+
+  /**
+   * How large muxer messages are allowed to be
+   */
+  maxMessageSize?: number
+}
+
+// CBOR encoding of non-message data fields
+const MESSAGE_OVERHEAD = 30
+
+class MockMuxer extends AbstractStreamMuxer<MockMuxedStream> {
+  private input: Pushable<Uint8Array | Uint8ArrayList>
+  private maxInputQueueSize: number
+  private encoding: SupportedEncodings
+  private maxMessageSize: number
+  private nextStreamId: number
+
+  constructor (maConn: MultiaddrConnection, init: MockMuxerInit) {
+    super(maConn, {
+      ...init,
+      protocol: '/mock-muxer/1.0.0',
+      name: 'mock-muxer'
+    })
+
+    this.maxInputQueueSize = init.maxInputQueueSize ?? 1024 * 1024 * 10
+    this.maxMessageSize = (init.maxMessageSize ?? 1024 * 1024 * 4) + MESSAGE_OVERHEAD
+    this.encoding = init.encoding ?? 'base64'
+    this.input = pushable()
+    this.sendMessage = this.sendMessage.bind(this)
+    this.nextStreamId = this.maConn.direction === 'outbound' ? 0 : 1
+
+    Promise.resolve()
+      .then(async () => {
+        for await (const buf of lp.decode(this.input, {
+          maxDataLength: this.maxMessageSize
+        })) {
+          this.onMessage(cborg.decode(buf.subarray()))
+        }
+      })
+      .catch(err => {
+        this.abort(err)
+      })
+  }
+
+  onData (data: Uint8Array | Uint8ArrayList): void {
+    if (this.input.readableLength >= this.maxInputQueueSize) {
+      this.abort(new Error(`Input queue exceeded maximum size ${this.input.readableLength} >= ${this.maxInputQueueSize}`))
+      return
+    }
+
+    this.input.push(data)
+  }
+
+  sendMessage (message: StreamMessage): boolean {
+    if (message.type === 'data') {
+      this.log.trace('send message %o', { ...message, chunk: `[ ${message.chunk.byteLength} bytes ]` })
+    } else {
+      this.log.trace('send message %o', message)
+    }
+
+    const buf = cborg.encode(message)
+    const encoded = lp.encode.single(buf, {
+      maxDataLength: this.maxMessageSize
+    })
+
+    return this.send(encoded)
+  }
+
+  onMessage (message: StreamMessage): void {
+    if (message.type === 'data') {
+      this.log.trace('incoming message %o', { ...message, chunk: `[ ${message.chunk.byteLength} bytes ]` })
+    } else {
+      this.log.trace('incoming message %o', message)
+    }
+
+    let stream: MockMuxedStream | undefined = this.streams.find(s => s.id === message.id)
+
+    if (message.type === 'create') {
+      if (stream != null) {
+        throw new Error(`Already had stream for ${message.id}`)
+      }
+
+      this.log.trace('create stream inbound %s', message.id)
+      stream = this._createStream(message.id, 'inbound', {
+        protocol: message.protocol
+      })
+
+      this.onRemoteStream(stream)
+    }
+
+    if (stream == null) {
+      this.log.error(`No stream found for ${message.id}`)
+      return
+    }
+
+    if (message.type === 'data') {
+      stream.onData(message.chunk)
+    } else if (message.type === 'reset') {
+      stream.onRemoteReset()
+    } else if (message.type === 'closeWrite') {
+      stream.onRemoteCloseWrite()
+    } else if (message.type === 'closeRead') {
+      stream.onRemoteCloseRead()
+    } else if (message.type === 'pause') {
+      stream.onRemotePaused()
+    } else if (message.type === 'resume') {
+      stream.onRemoteResumed()
+    }
+  }
+
+  async onCreateStream (options: CreateStreamOptions): Promise<MockMuxedStream> {
+    this.nextStreamId += 2
+
+    return this._createStream(`${this.nextStreamId}`, 'outbound', options)
+  }
+
+  _createStream (id: string, direction: MessageStreamDirection, options: CreateStreamOptions): MockMuxedStream {
+    this.log.trace('createStream %s %s', direction, id)
+
+    return new MockMuxedStream({
+      ...this.streamOptions,
+      ...options,
+      id,
+      direction,
+      log: this.log.newScope(`stream:${direction}:${id}`),
+      sendMessage: this.sendMessage,
+      encoding: this.encoding,
+      maxMessageSize: this.maxMessageSize - MESSAGE_OVERHEAD,
+      protocol: ''
+    })
+  }
+}
+
+class MockMuxerFactory implements StreamMuxerFactory {
+  public protocol: string = '/mock-muxer/1.0.0'
+  private init: MockMuxerInit
+
+  constructor (init: MockMuxerInit) {
+    this.init = init
+  }
+
+  createStreamMuxer (maConn: MultiaddrConnection): StreamMuxer {
+    return new MockMuxer(maConn, {
+      ...this.init
+    })
+  }
+}
+
+export function mockMuxer (init: MockMuxerInit = {}): StreamMuxerFactory {
+  return new MockMuxerFactory(init)
+}

package/src/mock-stream.ts

@@ -0,0 +1,101 @@
+import { StreamMessageEvent } from '@libp2p/interface'
+import { defaultLogger } from '@libp2p/logger'
+import { raceSignal } from 'race-signal'
+import { AbstractStream } from './abstract-stream.ts'
+import type { SendResult } from './abstract-message-stream.ts'
+import type { MessageQueue, MessageQueueEvents } from './message-queue.ts'
+import type { AbortOptions, MessageStreamDirection, TypedEventTarget } from '@libp2p/interface'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+interface MockStreamMessages extends MessageQueueEvents {
+  closeWrite: Event
+  closeRead: Event
+}
+
+interface MockStreamInit {
+  delay?: number
+  direction: MessageStreamDirection
+  local: MessageQueue<MockStreamMessages>
+  remote: TypedEventTarget<MockStreamMessages>
+}
+
+let streamId = 0
+
+export class MockStream extends AbstractStream {
+  private local: MessageQueue<MockStreamMessages>
+  private remote: TypedEventTarget<MockStreamMessages>
+
+  constructor (init: MockStreamInit) {
+    const id = `${streamId++}`
+
+    super({
+      ...init,
+      id,
+      log: defaultLogger().forComponent(`libp2p:stream-pair:${init.direction}:${id}`)
+    })
+
+    this.local = init.local
+    this.remote = init.remote
+
+    this.local.addEventListener('drain', () => {
+      this.safeDispatchEvent('drain')
+    })
+
+    this.remote.addEventListener('message', (evt) => {
+      this.onData(evt.data)
+    })
+    this.remote.addEventListener('reset', (evt) => {
+      this.onRemoteReset()
+    })
+    this.remote.addEventListener('closeWrite', (evt) => {
+      this.onRemoteCloseWrite()
+    })
+  }
+
+  sendData (data: Uint8ArrayList): SendResult {
+    const canSendMore = this.local.send(new StreamMessageEvent(data))
+
+    return {
+      sentBytes: data.byteLength,
+      canSendMore
+    }
+  }
+
+  sendReset (): void {
+    this.local.send(new Event('reset'))
+  }
+
+  async sendCloseWrite (options?: AbortOptions): Promise<void> {
+    return raceSignal(new Promise<void>((resolve, reject) => {
+      this.local.send(new Event('closeWrite'))
+      this.local.onIdle().then(resolve, reject)
+    }), options?.signal)
+  }
+
+  async sendCloseRead (options?: AbortOptions): Promise<void> {
+    return raceSignal(new Promise<void>((resolve, reject) => {
+      this.local.send(new Event('closeRead'))
+      this.local.onIdle().then(resolve, reject)
+    }), options?.signal)
+  }
+
+  sendPause (): void {
+    this.local.send(new Event('pause'))
+  }
+
+  sendResume (): void {
+    this.local.send(new Event('resume'))
+  }
+
+  onRemotePaused (): void {
+    this.local.pause()
+  }
+
+  onRemoteResumed (): void {
+    this.local.resume()
+  }
+
+  onMuxerDrain (): void {
+    this.local.resume()
+  }
+}