@libp2p/utils 6.7.1 → 6.7.2-a02cb0461

package/src/multiaddr-connection-pair.ts

@@ -0,0 +1,147 @@
+import { StreamMessageEvent } from '@libp2p/interface'
+import { defaultLogger } from '@libp2p/logger'
+import { multiaddr } from '@multiformats/multiaddr'
+import { pEvent } from 'p-event'
+import { raceSignal } from 'race-signal'
+import { AbstractMultiaddrConnection } from './abstract-multiaddr-connection.ts'
+import { MessageQueue } from './message-queue.ts'
+import type { SendResult } from './abstract-message-stream.ts'
+import type { AbstractMultiaddrConnectionInit } from './abstract-multiaddr-connection.ts'
+import type { MessageQueueEvents, MessageQueueInit } from './message-queue.ts'
+import type { AbortOptions, Logger, MultiaddrConnection, MessageStreamDirection, TypedEventTarget } from '@libp2p/interface'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+interface MockMultiaddrConnectionMessages extends MessageQueueEvents {
+  close: Event
+  pause: Event
+  resume: Event
+}
+
+export interface MockMultiaddrConnectionInit extends AbstractMultiaddrConnectionInit {
+  id: string,
+  log: Logger,
+  direction: MessageStreamDirection
+  local: MessageQueue<MockMultiaddrConnectionMessages>
+  remote: TypedEventTarget<MockMultiaddrConnectionMessages>
+  remoteAddr: Multiaddr
+}
+
+let multiaddrConnectionId = 0
+
+class MockMultiaddrConnection extends AbstractMultiaddrConnection {
+  private local: MessageQueue<MockMultiaddrConnectionMessages>
+  private remote: TypedEventTarget<MockMultiaddrConnectionMessages>
+
+  constructor (init: MockMultiaddrConnectionInit) {
+    super(init)
+
+    this.local = init.local
+    this.remote = init.remote
+
+    this.local.addEventListener('drain', () => {
+      this.safeDispatchEvent('drain')
+    })
+
+    this.remote.addEventListener('message', (evt) => {
+      if (this.status !== 'open') {
+        return
+      }
+
+      this.onData(evt.data)
+    })
+    this.remote.addEventListener('reset', (evt) => {
+      if (this.status !== 'open') {
+        return
+      }
+
+      this.onRemoteReset()
+    })
+    this.remote.addEventListener('close', (evt) => {
+      this.onTransportClosed()
+    })
+    this.remote.addEventListener('pause', (evt) => {
+      this.local.pause()
+    })
+    this.remote.addEventListener('resume', (evt) => {
+      this.local.resume()
+    })
+  }
+
+  sendData (data: Uint8ArrayList): SendResult {
+    const canSendMore = this.local.send(new StreamMessageEvent(data))
+
+    return {
+      sentBytes: data.byteLength,
+      canSendMore
+    }
+  }
+
+  sendReset (): void {
+    this.local.send(new Event('reset'))
+  }
+
+  async sendClose (options?: AbortOptions): Promise<void> {
+    if (this.local.needsDrain) {
+      await pEvent(this.local, 'drain', {
+        signal: options?.signal
+      })
+    }
+
+    return raceSignal(new Promise<void>((resolve, reject) => {
+      this.local.send(new Event('close'))
+      this.local.onIdle().then(resolve, reject)
+    }), options?.signal)
+  }
+
+  sendPause (): void {
+    this.local.send(new Event('pause'))
+  }
+
+  sendResume (): void {
+    this.local.send(new Event('resume'))
+  }
+}
+
+export interface MultiaddrConnectionPairOptions extends MessageQueueInit {
+  outbound?: Partial<MockMultiaddrConnectionInit>
+  inbound?: Partial<MockMultiaddrConnectionInit>
+}
+
+export function multiaddrConnectionPair (opts: MultiaddrConnectionPairOptions = {}): [MultiaddrConnection, MultiaddrConnection] {
+  const inboundId = `${multiaddrConnectionId++}`
+  const outboundId = `${multiaddrConnectionId++}`
+
+  const outboundLog = defaultLogger().forComponent(`libp2p:mock-maconn:outbound:${inboundId}`)
+  const inboundLog = defaultLogger().forComponent(`libp2p:mock-maconn:inbound:${outboundId}`)
+
+  const targetA = new MessageQueue<MockMultiaddrConnectionMessages>({
+    ...opts,
+    log: outboundLog
+  })
+  const targetB = new MessageQueue<MockMultiaddrConnectionMessages>({
+    ...opts,
+    log: inboundLog
+  })
+
+  return [
+    new MockMultiaddrConnection({
+      ...opts.outbound,
+      id: outboundId,
+      direction: 'outbound',
+      local: targetA,
+      remote: targetB,
+      remoteAddr: opts?.outbound?.remoteAddr ?? multiaddr(`/ip4/127.0.0.1/tcp/${outboundId}`),
+      log: outboundLog
+    }),
+    new MockMultiaddrConnection({
+      ...opts.inbound,
+      id: inboundId,
+      direction: 'inbound',
+      local: targetB,
+      remote: targetA,
+      remoteAddr: opts?.inbound?.remoteAddr ?? multiaddr(`/ip4/127.0.0.1/tcp/${inboundId}`),
+      log: inboundLog
+    })
+  ]
+}

package/src/queue/index.ts

@@ -1,7 +1,7 @@
 import { AbortError } from '@libp2p/interface'
 import { pushable } from 'it-pushable'
 import { TypedEventEmitter } from 'main-event'
-import { raceEvent } from 'race-event'
+import { pEvent } from 'p-event'
 import { debounce } from '../debounce.js'
 import { QueueFullError } from '../errors.js'
 import { Job } from './job.js'
@@ -101,6 +101,8 @@ export interface QueueEvents<JobReturnType, JobOptions extends AbortOptions = Ab
 
   /**
    * A job has failed
+   *
+   * @deprecated Listen for the 'failure' event instead - it gives more context and is generally more useful, this event will be removed in a future release
    */
   error: CustomEvent<Error>
 
@@ -129,6 +131,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
   public queue: Array<Job<JobOptions, JobReturnType>>
   private pending: number
   private readonly sort?: Comparator<Job<JobOptions, JobReturnType>>
+  private paused: boolean
 
   constructor (init: QueueInit<JobReturnType, JobOptions> = {}) {
     super()
@@ -136,6 +139,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
     this.concurrency = init.concurrency ?? Number.POSITIVE_INFINITY
     this.maxSize = init.maxSize ?? Number.POSITIVE_INFINITY
     this.pending = 0
+    this.paused = false
 
     if (init.metricName != null) {
       init.metrics?.registerMetricGroup(init.metricName, {
@@ -172,7 +176,24 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
     this.safeDispatchEvent('idle')
   }
 
+  pause (): void {
+    this.paused = true
+  }
+
+  resume (): void {
+    if (!this.paused) {
+      return
+    }
+
+    this.paused = false
+    this.tryToStartAnother()
+  }
+
   private tryToStartAnother (): boolean {
+    if (this.paused) {
+      return false
+    }
+
     if (this.size === 0) {
       this.emitEmpty()
 
@@ -263,7 +284,6 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
           }
         }
 
-        this.safeDispatchEvent('error', { detail: err })
         this.safeDispatchEvent('failure', { detail: { job, error: err } })
 
         throw err
@@ -299,7 +319,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
       return
     }
 
-    await raceEvent(this, 'empty', options?.signal)
+    await pEvent(this, 'empty', options)
   }
 
   /**
@@ -319,7 +339,8 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
       return
     }
 
-    await raceEvent(this, 'next', options?.signal, {
+    await pEvent(this, 'next', {
+      ...options,
       filter: () => this.size < limit
     })
   }
@@ -338,7 +359,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
       return
     }
 
-    await raceEvent(this, 'idle', options?.signal)
+    await pEvent(this, 'idle', options)
   }
 
   /**
@@ -395,8 +416,8 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
       }
     }
 
-    const onQueueError = (evt: CustomEvent<Error>): void => {
-      cleanup(evt.detail)
+    const onQueueFailure = (evt: CustomEvent<QueueJobFailure<JobReturnType, JobOptions>>): void => {
+      cleanup(evt.detail.error)
     }
 
     const onQueueIdle = (): void => {
@@ -410,7 +431,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
 
     // add listeners
     this.addEventListener('completed', onQueueJobComplete)
-    this.addEventListener('error', onQueueError)
+    this.addEventListener('failure', onQueueFailure)
     this.addEventListener('idle', onQueueIdle)
     options?.signal?.addEventListener('abort', onSignalAbort)
 
@@ -419,7 +440,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = Ab
     } finally {
       // remove listeners
       this.removeEventListener('completed', onQueueJobComplete)
-      this.removeEventListener('error', onQueueError)
+      this.removeEventListener('failure', onQueueFailure)
       this.removeEventListener('idle', onQueueIdle)
       options?.signal?.removeEventListener('abort', onSignalAbort)
 

package/src/rate-limiter.ts

@@ -1,4 +1,3 @@
-import delay from 'delay'
 import { RateLimitError } from './errors.js'
 
 export interface RateLimiterInit {
@@ -23,20 +22,6 @@ export interface RateLimiterInit {
    */
   blockDuration?: number
 
-  /**
-   * Execute allowed actions evenly over duration
-   *
-   * @default false
-   */
-  execEvenly?: boolean
-
-  /**
-   * ms, works with execEvenly=true option
-   *
-   * @default duration * 1000 / points
-   */
-  execEvenlyMinDelayMs?: number
-
   /**
    * @default "rlflx"
    */
@@ -65,21 +50,17 @@ export class RateLimiter {
   protected points: number
   protected duration: number
   protected blockDuration: number
-  protected execEvenly: boolean
-  protected execEvenlyMinDelayMs: number
   protected keyPrefix: string
 
   constructor (opts: RateLimiterInit = {}) {
     this.points = opts.points ?? 4
     this.duration = opts.duration ?? 1
     this.blockDuration = opts.blockDuration ?? 0
-    this.execEvenly = opts.execEvenly ?? false
-    this.execEvenlyMinDelayMs = opts.execEvenlyMinDelayMs ?? (this.duration * 1000 / this.points)
     this.keyPrefix = opts.keyPrefix ?? 'rlflx'
     this.memoryStorage = new MemoryStorage()
   }
 
-  async consume (key: string, pointsToConsume: number = 1, options: GetKeySecDurationOptions = {}): Promise<RateLimiterResult> {
+  consume (key: string, pointsToConsume: number = 1, options: GetKeySecDurationOptions = {}): RateLimiterResult {
     const rlKey = this.getKey(key)
     const secDuration = this._getKeySecDuration(options)
     let res = this.memoryStorage.incrby(rlKey, pointsToConsume, secDuration)
@@ -93,14 +74,6 @@ export class RateLimiter {
       }
 
       throw new RateLimitError('Rate limit exceeded', res)
-    } else if (this.execEvenly && res.msBeforeNext > 0 && !res.isFirstInDuration) {
-      // Execute evenly
-      let delayMs = Math.ceil(res.msBeforeNext / (res.remainingPoints + 2))
-      if (delayMs < this.execEvenlyMinDelayMs) {
-        delayMs = res.consumedPoints * this.execEvenlyMinDelayMs
-      }
-
-      await delay(delayMs)
     }
 
     return res
@@ -241,8 +214,8 @@ export class MemoryStorage {
         this.storage.delete(key)
       }, durationMs)
 
-      if (record.timeoutId.unref != null) {
-        record.timeoutId.unref()
+      if ((record.timeoutId as any).unref != null) {
+        (record.timeoutId as any).unref()
       }
     }
 

package/src/socket-writer.browser.ts

@@ -0,0 +1,3 @@
+export function socketWriter (): void {
+  throw new Error('Unsupported in browsers')
+}

package/src/socket-writer.ts

@@ -0,0 +1,64 @@
+import stream from 'node:stream'
+import { Uint8ArrayList } from 'uint8arraylist'
+
+export interface SocketWriter {
+  /**
+   * Write any available data into the socket, if the socket's internal write
+   * buffer has available capacity
+   */
+  pull (): boolean
+
+  /**
+   * Write data into the socket, returns false if the socket's internal write
+   * buffer is at capacity
+   */
+  write (data: Uint8Array | Uint8Array[] | Uint8ArrayList): boolean
+}
+
+/**
+ * @deprecated delete if unused
+ */
+export function socketWriter (socket: stream.Duplex): SocketWriter {
+  const queue = new Uint8ArrayList()
+
+  return {
+    pull (): boolean {
+      if (socket.writableNeedDrain) {
+        return false
+      }
+
+      for (const buf of queue) {
+        queue.consume(buf.byteLength)
+
+        if (!socket.write(buf)) {
+          // continue writing after drain event. this is a synchronous operation
+          // so it will not interleave with the `this.writeToSocket()`
+          // invocation in this.sendData so all data will be sent in-order
+          if (queue.byteLength > 0) {
+            socket.once('drain', () => {
+              this.pull()
+            })
+          }
+
+          return false
+        }
+      }
+
+      return true
+    },
+
+    write (data: Uint8Array | Uint8Array[] | Uint8ArrayList): boolean {
+      if (Array.isArray(data)) {
+        queue.appendAll(data)
+      } else {
+        queue.append(data)
+      }
+
+      if (socket.writableNeedDrain) {
+        return false
+      }
+
+      return this.pull()
+    }
+  }
+}

package/src/stream-pair.ts

@@ -0,0 +1,90 @@
+import { pEvent } from 'p-event'
+import { mockMuxer } from './mock-muxer.ts'
+import { multiaddrConnectionPair } from './multiaddr-connection-pair.ts'
+import { echo } from './stream-utils.ts'
+import type { MockMultiaddrConnectionInit } from './multiaddr-connection-pair.ts'
+import type { Stream, StreamOptions } from '@libp2p/interface'
+
+export interface StreamPairOptions {
+  /**
+   * How long to wait in ms before sending messages
+   *
+   * @default 1
+   */
+  delay?: number
+
+  /**
+   * If more than this many messages are sent within delay, write backpressure
+   * will be applied
+   */
+  capacity?: number
+
+  /**
+   * Simulate having pre-negotiated a protocol by passing it here
+   */
+  protocol?: string
+
+  /**
+   * Configuration options for the outbound stream
+   */
+  outbound?: StreamOptions
+
+  /**
+   * Configuration options for underlying outbound connection
+   */
+  outboundConnection?: Partial<MockMultiaddrConnectionInit>
+
+  /**
+   * Configuration options for the inbound stream
+   */
+  inbound?: StreamOptions
+
+  /**
+   * Configuration options for underlying inbound connection
+   */
+  inboundConnection?: Partial<MockMultiaddrConnectionInit>
+}
+
+/**
+ * Returns two streams connected to each other with a slight delay in sending
+ * messages to simulate a network
+ */
+export async function streamPair (opts: StreamPairOptions = {}): Promise<[Stream, Stream]> {
+  const [outboundConnection, inboundConnection] = multiaddrConnectionPair({
+    ...opts,
+    outbound: opts.outboundConnection,
+    inbound: opts.inboundConnection
+  })
+
+  const localMuxer = mockMuxer({
+    streamOptions: opts.outbound
+  }).createStreamMuxer(outboundConnection)
+  const remoteMuxer = mockMuxer({
+    streamOptions: opts.inbound
+  }).createStreamMuxer(inboundConnection)
+
+  const [
+    inboundStream,
+    outboundStream
+  ] = await Promise.all([
+    pEvent<'stream', CustomEvent<Stream>>(remoteMuxer, 'stream').then(evt => {
+      return evt.detail
+    }),
+    localMuxer.createStream({
+      ...opts.outbound,
+      protocol: opts.protocol
+    })
+  ])
+
+  return [
+    outboundStream,
+    inboundStream
+  ]
+}
+
+export async function echoStream (opts: StreamPairOptions = {}): Promise<Stream> {
+  const [outbound, inbound] = await streamPair(opts)
+  echo(inbound)
+
+  return outbound
+}