@libp2p/utils 5.1.1 → 5.2.0

package/dist/typedoc-urls.json

@@ -25,12 +25,36 @@
   "./multiaddr/is-loopback:isLoopback": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_loopback.isLoopback.html",
   "isPrivate": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_private.isPrivate.html",
   "./multiaddr/is-private:isPrivate": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_private.isPrivate.html",
-  "PeerJobQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_job_queue.PeerJobQueue.html",
-  "./peer-job-queue:PeerJobQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_job_queue.PeerJobQueue.html",
-  "PeerPriorityQueueOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_job_queue.PeerPriorityQueueOptions.html",
-  "./peer-job-queue:PeerPriorityQueueOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_job_queue.PeerPriorityQueueOptions.html",
+  "PeerQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_queue.PeerQueue.html",
+  "./peer-queue:PeerQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_queue.PeerQueue.html",
+  "PeerQueueOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_queue.PeerQueueOptions.html",
+  "./peer-queue:PeerQueueOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_queue.PeerQueueOptions.html",
+  "Queue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.queue.Queue.html",
+  "./queue:Queue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.queue.Queue.html",
+  "JobMatcher": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.JobMatcher.html",
+  "./queue:JobMatcher": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.JobMatcher.html",
+  "QueueAddOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueAddOptions.html",
+  "./queue:QueueAddOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueAddOptions.html",
+  "QueueEvents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueEvents.html",
+  "./queue:QueueEvents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueEvents.html",
+  "QueueInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueInit.html",
+  "./queue:QueueInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueInit.html",
+  "RunFunction": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.RunFunction.html",
+  "./queue:RunFunction": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.RunFunction.html",
+  "JobStatus": "https://libp2p.github.io/js-libp2p/types/_libp2p_utils.queue.JobStatus.html",
+  "./queue:JobStatus": "https://libp2p.github.io/js-libp2p/types/_libp2p_utils.queue.JobStatus.html",
   "StreamProperties": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.stream_to_ma_conn.StreamProperties.html",
   "./stream-to-ma-conn:StreamProperties": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.stream_to_ma_conn.StreamProperties.html",
   "streamToMaConnection": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.stream_to_ma_conn.streamToMaConnection.html",
-  "./stream-to-ma-conn:streamToMaConnection": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.stream_to_ma_conn.streamToMaConnection.html"
+  "./stream-to-ma-conn:streamToMaConnection": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.stream_to_ma_conn.streamToMaConnection.html",
+  "CreateTrackedListInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_list.CreateTrackedListInit.html",
+  "./tracked-list:CreateTrackedListInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_list.CreateTrackedListInit.html",
+  "trackedList": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_list.trackedList.html",
+  "./tracked-list:trackedList": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_list.trackedList.html",
+  "CreateTrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.CreateTrackedMapInit.html",
+  "./tracked-map:CreateTrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.CreateTrackedMapInit.html",
+  "TrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.TrackedMapInit.html",
+  "./tracked-map:TrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.TrackedMapInit.html",
+  "trackedMap": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_map.trackedMap.html",
+  "./tracked-map:trackedMap": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_map.trackedMap.html"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@libp2p/utils",
-  "version": "5.1.1",
+  "version": "5.2.0",
   "description": "Package to aggregate shared logic and dependencies for the libp2p ecosystem",
   "license": "Apache-2.0 OR MIT",
   "homepage": "https://github.com/libp2p/js-libp2p/tree/main/packages/utils#readme",
@@ -76,14 +76,22 @@
       "types": "./dist/src/multiaddr/is-private.d.ts",
       "import": "./dist/src/multiaddr/is-private.js"
     },
-    "./peer-job-queue": {
-      "types": "./dist/src/peer-job-queue.d.ts",
-      "import": "./dist/src/peer-job-queue.js"
+    "./queue": {
+      "types": "./dist/src/queue/index.d.ts",
+      "import": "./dist/src/queue/index.js"
+    },
+    "./peer-queue": {
+      "types": "./dist/src/peer-queue.d.ts",
+      "import": "./dist/src/peer-queue.js"
     },
     "./stream-to-ma-conn": {
       "types": "./dist/src/stream-to-ma-conn.d.ts",
       "import": "./dist/src/stream-to-ma-conn.js"
     },
+    "./tracked-list": {
+      "types": "./dist/src/tracked-list.d.ts",
+      "import": "./dist/src/tracked-list.js"
+    },
     "./tracked-map": {
       "types": "./dist/src/tracked-map.d.ts",
       "import": "./dist/src/tracked-map.js"
@@ -111,29 +119,28 @@
   },
   "dependencies": {
     "@chainsafe/is-ip": "^2.0.2",
-    "@libp2p/interface": "^1.1.0",
-    "@libp2p/peer-collections": "^5.1.2",
+    "@libp2p/interface": "^1.1.1",
+    "@libp2p/logger": "^4.0.4",
     "@multiformats/multiaddr": "^12.1.10",
     "@multiformats/multiaddr-matcher": "^1.1.0",
     "get-iterator": "^2.0.1",
     "is-loopback-addr": "^2.0.1",
     "it-pushable": "^3.2.2",
     "it-stream-types": "^2.0.1",
-    "p-queue": "^8.0.0",
+    "p-defer": "^4.0.0",
     "private-ip": "^3.0.1",
+    "race-event": "^1.1.0",
     "race-signal": "^1.0.1",
     "uint8arraylist": "^2.4.3"
   },
   "devDependencies": {
-    "@libp2p/logger": "^4.0.3",
-    "@libp2p/peer-id-factory": "^4.0.2",
-    "aegir": "^41.0.2",
+    "@libp2p/peer-id-factory": "^4.0.3",
+    "aegir": "^42.0.0",
     "delay": "^6.0.0",
     "it-all": "^3.0.3",
     "it-drain": "^3.0.5",
     "it-pair": "^2.0.6",
     "it-pipe": "^3.0.1",
-    "p-defer": "^4.0.0",
     "sinon": "^17.0.1",
     "sinon-ts": "^2.0.0",
     "uint8arrays": "^5.0.0"

package/src/peer-queue.ts

@@ -0,0 +1,24 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+
+import { Queue, type QueueAddOptions } from './queue/index.js'
+import type { Job } from './queue/job.js'
+import type { PeerId } from '@libp2p/interface'
+
+export interface PeerQueueOptions extends QueueAddOptions {
+  peerId: PeerId
+}
+
+/**
+ * Extends Queue to add support for querying queued jobs by peer id
+ */
+export class PeerQueue<JobReturnType = void> extends Queue<JobReturnType, PeerQueueOptions> {
+  has (peerId: PeerId): boolean {
+    return this.find(peerId) != null
+  }
+
+  find (peerId: PeerId): Job<PeerQueueOptions, JobReturnType> | undefined {
+    return this.queue.find(job => {
+      return peerId.equals(job.options.peerId)
+    })
+  }
+}

package/src/queue/index.ts

@@ -0,0 +1,365 @@
+import { AbortError, CodeError, TypedEventEmitter } from '@libp2p/interface'
+import { pushable } from 'it-pushable'
+import { raceEvent } from 'race-event'
+import { Job } from './job.js'
+import type { AbortOptions, Metrics } from '@libp2p/interface'
+
+export interface QueueAddOptions extends AbortOptions {
+  /**
+   * Priority of operation. Operations with greater priority will be scheduled first.
+   *
+   * @default 0
+   */
+  priority?: number
+}
+
+export interface QueueInit {
+  /**
+   * Concurrency limit.
+   *
+   * Minimum: `1`.
+   *
+   * @default Infinity
+   */
+  concurrency?: number
+
+  /**
+   * The name of the metric for the queue length
+   */
+  metricName?: string
+
+  /**
+   * An implementation of the libp2p Metrics interface
+   */
+  metrics?: Metrics
+}
+
+export type JobStatus = 'queued' | 'running' | 'errored' | 'complete'
+
+export interface RunFunction<Options = AbortOptions, ReturnType = void> {
+  (opts?: Options): Promise<ReturnType>
+}
+
+export interface JobMatcher<JobOptions extends QueueAddOptions = QueueAddOptions> {
+  (options?: Partial<JobOptions>): boolean
+}
+
+export interface QueueEvents<JobReturnType> {
+  'active': CustomEvent
+  'idle': CustomEvent
+  'empty': CustomEvent
+  'add': CustomEvent
+  'next': CustomEvent
+  'completed': CustomEvent<JobReturnType>
+  'error': CustomEvent<Error>
+}
+
+// Port of lower_bound from https://en.cppreference.com/w/cpp/algorithm/lower_bound
+// Used to compute insertion index to keep queue sorted after insertion
+function lowerBound<T> (array: readonly T[], value: T, comparator: (a: T, b: T) => number): number {
+  let first = 0
+  let count = array.length
+
+  while (count > 0) {
+    const step = Math.trunc(count / 2)
+    let it = first + step
+
+    if (comparator(array[it], value) <= 0) {
+      first = ++it
+      count -= step + 1
+    } else {
+      count = step
+    }
+  }
+
+  return first
+}
+
+/**
+ * Heavily influence by `p-queue` with the following differences:
+ *
+ * 1. Items remain at the head of the queue while they are running so `queue.size` includes `queue.pending` items - this is so interested parties can join the results of a queue item while it is running
+ * 2. The options for a job are stored separately to the job in order for them to be modified while they are still in the queue
+ */
+export class Queue<JobReturnType = unknown, JobOptions extends QueueAddOptions = QueueAddOptions> extends TypedEventEmitter<QueueEvents<JobReturnType>> {
+  public concurrency: number
+  public queue: Array<Job<JobOptions, JobReturnType>>
+  private pending: number
+
+  constructor (init: QueueInit = {}) {
+    super()
+
+    this.concurrency = init.concurrency ?? Number.POSITIVE_INFINITY
+    this.pending = 0
+
+    if (init.metricName != null) {
+      init.metrics?.registerMetricGroup(init.metricName, {
+        calculate: () => {
+          return {
+            size: this.queue.length,
+            running: this.pending,
+            queued: this.queue.length - this.pending
+          }
+        }
+      })
+    }
+
+    this.queue = []
+  }
+
+  private tryToStartAnother (): boolean {
+    if (this.size === 0) {
+      // do this in the microtask queue so all job recipients receive the
+      // result before the "empty" event fires
+      queueMicrotask(() => {
+        this.safeDispatchEvent('empty')
+      })
+
+      if (this.running === 0) {
+        // do this in the microtask queue so all job recipients receive the
+        // result before the "idle" event fires
+        queueMicrotask(() => {
+          this.safeDispatchEvent('idle')
+        })
+      }
+
+      return false
+    }
+
+    if (this.pending < this.concurrency) {
+      let job: Job<JobOptions, JobReturnType> | undefined
+
+      for (const j of this.queue) {
+        if (j.status === 'queued') {
+          job = j
+          break
+        }
+      }
+
+      if (job == null) {
+        return false
+      }
+
+      this.safeDispatchEvent('active')
+
+      this.pending++
+
+      job.run()
+        .finally(() => {
+          // remove the job from the queue
+          for (let i = 0; i < this.queue.length; i++) {
+            if (this.queue[i] === job) {
+              this.queue.splice(i, 1)
+              break
+            }
+          }
+
+          this.pending--
+          this.tryToStartAnother()
+          this.safeDispatchEvent('next')
+        })
+
+      return true
+    }
+
+    return false
+  }
+
+  private enqueue (job: Job<JobOptions, JobReturnType>): void {
+    if (this.queue[this.size - 1]?.priority >= job.priority) {
+      this.queue.push(job)
+      return
+    }
+
+    const index = lowerBound(
+      this.queue, job,
+      (a: Readonly< Job<JobOptions, JobReturnType>>, b: Readonly< Job<JobOptions, JobReturnType>>) => b.priority - a.priority
+    )
+    this.queue.splice(index, 0, job)
+  }
+
+  /**
+   * Adds a sync or async task to the queue. Always returns a promise.
+   */
+  async add (fn: RunFunction<JobOptions, JobReturnType>, options?: JobOptions): Promise<JobReturnType> {
+    options?.signal?.throwIfAborted()
+
+    const job = new Job<JobOptions, JobReturnType>(fn, options, options?.priority)
+
+    const p = job.join(options)
+      .then(result => {
+        this.safeDispatchEvent('completed', { detail: result })
+
+        return result
+      })
+      .catch(err => {
+        this.safeDispatchEvent('error', { detail: err })
+
+        throw err
+      })
+
+    this.enqueue(job)
+    this.safeDispatchEvent('add')
+    this.tryToStartAnother()
+
+    return p
+  }
+
+  /**
+   * Clear the queue
+   */
+  clear (): void {
+    this.queue.splice(0, this.queue.length)
+  }
+
+  /**
+   * Abort all jobs in the queue and clear it
+   */
+  abort (): void {
+    this.queue.forEach(job => {
+      job.abort(new AbortError())
+    })
+
+    this.clear()
+  }
+
+  /**
+   * Can be called multiple times. Useful if you for example add additional items at a later time.
+   *
+   * @returns A promise that settles when the queue becomes empty.
+   */
+  async onEmpty (options?: AbortOptions): Promise<void> {
+    // Instantly resolve if the queue is empty
+    if (this.size === 0) {
+      return
+    }
+
+    await raceEvent(this, 'empty', options?.signal)
+  }
+
+  /**
+   * @returns A promise that settles when the queue size is less than the given
+   * limit: `queue.size < limit`.
+   *
+   * If you want to avoid having the queue grow beyond a certain size you can
+   * `await queue.onSizeLessThan()` before adding a new item.
+   *
+   * Note that this only limits the number of items waiting to start. There
+   * could still be up to `concurrency` jobs already running that this call does
+   * not include in its calculation.
+   */
+  async onSizeLessThan (limit: number, options?: AbortOptions): Promise<void> {
+    // Instantly resolve if the queue is empty.
+    if (this.size < limit) {
+      return
+    }
+
+    await raceEvent(this, 'next', options?.signal, {
+      filter: () => this.size < limit
+    })
+  }
+
+  /**
+   * The difference with `.onEmpty` is that `.onIdle` guarantees that all work
+   * from the queue has finished. `.onEmpty` merely signals that the queue is
+   * empty, but it could mean that some promises haven't completed yet.
+   *
+   * @returns A promise that settles when the queue becomes empty, and all
+   * promises have completed; `queue.size === 0 && queue.pending === 0`.
+   */
+  async onIdle (options?: AbortOptions): Promise<void> {
+    // Instantly resolve if none pending and if nothing else is queued
+    if (this.pending === 0 && this.size === 0) {
+      return
+    }
+
+    await raceEvent(this, 'idle', options?.signal)
+  }
+
+  /**
+   * Size of the queue including running items
+   */
+  get size (): number {
+    return this.queue.length
+  }
+
+  /**
+   * The number of queued items waiting to run.
+   */
+  get queued (): number {
+    return this.queue.length - this.pending
+  }
+
+  /**
+   * The number of items currently running.
+   */
+  get running (): number {
+    return this.pending
+  }
+
+  /**
+   * Returns an async generator that makes it easy to iterate over the results
+   * of jobs added to the queue.
+   *
+   * The generator will end when the queue becomes idle, that is there are no
+   * jobs running and no jobs that have yet to run.
+   *
+   * If you need to keep the queue open indefinitely, consider using it-pushable
+   * instead.
+   */
+  async * toGenerator (options?: AbortOptions): AsyncGenerator<JobReturnType, void, unknown> {
+    options?.signal?.throwIfAborted()
+
+    const stream = pushable<JobReturnType>({
+      objectMode: true
+    })
+
+    const cleanup = (err?: Error): void => {
+      if (err != null) {
+        this.abort()
+      } else {
+        this.clear()
+      }
+
+      stream.end(err)
+    }
+
+    const onQueueJobComplete = (evt: CustomEvent<JobReturnType>): void => {
+      if (evt.detail != null) {
+        stream.push(evt.detail)
+      }
+    }
+
+    const onQueueError = (evt: CustomEvent<Error>): void => {
+      cleanup(evt.detail)
+    }
+
+    const onQueueIdle = (): void => {
+      cleanup()
+    }
+
+    // clear the queue and throw if the query is aborted
+    const onSignalAbort = (): void => {
+      cleanup(new CodeError('Queue aborted', 'ERR_QUEUE_ABORTED'))
+    }
+
+    // add listeners
+    this.addEventListener('completed', onQueueJobComplete)
+    this.addEventListener('error', onQueueError)
+    this.addEventListener('idle', onQueueIdle)
+    options?.signal?.addEventListener('abort', onSignalAbort)
+
+    try {
+      yield * stream
+    } finally {
+      // remove listeners
+      this.removeEventListener('completed', onQueueJobComplete)
+      this.removeEventListener('error', onQueueError)
+      this.removeEventListener('idle', onQueueIdle)
+      options?.signal?.removeEventListener('abort', onSignalAbort)
+
+      // empty the queue for when the user has broken out of a loop early
+      cleanup()
+    }
+  }
+}

package/src/queue/job.ts

@@ -0,0 +1,105 @@
+import { AbortError, setMaxListeners } from '@libp2p/interface'
+import { raceSignal } from 'race-signal'
+import { JobRecipient } from './recipient.js'
+import type { JobStatus } from './index.js'
+import type { AbortOptions } from '@libp2p/interface'
+
+/**
+ * Returns a random string
+ */
+function randomId (): string {
+  return `${(parseInt(String(Math.random() * 1e9), 10)).toString()}${Date.now()}`
+}
+
+export interface JobTimeline {
+  created: number
+  started?: number
+  finished?: number
+}
+
+export class Job <JobOptions extends AbortOptions = AbortOptions, JobReturnType = unknown> {
+  public id: string
+  public fn: (options: JobOptions) => Promise<JobReturnType>
+  public options: JobOptions
+  public priority: number
+  public recipients: Array<JobRecipient<JobReturnType>>
+  public status: JobStatus
+  public readonly timeline: JobTimeline
+  private readonly controller: AbortController
+
+  constructor (fn: (options: JobOptions) => Promise<JobReturnType>, options: any, priority: number = 0) {
+    this.id = randomId()
+    this.status = 'queued'
+    this.fn = fn
+    this.priority = priority
+    this.options = options
+    this.recipients = []
+    this.timeline = {
+      created: Date.now()
+    }
+
+    this.controller = new AbortController()
+    setMaxListeners(Infinity, this.controller.signal)
+
+    this.onAbort = this.onAbort.bind(this)
+  }
+
+  abort (err: Error): void {
+    this.controller.abort(err)
+  }
+
+  onAbort (): void {
+    const allAborted = this.recipients.reduce((acc, curr) => {
+      return acc && (curr.signal?.aborted === true)
+    }, true)
+
+    // if all recipients have aborted the job, actually abort the job
+    if (allAborted) {
+      this.controller.abort(new AbortError())
+    }
+  }
+
+  async join (options: AbortOptions = {}): Promise<JobReturnType> {
+    const recipient = new JobRecipient<JobReturnType>((new Error('where')).stack, options.signal)
+    this.recipients.push(recipient)
+
+    options.signal?.addEventListener('abort', this.onAbort)
+
+    return recipient.deferred.promise
+  }
+
+  async run (): Promise<void> {
+    this.status = 'running'
+    this.timeline.started = Date.now()
+
+    try {
+      this.controller.signal.throwIfAborted()
+
+      const result = await raceSignal(this.fn({
+        ...(this.options ?? {}),
+        signal: this.controller.signal
+      }), this.controller.signal)
+
+      this.recipients.forEach(recipient => {
+        recipient.deferred.resolve(result)
+      })
+
+      this.status = 'complete'
+    } catch (err) {
+      this.recipients.forEach(recipient => {
+        recipient.deferred.reject(err)
+      })
+
+      this.status = 'errored'
+    } finally {
+      this.timeline.finished = Date.now()
+      this.cleanup()
+    }
+  }
+
+  cleanup (): void {
+    this.recipients.forEach(recipient => {
+      recipient.signal?.removeEventListener('abort', this.onAbort)
+    })
+  }
+}

package/src/queue/recipient.ts

@@ -0,0 +1,26 @@
+import { AbortError } from '@libp2p/interface'
+import pDefer from 'p-defer'
+import type { DeferredPromise } from 'p-defer'
+
+export class JobRecipient<JobReturnType> {
+  public deferred: DeferredPromise<JobReturnType>
+  public signal?: AbortSignal
+  public where?: string
+
+  constructor (where?: string, signal?: AbortSignal) {
+    this.signal = signal
+    this.deferred = pDefer()
+    this.where = where
+
+    this.onAbort = this.onAbort.bind(this)
+    this.signal?.addEventListener('abort', this.onAbort)
+  }
+
+  onAbort (): void {
+    this.deferred.reject(new AbortError())
+  }
+
+  cleanup (): void {
+    this.signal?.removeEventListener('abort', this.onAbort)
+  }
+}

package/src/tracked-list.ts

@@ -0,0 +1,26 @@
+import type { Metrics } from '@libp2p/interface'
+
+export interface CreateTrackedListInit {
+  /**
+   * The metric name to use
+   */
+  name: string
+
+  /**
+   * A metrics implementation
+   */
+  metrics?: Metrics
+}
+
+export function trackedList <V> (config: CreateTrackedListInit): V[] {
+  const { name, metrics } = config
+  const list: V[] = []
+
+  metrics?.registerMetric(name, {
+    calculate: () => {
+      return list.length
+    }
+  })
+
+  return list
+}