@libp2p/utils 5.3.2 → 5.4.0-43046b9ae

package/src/filters/cuckoo-filter.ts

@@ -0,0 +1,197 @@
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { Bucket } from './bucket.js'
+import { Fingerprint, MAX_FINGERPRINT_SIZE } from './fingerprint.js'
+import { fnv1a, type Hash } from './hashes.js'
+import { getRandomInt } from './utils.js'
+import type { Filter } from './index.js'
+
+const maxCuckooCount = 500
+
+export interface CuckooFilterInit {
+  /**
+   * How many items the filter is expected to contain
+   */
+  filterSize: number
+
+  /**
+   * How many items to put in each bucket
+   */
+  bucketSize?: number
+
+  /**
+   * How many bytes the fingerprint is expected to be
+   */
+  fingerprintSize?: number
+
+  /**
+   * A non-cryptographic hash implementation
+   */
+  hash?: Hash
+
+  /**
+   * A number used to seed the hash
+   */
+  seed?: number
+}
+
+export class CuckooFilter implements Filter {
+  private readonly bucketSize: number
+  private readonly filterSize: number
+  private readonly fingerprintSize: number
+  private readonly buckets: Bucket[]
+  public count: number
+  private readonly hash: Hash
+  private readonly seed: number
+
+  constructor (init: CuckooFilterInit) {
+    this.filterSize = init.filterSize
+    this.bucketSize = init.bucketSize ?? 4
+    this.fingerprintSize = init.fingerprintSize ?? 2
+    this.count = 0
+    this.buckets = []
+    this.hash = init.hash ?? fnv1a
+    this.seed = init.seed ?? getRandomInt(0, Math.pow(2, 10))
+  }
+
+  add (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    const fingerprint = new Fingerprint(item, this.hash, this.seed, this.fingerprintSize)
+    const j = this.hash.hash(item, this.seed) % this.filterSize
+    const k = (j ^ fingerprint.hash()) % this.filterSize
+
+    if (this.buckets[j] == null) {
+      this.buckets[j] = new Bucket(this.bucketSize)
+    }
+
+    if (this.buckets[k] == null) {
+      this.buckets[k] = new Bucket(this.bucketSize)
+    }
+
+    if (this.buckets[j].add(fingerprint) || this.buckets[k].add(fingerprint)) {
+      this.count++
+      return true
+    }
+
+    const rand = [j, k]
+    let i = rand[getRandomInt(0, rand.length - 1)]
+
+    if (this.buckets[i] == null) {
+      this.buckets[i] = new Bucket(this.bucketSize)
+    }
+
+    for (let n = 0; n < maxCuckooCount; n++) {
+      const swapped = this.buckets[i].swap(fingerprint)
+
+      if (swapped == null) {
+        continue
+      }
+
+      i = (i ^ swapped.hash()) % this.filterSize
+
+      if (this.buckets[i] == null) {
+        this.buckets[i] = new Bucket(this.bucketSize)
+      }
+
+      if (this.buckets[i].add(swapped)) {
+        this.count++
+
+        return true
+      } else {
+        continue
+      }
+    }
+
+    return false
+  }
+
+  has (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    const fingerprint = new Fingerprint(item, this.hash, this.seed, this.fingerprintSize)
+    const j = this.hash.hash(item, this.seed) % this.filterSize
+    const inJ = this.buckets[j]?.has(fingerprint) ?? false
+
+    if (inJ) {
+      return inJ
+    }
+
+    const k = (j ^ fingerprint.hash()) % this.filterSize
+
+    return this.buckets[k]?.has(fingerprint) ?? false
+  }
+
+  remove (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    const fingerprint = new Fingerprint(item, this.hash, this.seed, this.fingerprintSize)
+    const j = this.hash.hash(item, this.seed) % this.filterSize
+    const inJ = this.buckets[j]?.remove(fingerprint) ?? false
+
+    if (inJ) {
+      this.count--
+      return inJ
+    }
+
+    const k = (j ^ fingerprint.hash()) % this.filterSize
+    const inK = this.buckets[k]?.remove(fingerprint) ?? false
+
+    if (inK) {
+      this.count--
+    }
+
+    return inK
+  }
+
+  get reliable (): boolean {
+    return Math.floor(100 * (this.count / this.filterSize)) <= 95
+  }
+}
+
+// max load constants, defined in the cuckoo paper
+const MAX_LOAD = {
+  1: 0.5,
+  2: 0.84,
+  4: 0.95,
+  8: 0.98
+}
+
+function calculateBucketSize (errorRate: number = 0.001): 2 | 4 | 8 {
+  if (errorRate > 0.002) {
+    return 2
+  }
+
+  if (errorRate > 0.00001) {
+    return 4
+  }
+
+  return 8
+}
+
+export function optimize (maxItems: number, errorRate: number = 0.001): CuckooFilterInit {
+  // https://www.eecs.harvard.edu/~michaelm/postscripts/cuckoo-conext2014.pdf
+  // Section 5.1 Optimal Bucket Size
+  const bucketSize = calculateBucketSize(errorRate)
+  const load = MAX_LOAD[bucketSize]
+
+  // https://stackoverflow.com/questions/57555236/how-to-size-a-cuckoo-filter/57617208#57617208
+  const filterSize = Math.round(maxItems / load)
+  const fingerprintSize = Math.min(Math.ceil(Math.log(filterSize / bucketSize)) + 2, MAX_FINGERPRINT_SIZE)
+
+  return {
+    filterSize,
+    bucketSize,
+    fingerprintSize
+  }
+}
+
+export function createCuckooFilter (maxItems: number, errorRate: number = 0.005): Filter {
+  const opts = optimize(maxItems, errorRate)
+  return new CuckooFilter(opts)
+}

package/src/filters/fingerprint.ts

@@ -0,0 +1,44 @@
+import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
+import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
+import type { Hash } from './hashes'
+
+export const MAX_FINGERPRINT_SIZE = 64
+
+export class Fingerprint {
+  private readonly fp: Uint8Array
+  private readonly h: Hash
+  private readonly seed: number
+
+  constructor (buf: Uint8Array, hash: Hash, seed: number, fingerprintSize: number = 2) {
+    if (fingerprintSize > MAX_FINGERPRINT_SIZE) {
+      throw new TypeError('Invalid Fingerprint Size')
+    }
+
+    const fnv = hash.hashV(buf, seed)
+    const fp = uint8ArrayAlloc(fingerprintSize)
+
+    for (let i = 0; i < fp.length; i++) {
+      fp[i] = fnv[i]
+    }
+
+    if (fp.length === 0) {
+      fp[0] = 7
+    }
+
+    this.fp = fp
+    this.h = hash
+    this.seed = seed
+  }
+
+  hash (): number {
+    return this.h.hash(this.fp, this.seed)
+  }
+
+  equals (other?: any): boolean {
+    if (!(other?.fp instanceof Uint8Array)) {
+      return false
+    }
+
+    return uint8ArrayEquals(this.fp, other.fp)
+  }
+}

package/src/filters/hashes.ts

@@ -0,0 +1,38 @@
+import fnv1aHash from '@sindresorhus/fnv1a'
+import mur from 'murmurhash3js-revisited'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+
+export interface Hash {
+  hash(input: Uint8Array, seed: number): number
+  hashV(input: Uint8Array, seed: number): Uint8Array
+}
+
+export const murmur3: Hash = {
+  hash: (input, seed) => {
+    return mur.x86.hash32(input, seed)
+  },
+  hashV: (input, seed) => {
+    return numberToBuffer(murmur3.hash(input, seed))
+  }
+}
+
+export const fnv1a: Hash = {
+  hash: (input) => {
+    return Number(fnv1aHash(input, {
+      size: 32
+    }))
+  },
+  hashV: (input, seed) => {
+    return numberToBuffer(fnv1a.hash(input, seed))
+  }
+}
+
+export function numberToBuffer (num: bigint | number): Uint8Array {
+  let hex = num.toString(16)
+
+  if (hex.length % 2 === 1) {
+    hex = `0${hex}`
+  }
+
+  return uint8ArrayFromString(hex, 'base16')
+}

package/src/filters/index.ts

@@ -0,0 +1,9 @@
+export { BloomFilter, createBloomFilter, type BloomFilterOptions } from './bloom-filter.js'
+export { CuckooFilter, createCuckooFilter, type CuckooFilterInit } from './cuckoo-filter.js'
+export { ScalableCuckooFilter, createScalableCuckooFilter, type ScalableCuckooFilterInit } from './scalable-cuckoo-filter.js'
+
+export interface Filter {
+  add(item: Uint8Array | string): void
+  has(item: Uint8Array | string): boolean
+  remove?(buf: Uint8Array | string): boolean
+}

package/src/filters/scalable-cuckoo-filter.ts

@@ -0,0 +1,111 @@
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { CuckooFilter, optimize, type CuckooFilterInit } from './cuckoo-filter.js'
+import { fnv1a, type Hash } from './hashes.js'
+import { getRandomInt } from './utils.js'
+import type { Filter } from './index.js'
+
+export interface ScalableCuckooFilterInit extends CuckooFilterInit {
+  /**
+   * A number to multiply maxItems by when adding new sub-filters
+   */
+  scale?: number
+}
+
+export class ScalableCuckooFilter implements Filter {
+  private readonly filterSize: number
+  private readonly bucketSize: number
+  private readonly fingerprintSize: number
+  private readonly scale: number
+  private readonly filterSeries: CuckooFilter[]
+  private readonly hash: Hash
+  private readonly seed: number
+
+  constructor (init: ScalableCuckooFilterInit) {
+    this.bucketSize = init.bucketSize ?? 4
+    this.filterSize = init.filterSize ?? (1 << 18) / this.bucketSize
+    this.fingerprintSize = init.fingerprintSize ?? 2
+    this.scale = init.scale ?? 2
+    this.hash = init.hash ?? fnv1a
+    this.seed = init.seed ?? getRandomInt(0, Math.pow(2, 10))
+    this.filterSeries = [
+      new CuckooFilter({
+        filterSize: this.filterSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+    ]
+  }
+
+  add (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    if (this.has(item)) {
+      return true
+    }
+
+    let current = this.filterSeries.find((cuckoo) => {
+      return cuckoo.reliable
+    })
+
+    if (current == null) {
+      const curSize = this.filterSize * Math.pow(this.scale, this.filterSeries.length)
+
+      current = new CuckooFilter({
+        filterSize: curSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+
+      this.filterSeries.push(current)
+    }
+
+    return current.add(item)
+  }
+
+  has (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].has(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  remove (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].remove(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  get count (): number {
+    return this.filterSeries.reduce((acc, curr) => {
+      return acc + curr.count
+    }, 0)
+  }
+}
+
+export function createScalableCuckooFilter (maxItems: number, errorRate: number = 0.001, options?: Pick<ScalableCuckooFilterInit, 'hash' | 'seed' | 'scale'>): Filter {
+  return new ScalableCuckooFilter({
+    ...optimize(maxItems, errorRate),
+    ...(options ?? {})
+  })
+}

package/src/filters/utils.ts

@@ -0,0 +1,3 @@
+export function getRandomInt (min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min)) + min
+}

package/src/moving-average.ts

@@ -0,0 +1,45 @@
+/**
+ * Implements exponential moving average. Ported from `moving-average`.
+ *
+ * @see https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
+ * @see https://www.npmjs.com/package/moving-average
+ */
+export class MovingAverage {
+  public movingAverage: number
+  public variance: number
+  public deviation: number
+  public forecast: number
+  private readonly timespan: number
+  private previousTime?: number
+
+  constructor (timespan: number) {
+    this.timespan = timespan
+    this.movingAverage = 0
+    this.variance = 0
+    this.deviation = 0
+    this.forecast = 0
+  }
+
+  alpha (t: number, pt: number): number {
+    return 1 - (Math.exp(-(t - pt) / this.timespan))
+  }
+
+  push (value: number, time: number = Date.now()): void {
+    if (this.previousTime != null) {
+      // calculate moving average
+      const a = this.alpha(time, this.previousTime)
+      const diff = value - this.movingAverage
+      const incr = a * diff
+      this.movingAverage = a * value + (1 - a) * this.movingAverage
+      // calculate variance & deviation
+      this.variance = (1 - a) * (this.variance + diff * incr)
+      this.deviation = Math.sqrt(this.variance)
+      // calculate forecast
+      this.forecast = this.movingAverage + a * diff
+    } else {
+      this.movingAverage = value
+    }
+
+    this.previousTime = time
+  }
+}

package/src/peer-queue.ts

@@ -1,10 +1,8 @@
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-
-import { Queue, type QueueAddOptions } from './queue/index.js'
+import { Queue } from './queue/index.js'
 import type { Job } from './queue/job.js'
-import type { PeerId } from '@libp2p/interface'
+import type { AbortOptions, PeerId } from '@libp2p/interface'
 
-export interface PeerQueueJobOptions extends QueueAddOptions {
+export interface PeerQueueJobOptions extends AbortOptions {
   peerId: PeerId
 }
 

package/src/priority-queue.ts

@@ -0,0 +1,26 @@
+import { Queue } from './queue/index.js'
+import type { QueueInit } from './queue/index.js'
+import type { AbortOptions } from '@libp2p/interface'
+
+export interface PriorityQueueJobOptions extends AbortOptions {
+  priority: number
+}
+
+export class PriorityQueue <JobReturnType = void, JobOptions extends PriorityQueueJobOptions = PriorityQueueJobOptions> extends Queue<JobReturnType, JobOptions> {
+  constructor (init: QueueInit<JobReturnType, JobOptions> = {}) {
+    super({
+      ...init,
+      sort: (a, b) => {
+        if (a.options.priority > b.options.priority) {
+          return -1
+        }
+
+        if (a.options.priority < b.options.priority) {
+          return 1
+        }
+
+        return 0
+      }
+    })
+  }
+}

package/src/queue/index.ts

@@ -4,16 +4,11 @@ import { raceEvent } from 'race-event'
 import { Job } from './job.js'
 import type { AbortOptions, Metrics } from '@libp2p/interface'
 
-export interface QueueAddOptions extends AbortOptions {
-  /**
-   * Priority of operation. Operations with greater priority will be scheduled first.
-   *
-   * @default 0
-   */
-  priority?: number
+export interface Comparator<T> {
+  (a: T, b: T): -1 | 0 | 1
 }
 
-export interface QueueInit {
+export interface QueueInit<JobReturnType, JobOptions extends AbortOptions = AbortOptions> {
   /**
    * Concurrency limit.
    *
@@ -32,6 +27,11 @@ export interface QueueInit {
    * An implementation of the libp2p Metrics interface
    */
   metrics?: Metrics
+
+  /**
+   * An optional function that will sort the queue after a job has been added
+   */
+  sort?: Comparator<Job<JobOptions, JobReturnType>>
 }
 
 export type JobStatus = 'queued' | 'running' | 'errored' | 'complete'
@@ -40,21 +40,21 @@ export interface RunFunction<Options = AbortOptions, ReturnType = void> {
   (opts?: Options): Promise<ReturnType>
 }
 
-export interface JobMatcher<JobOptions extends QueueAddOptions = QueueAddOptions> {
+export interface JobMatcher<JobOptions extends AbortOptions = AbortOptions> {
   (options?: Partial<JobOptions>): boolean
 }
 
-export interface QueueJobSuccess<JobReturnType, JobOptions extends QueueAddOptions = QueueAddOptions> {
+export interface QueueJobSuccess<JobReturnType, JobOptions extends AbortOptions = AbortOptions> {
   job: Job<JobOptions, JobReturnType>
   result: JobReturnType
 }
 
-export interface QueueJobFailure<JobReturnType, JobOptions extends QueueAddOptions = QueueAddOptions> {
+export interface QueueJobFailure<JobReturnType, JobOptions extends AbortOptions = AbortOptions> {
   job: Job<JobOptions, JobReturnType>
   error: Error
 }
 
-export interface QueueEvents<JobReturnType, JobOptions extends QueueAddOptions = QueueAddOptions> {
+export interface QueueEvents<JobReturnType, JobOptions extends AbortOptions = AbortOptions> {
   /**
    * A job is about to start running
    */
@@ -103,39 +103,19 @@ export interface QueueEvents<JobReturnType, JobOptions extends QueueAddOptions =
   'failure': CustomEvent<QueueJobFailure<JobReturnType, JobOptions>>
 }
 
-// Port of lower_bound from https://en.cppreference.com/w/cpp/algorithm/lower_bound
-// Used to compute insertion index to keep queue sorted after insertion
-function lowerBound<T> (array: readonly T[], value: T, comparator: (a: T, b: T) => number): number {
-  let first = 0
-  let count = array.length
-
-  while (count > 0) {
-    const step = Math.trunc(count / 2)
-    let it = first + step
-
-    if (comparator(array[it], value) <= 0) {
-      first = ++it
-      count -= step + 1
-    } else {
-      count = step
-    }
-  }
-
-  return first
-}
-
 /**
  * Heavily influence by `p-queue` with the following differences:
  *
  * 1. Items remain at the head of the queue while they are running so `queue.size` includes `queue.pending` items - this is so interested parties can join the results of a queue item while it is running
  * 2. The options for a job are stored separately to the job in order for them to be modified while they are still in the queue
  */
-export class Queue<JobReturnType = unknown, JobOptions extends QueueAddOptions = QueueAddOptions> extends TypedEventEmitter<QueueEvents<JobReturnType, JobOptions>> {
+export class Queue<JobReturnType = unknown, JobOptions extends AbortOptions = AbortOptions> extends TypedEventEmitter<QueueEvents<JobReturnType, JobOptions>> {
   public concurrency: number
   public queue: Array<Job<JobOptions, JobReturnType>>
   private pending: number
+  private readonly sort?: Comparator<Job<JobOptions, JobReturnType>>
 
-  constructor (init: QueueInit = {}) {
+  constructor (init: QueueInit<JobReturnType, JobOptions> = {}) {
     super()
 
     this.concurrency = init.concurrency ?? Number.POSITIVE_INFINITY
@@ -153,6 +133,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends QueueAddOptions =
       })
     }
 
+    this.sort = init.sort
     this.queue = []
   }
 
@@ -215,16 +196,11 @@ export class Queue<JobReturnType = unknown, JobOptions extends QueueAddOptions =
   }
 
   private enqueue (job: Job<JobOptions, JobReturnType>): void {
-    if (this.queue[this.size - 1]?.priority >= job.priority) {
-      this.queue.push(job)
-      return
-    }
+    this.queue.push(job)
 
-    const index = lowerBound(
-      this.queue, job,
-      (a: Readonly< Job<JobOptions, JobReturnType>>, b: Readonly< Job<JobOptions, JobReturnType>>) => b.priority - a.priority
-    )
-    this.queue.splice(index, 0, job)
+    if (this.sort != null) {
+      this.queue.sort(this.sort)
+    }
   }
 
   /**
@@ -233,7 +209,7 @@ export class Queue<JobReturnType = unknown, JobOptions extends QueueAddOptions =
   async add (fn: RunFunction<JobOptions, JobReturnType>, options?: JobOptions): Promise<JobReturnType> {
     options?.signal?.throwIfAborted()
 
-    const job = new Job<JobOptions, JobReturnType>(fn, options, options?.priority)
+    const job = new Job<JobOptions, JobReturnType>(fn, options)
 
     const p = job.join(options)
       .then(result => {

package/src/queue/job.ts

@@ -21,17 +21,15 @@ export class Job <JobOptions extends AbortOptions = AbortOptions, JobReturnType
   public id: string
   public fn: (options: JobOptions) => Promise<JobReturnType>
   public options: JobOptions
-  public priority: number
   public recipients: Array<JobRecipient<JobReturnType>>
   public status: JobStatus
   public readonly timeline: JobTimeline
   private readonly controller: AbortController
 
-  constructor (fn: (options: JobOptions) => Promise<JobReturnType>, options: any, priority: number = 0) {
+  constructor (fn: (options: JobOptions) => Promise<JobReturnType>, options: any) {
     this.id = randomId()
     this.status = 'queued'
     this.fn = fn
-    this.priority = priority
     this.options = options
     this.recipients = []
     this.timeline = {