@libp2p/utils 5.3.1 → 5.3.2-3d7a9da17

package/src/filters/scalable-cuckoo-filter.ts

@@ -0,0 +1,111 @@
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { CuckooFilter, optimize, type CuckooFilterInit } from './cuckoo-filter.js'
+import { fnv1a, type Hash } from './hashes.js'
+import { getRandomInt } from './utils.js'
+import type { Filter } from './index.js'
+
+export interface ScalableCuckooFilterInit extends CuckooFilterInit {
+  /**
+   * A number to multiply maxItems by when adding new sub-filters
+   */
+  scale?: number
+}
+
+export class ScalableCuckooFilter implements Filter {
+  private readonly filterSize: number
+  private readonly bucketSize: number
+  private readonly fingerprintSize: number
+  private readonly scale: number
+  private readonly filterSeries: CuckooFilter[]
+  private readonly hash: Hash
+  private readonly seed: number
+
+  constructor (init: ScalableCuckooFilterInit) {
+    this.bucketSize = init.bucketSize ?? 4
+    this.filterSize = init.filterSize ?? (1 << 18) / this.bucketSize
+    this.fingerprintSize = init.fingerprintSize ?? 2
+    this.scale = init.scale ?? 2
+    this.hash = init.hash ?? fnv1a
+    this.seed = init.seed ?? getRandomInt(0, Math.pow(2, 10))
+    this.filterSeries = [
+      new CuckooFilter({
+        filterSize: this.filterSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+    ]
+  }
+
+  add (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    if (this.has(item)) {
+      return true
+    }
+
+    let current = this.filterSeries.find((cuckoo) => {
+      return cuckoo.reliable
+    })
+
+    if (current == null) {
+      const curSize = this.filterSize * Math.pow(this.scale, this.filterSeries.length)
+
+      current = new CuckooFilter({
+        filterSize: curSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+
+      this.filterSeries.push(current)
+    }
+
+    return current.add(item)
+  }
+
+  has (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].has(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  remove (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].remove(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  get count (): number {
+    return this.filterSeries.reduce((acc, curr) => {
+      return acc + curr.count
+    }, 0)
+  }
+}
+
+export function createScalableCuckooFilter (maxItems: number, errorRate: number = 0.001, options?: Pick<ScalableCuckooFilterInit, 'hash' | 'seed' | 'scale'>): Filter {
+  return new ScalableCuckooFilter({
+    ...optimize(maxItems, errorRate),
+    ...(options ?? {})
+  })
+}

package/src/filters/utils.ts

@@ -0,0 +1,3 @@
+export function getRandomInt (min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min)) + min
+}

package/dist/typedoc-urls.json

@@ -1,76 +0,0 @@
-{
-  "AbstractStream": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.abstract_stream.AbstractStream.html",
-  "./abstract-stream:AbstractStream": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.abstract_stream.AbstractStream.html",
-  "AbstractStreamInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.abstract_stream.AbstractStreamInit.html",
-  "./abstract-stream:AbstractStreamInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.abstract_stream.AbstractStreamInit.html",
-  "certifiedAddressesFirst": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.certifiedAddressesFirst.html",
-  "./address-sort:certifiedAddressesFirst": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.certifiedAddressesFirst.html",
-  "circuitRelayAddressesLast": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.circuitRelayAddressesLast.html",
-  "./address-sort:circuitRelayAddressesLast": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.circuitRelayAddressesLast.html",
-  "defaultAddressSort": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.defaultAddressSort.html",
-  "./address-sort:defaultAddressSort": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.defaultAddressSort.html",
-  "publicAddressesFirst": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.publicAddressesFirst.html",
-  "./address-sort:publicAddressesFirst": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.address_sort.publicAddressesFirst.html",
-  "arrayEquals": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.array_equals.arrayEquals.html",
-  "./array-equals:arrayEquals": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.array_equals.arrayEquals.html",
-  "closeSource": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.close_source.closeSource.html",
-  "./close-source:closeSource": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.close_source.closeSource.html",
-  "Errors": "https://libp2p.github.io/js-libp2p/variables/_libp2p_utils.ip_port_to_multiaddr.Errors.html",
-  "./ip-port-to-multiaddr:Errors": "https://libp2p.github.io/js-libp2p/variables/_libp2p_utils.ip_port_to_multiaddr.Errors.html",
-  "ipPortToMultiaddr": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.ip_port_to_multiaddr.ipPortToMultiaddr.html",
-  "./ip-port-to-multiaddr:ipPortToMultiaddr": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.ip_port_to_multiaddr.ipPortToMultiaddr.html",
-  "isPromise": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.is_promise.isPromise.html",
-  "./is-promise:isPromise": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.is_promise.isPromise.html",
-  "isLoopback": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_loopback.isLoopback.html",
-  "./multiaddr/is-loopback:isLoopback": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_loopback.isLoopback.html",
-  "isPrivate": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_private.isPrivate.html",
-  "./multiaddr/is-private:isPrivate": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.multiaddr_is_private.isPrivate.html",
-  "PeerQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_queue.PeerQueue.html",
-  "./peer-queue:PeerQueue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.peer_queue.PeerQueue.html",
-  "PeerQueueJobOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_queue.PeerQueueJobOptions.html",
-  "./peer-queue:PeerQueueJobOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.peer_queue.PeerQueueJobOptions.html",
-  "isPrivateIp": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.private_ip.isPrivateIp.html",
-  "./private-ip:isPrivateIp": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.private_ip.isPrivateIp.html",
-  "Queue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.queue.Queue.html",
-  "./queue:Queue": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.queue.Queue.html",
-  "JobMatcher": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.JobMatcher.html",
-  "./queue:JobMatcher": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.JobMatcher.html",
-  "QueueAddOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueAddOptions.html",
-  "./queue:QueueAddOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueAddOptions.html",
-  "QueueEvents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueEvents.html",
-  "./queue:QueueEvents": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueEvents.html",
-  "QueueInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueInit.html",
-  "./queue:QueueInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueInit.html",
-  "QueueJobFailure": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueJobFailure.html",
-  "./queue:QueueJobFailure": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueJobFailure.html",
-  "QueueJobSuccess": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueJobSuccess.html",
-  "./queue:QueueJobSuccess": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.QueueJobSuccess.html",
-  "RunFunction": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.RunFunction.html",
-  "./queue:RunFunction": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.queue.RunFunction.html",
-  "JobStatus": "https://libp2p.github.io/js-libp2p/types/_libp2p_utils.queue.JobStatus.html",
-  "./queue:JobStatus": "https://libp2p.github.io/js-libp2p/types/_libp2p_utils.queue.JobStatus.html",
-  "RateLimiter": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.rate_limiter.RateLimiter.html",
-  "./rate-limiter:RateLimiter": "https://libp2p.github.io/js-libp2p/classes/_libp2p_utils.rate_limiter.RateLimiter.html",
-  "GetKeySecDurationOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.GetKeySecDurationOptions.html",
-  "./rate-limiter:GetKeySecDurationOptions": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.GetKeySecDurationOptions.html",
-  "RateLimiterInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateLimiterInit.html",
-  "./rate-limiter:RateLimiterInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateLimiterInit.html",
-  "RateLimiterResult": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateLimiterResult.html",
-  "./rate-limiter:RateLimiterResult": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateLimiterResult.html",
-  "RateRecord": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateRecord.html",
-  "./rate-limiter:RateRecord": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.rate_limiter.RateRecord.html",
-  "StreamProperties": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.stream_to_ma_conn.StreamProperties.html",
-  "./stream-to-ma-conn:StreamProperties": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.stream_to_ma_conn.StreamProperties.html",
-  "streamToMaConnection": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.stream_to_ma_conn.streamToMaConnection.html",
-  "./stream-to-ma-conn:streamToMaConnection": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.stream_to_ma_conn.streamToMaConnection.html",
-  "CreateTrackedListInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_list.CreateTrackedListInit.html",
-  "./tracked-list:CreateTrackedListInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_list.CreateTrackedListInit.html",
-  "trackedList": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_list.trackedList.html",
-  "./tracked-list:trackedList": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_list.trackedList.html",
-  "CreateTrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.CreateTrackedMapInit.html",
-  "./tracked-map:CreateTrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.CreateTrackedMapInit.html",
-  "TrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.TrackedMapInit.html",
-  "./tracked-map:TrackedMapInit": "https://libp2p.github.io/js-libp2p/interfaces/_libp2p_utils.tracked_map.TrackedMapInit.html",
-  "trackedMap": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_map.trackedMap.html",
-  "./tracked-map:trackedMap": "https://libp2p.github.io/js-libp2p/functions/_libp2p_utils.tracked_map.trackedMap.html"
-}