@libp2p/utils 5.3.2-e1923b0a7 → 5.4.0-1488a7371

package/src/filters/scalable-cuckoo-filter.ts

@@ -0,0 +1,111 @@
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { CuckooFilter, optimize, type CuckooFilterInit } from './cuckoo-filter.js'
+import { fnv1a, type Hash } from './hashes.js'
+import { getRandomInt } from './utils.js'
+import type { Filter } from './index.js'
+
+export interface ScalableCuckooFilterInit extends CuckooFilterInit {
+  /**
+   * A number to multiply maxItems by when adding new sub-filters
+   */
+  scale?: number
+}
+
+export class ScalableCuckooFilter implements Filter {
+  private readonly filterSize: number
+  private readonly bucketSize: number
+  private readonly fingerprintSize: number
+  private readonly scale: number
+  private readonly filterSeries: CuckooFilter[]
+  private readonly hash: Hash
+  private readonly seed: number
+
+  constructor (init: ScalableCuckooFilterInit) {
+    this.bucketSize = init.bucketSize ?? 4
+    this.filterSize = init.filterSize ?? (1 << 18) / this.bucketSize
+    this.fingerprintSize = init.fingerprintSize ?? 2
+    this.scale = init.scale ?? 2
+    this.hash = init.hash ?? fnv1a
+    this.seed = init.seed ?? getRandomInt(0, Math.pow(2, 10))
+    this.filterSeries = [
+      new CuckooFilter({
+        filterSize: this.filterSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+    ]
+  }
+
+  add (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    if (this.has(item)) {
+      return true
+    }
+
+    let current = this.filterSeries.find((cuckoo) => {
+      return cuckoo.reliable
+    })
+
+    if (current == null) {
+      const curSize = this.filterSize * Math.pow(this.scale, this.filterSeries.length)
+
+      current = new CuckooFilter({
+        filterSize: curSize,
+        bucketSize: this.bucketSize,
+        fingerprintSize: this.fingerprintSize,
+        hash: this.hash,
+        seed: this.seed
+      })
+
+      this.filterSeries.push(current)
+    }
+
+    return current.add(item)
+  }
+
+  has (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].has(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  remove (item: Uint8Array | string): boolean {
+    if (typeof item === 'string') {
+      item = uint8ArrayFromString(item)
+    }
+
+    for (let i = 0; i < this.filterSeries.length; i++) {
+      if (this.filterSeries[i].remove(item)) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  get count (): number {
+    return this.filterSeries.reduce((acc, curr) => {
+      return acc + curr.count
+    }, 0)
+  }
+}
+
+export function createScalableCuckooFilter (maxItems: number, errorRate: number = 0.001, options?: Pick<ScalableCuckooFilterInit, 'hash' | 'seed' | 'scale'>): Filter {
+  return new ScalableCuckooFilter({
+    ...optimize(maxItems, errorRate),
+    ...(options ?? {})
+  })
+}

package/src/filters/utils.ts

@@ -0,0 +1,3 @@
+export function getRandomInt (min: number, max: number): number {
+  return Math.floor(Math.random() * (max - min)) + min
+}

package/dist/src/bloom-filter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"bloom-filter.d.ts","sourceRoot":"","sources":["../../src/bloom-filter.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,qBAAa,WAAW;IACtB;;;OAGG;IACH,MAAM,CAAC,MAAM,CAAE,SAAS,EAAE,MAAM,EAAE,SAAS,GAAE,MAAc,GAAG,WAAW;IAKzE,SAAgB,KAAK,EAAE,MAAM,EAAE,CAAA;IAC/B,SAAgB,IAAI,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,UAAU,CAAA;gBAEZ,OAAO,GAAE,kBAAuB;IAW7C;;OAEG;IACH,GAAG,CAAE,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI;IAarC;;;;OAIG;IACH,GAAG,CAAE,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,OAAO;IAmBxC;;OAEG;IACH,KAAK,IAAK,IAAI;IAId,MAAM,CAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAa1B,MAAM,CAAE,GAAG,EAAE,MAAM,GAAG,OAAO;CAW9B"}

package/dist/src/bloom-filter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"bloom-filter.js","sourceRoot":"","sources":["../../src/bloom-filter.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,GAAG,MAAM,yBAAyB,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAE,UAAU,IAAI,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AAE5E,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAA;AAQvC,MAAM,OAAO,WAAW;IACtB;;;OAGG;IACH,MAAM,CAAC,MAAM,CAAE,SAAiB,EAAE,YAAoB,KAAK;QACzD,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;QAC3C,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,CAAA;IAC9B,CAAC;IAEe,KAAK,CAAU;IACf,IAAI,CAAQ;IACrB,MAAM,CAAY;IAEzB,YAAa,UAA8B,EAAE;QAC3C,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAA;QACjD,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED;;OAEG;IACH,GAAG,CAAE,IAAyB;QAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;QACnC,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;YAE5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAE,IAAyB;QAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;QACnC,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;YAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAE9B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,CAAE,GAAW;QACjB,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,KAAK,GAAG,GAAG,CAAA;QACf,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,GAAG,EAAE,CAAA;YACL,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC;QAED,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC/B,QAAQ,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAA;QAC1B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;IAC7B,CAAC;IAED,MAAM,CAAE,GAAW;QACjB,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,KAAK,GAAG,GAAG,CAAA;QACf,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,GAAG,EAAE,CAAA;YACL,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACjC,OAAO,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAA;IAC1C,CAAC;CACF;AAED,SAAS,QAAQ,CAAE,SAAiB,EAAE,YAAoB,KAAK;IAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,WAAW,CAAC,CAAA;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAA;IAExD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAED,SAAS,aAAa,CAAE,KAAa;IACnC,IAAI,GAAmB,CAAA;IACvB,IAAI,CAAS,CAAA;IACb,MAAM,KAAK,GAAG,EAAE,CAAA;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,GAAG,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QAEjC,sDAAsD;QACtD,kCAAkC;QAClC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACvB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1B,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC"}