distillate 0.10.0 → 0.12.0

package/dist/scalable/index.d.ts

@@ -0,0 +1,173 @@
+import { t as BytesLike } from "../bytes-DCuYtUVS.js";
+import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
+import { t as ParamError } from "../params-DnqJBqLS.js";
+//#region src/scalable/scalable.d.ts
+/**
+ * Thrown by {@link ScalableBloomFilter.union} when the two filters were built
+ * with different settings. Stage `i` has the same geometry in both only when
+ * every setting matches, so no stage-by-stage merge exists otherwise.
+ */
+export declare class ScalableParamMismatchError extends Error {
+  /** Discriminates this error from other `Error`s. */
+  override readonly name = "ScalableParamMismatchError";
+}
+/** Settings for a {@link ScalableBloomFilter}. */
+interface ScalableBloomParams {
+  /** Keys the first stage holds before the next one opens. */
+  n: number;
+  /** Target false-positive rate for the whole chain, e.g. `0.01` for 1%. */
+  epsilon: number;
+  /** Capacity multiplier from one stage to the next; defaults to `2`. */
+  growth?: number;
+  /**
+   * Multiplier on each stage's false-positive target relative to the one
+   * before; defaults to `0.85`. Lower spends more bits per stage to open
+   * fewer of them.
+   */
+  tightening?: number;
+  /** Hash seed; defaults to `0`. */
+  seed?: number;
+}
+/** The optional {@link ScalableBloomParams}, as `create` and `from` take them. */
+type ScalableBloomOptions = Omit<ScalableBloomParams, "n" | "epsilon">;
+/**
+ * A Bloom filter that grows with its key count and keeps its false-positive
+ * rate under `epsilon`, for when the number of keys is not known up front.
+ *
+ * It is a chain of Bloom stages. Stage `i` holds `n * growth ** i` keys at a
+ * false-positive target of `epsilon * (1 - tightening) * tightening ** i`;
+ * those targets sum to at most `epsilon`, however many stages open.
+ *
+ * @example
+ * ```ts
+ * const seen = ScalableBloomFilter.create(1000, 0.01);
+ * seen.add("alice");
+ * seen.has("alice"); // true
+ * ```
+ */
+export declare class ScalableBloomFilter {
+  #private;
+  /**
+   * Creates a filter whose first stage holds `n` keys, for a chain-wide
+   * false-positive target.
+   *
+   * @param n - Keys the first stage holds.
+   * @param epsilon - Target false-positive rate for the whole chain.
+   * @param options - Optional growth, tightening and seed.
+   * @returns A new, empty filter.
+   */
+  static create(n: number, epsilon: number, options?: ScalableBloomOptions): ScalableBloomFilter;
+  /**
+   * Builds a filter from `keys`, with the first stage sized for their count;
+   * it keeps growing as more keys arrive. Duplicates only oversize the first
+   * stage, since a key already held is not counted again.
+   *
+   * @param keys - The keys to insert.
+   * @param epsilon - Target false-positive rate for the whole chain.
+   * @param options - Optional growth, tightening and seed.
+   * @returns A new filter containing every key.
+   */
+  static from(keys: Iterable<BytesLike>, epsilon: number, options?: ScalableBloomOptions): ScalableBloomFilter;
+  /**
+   * @param params - The filter's settings.
+   * @throws {@link ParamError} if a setting is out of range, or the first
+   *   stage would need more than `2^32 - 1` bits.
+   */
+  constructor({ n, epsilon, growth, tightening, seed }: ScalableBloomParams);
+  /**
+   * Adds a key. A key the filter already holds is not counted again, so
+   * duplicates never use up a stage's capacity.
+   *
+   * @param key - The key to insert, as a string or bytes.
+   * @throws RangeError if the next stage would need more than `2^32 - 1`
+   *   bits; the filter is left unchanged and still answers queries.
+   */
+  add(key: BytesLike): void;
+  /**
+   * Tests whether a key is in the set.
+   *
+   * @param key - The key to test.
+   * @returns `true` if present (possibly a false positive); `false` guarantees absence.
+   */
+  has(key: BytesLike): boolean;
+  /**
+   * Estimates the chain's current false-positive rate from each stage's actual
+   * fill: a key is a false positive if any stage wrongly holds it, so this is
+   * `1 - prod(1 - (setBits / m) ** k)`. It rises as keys are added and stays
+   * under `epsilon` while every stage is within its capacity.
+   *
+   * @returns The estimated false-positive rate, in `[0, 1]`.
+   */
+  rate(): number;
+  /**
+   * Returns a new filter holding every key of this filter and `other`. Stage
+   * `i` of the result is the OR of both inputs' stage `i`, and the result has
+   * the longer of the two chains. Each stage's count becomes the smaller of
+   * its capacity and the two counts summed: an upper bound, so the merged
+   * chain opens its next stage early rather than late.
+   *
+   * @param other - A filter built with identical settings.
+   * @returns A new filter; neither input is changed.
+   * @throws {@link ScalableParamMismatchError} if any setting, or the
+   *   geometry of a stage both filters have, differs.
+   */
+  union(other: ScalableBloomFilter): ScalableBloomFilter;
+  /**
+   * Tests structural equality: identical settings, the same stages with the
+   * same counts, and identical bits, which is exactly when the two serialize
+   * to the same bytes. Which stage a key lands in depends on when it arrived,
+   * so two filters given the same keys in a different order can be unequal.
+   *
+   * @param other - The filter to compare against.
+   * @returns `true` if the two filters are identical.
+   */
+  equals(other: ScalableBloomFilter): boolean;
+  /**
+   * Restores a filter from its {@link ScalableBloomFilter.toBytes} serialization.
+   *
+   * @param bytes - A frame produced by `toBytes`.
+   * @returns The reconstructed filter.
+   * @throws {@link SerializationError} (or a subclass) if the frame is malformed.
+   */
+  static fromBytes(bytes: Uint8Array): ScalableBloomFilter;
+  /**
+   * Serializes the filter to a portable little-endian frame, DSTL type 6.
+   *
+   * @returns The serialized filter, readable by {@link ScalableBloomFilter.fromBytes}.
+   */
+  toBytes(): Uint8Array;
+  /**
+   * Serializes the filter to a JSON-friendly envelope wrapping the base64 of
+   * {@link ScalableBloomFilter.toBytes}.
+   *
+   * @returns The JSON envelope.
+   */
+  toJSON(): FilterJSON;
+  /**
+   * Restores a filter from its {@link ScalableBloomFilter.toJSON} envelope.
+   *
+   * @param value - The JSON envelope.
+   * @returns The reconstructed filter.
+   */
+  static fromJSON(value: unknown): ScalableBloomFilter;
+  /** Keys added that the filter did not already hold. */
+  get count(): number;
+  /** Keys the open stages hold between them before the next one opens. */
+  get capacity(): number;
+  /** Number of open stages. */
+  get stages(): number;
+  /** Total bits across every stage. */
+  get m(): number;
+  /** Analytic design bits-per-key, `m / capacity`. */
+  get bitsPerKey(): number;
+  /** Hash seed shared by every stage. */
+  get seed(): number;
+  /** Target false-positive rate for the whole chain. */
+  get epsilon(): number;
+  /** Capacity multiplier from one stage to the next. */
+  get growth(): number;
+  /** False-positive target multiplier from one stage to the next. */
+  get tightening(): number;
+}
+//#endregion
+export { BadMagicError, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, type ScalableBloomOptions, type ScalableBloomParams, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };

package/dist/scalable/index.js

@@ -0,0 +1,337 @@
+import { t as BitSet } from "../bitset-Cf2jwRJX.js";
+import { i as hash128KeyInto, l as probeLanesInto, s as probeAt } from "../hasher-3Cyn7Yg-.js";
+import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
+import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
+import { t as bloomSizing } from "../sizing-BtUrtvF_.js";
+//#region src/scalable/scalable.ts
+const MAX_BITS = 4294967295;
+const TYPE = 6;
+const PARAMS_SIZE = 40;
+const ENTRY_SIZE = 16;
+const padded8 = (bytes) => Math.ceil(bytes / 8) * 8;
+let restoring;
+var ScalableParamMismatchError = class extends Error {
+	name = "ScalableParamMismatchError";
+};
+function readTable(view, stageCount) {
+	const entries = [];
+	for (let i = 0; i < stageCount; i++) {
+		const at = PARAMS_SIZE + ENTRY_SIZE * i;
+		const entry = {
+			m: view.getUint32(at, true),
+			k: view.getUint16(at + 4, true),
+			capacity: view.getUint32(at + 8, true),
+			count: view.getUint32(at + 12, true)
+		};
+		if (view.getUint16(at + 6, true) !== 0) throw new SerializationError(`scalable: stage ${String(i)} table padding is not zero`);
+		if (entry.m === 0 || entry.k === 0 || entry.capacity === 0) throw new SerializationError(`scalable: stage ${String(i)} declares m=${String(entry.m)}, k=${String(entry.k)}, capacity=${String(entry.capacity)}; all must be positive`);
+		if (entry.count > entry.capacity) throw new SerializationError(`scalable: stage ${String(i)} holds ${String(entry.count)} keys, above its capacity of ${String(entry.capacity)}`);
+		entries.push(entry);
+	}
+	return entries;
+}
+function readStages(body, entries, start) {
+	let at = start;
+	return entries.map((entry, i) => {
+		const length = Math.ceil(entry.m / 8);
+		const bits = new BitSet(entry.m);
+		bits.bytes.set(body.subarray(at, at + length));
+		for (let j = at + length; j < at + padded8(length); j++) if (body[j] !== 0) throw new SerializationError(`scalable: stage ${String(i)} bit padding is not zero`);
+		at += padded8(length);
+		return {
+			...entry,
+			bits
+		};
+	});
+}
+var ScalableBloomFilter = class ScalableBloomFilter {
+	#n;
+	#epsilon;
+	#growth;
+	#tightening;
+	#seed;
+	#stages = [];
+	#hash = {
+		w0: 0,
+		w1: 0,
+		w2: 0,
+		w3: 0
+	};
+	#probes = new Uint32Array(0);
+	#newest;
+	static create(n, epsilon, options = {}) {
+		return new ScalableBloomFilter({
+			...options,
+			n,
+			epsilon
+		});
+	}
+	static from(keys, epsilon, options = {}) {
+		const arr = [...keys];
+		const f = ScalableBloomFilter.create(Math.max(1, arr.length), epsilon, options);
+		for (const key of arr) f.add(key);
+		return f;
+	}
+	constructor({ n, epsilon, growth = 2, tightening = .85, seed = 0 }) {
+		assertPositiveInt(n, "n");
+		assertUint32(n, "n");
+		assertProbability(epsilon, "epsilon");
+		if (!Number.isFinite(growth) || growth <= 1) throw new ParamError(`growth must be a finite number greater than 1, got ${String(growth)}`);
+		assertProbability(tightening, "tightening");
+		assertUint32(seed, "seed");
+		this.#n = n;
+		this.#epsilon = epsilon;
+		this.#growth = growth;
+		this.#tightening = tightening;
+		this.#seed = seed;
+		if (restoring) {
+			this.#newest = this.#adopt(restoring);
+			return;
+		}
+		const first = this.#geometry(0);
+		if (first.m > MAX_BITS) throw new ParamError(`the first stage needs ${String(first.m)} bits, more than ${String(MAX_BITS)}`);
+		this.#newest = this.#open(first);
+	}
+	#geometry(i) {
+		const capacity = Math.ceil(this.#n * this.#growth ** i);
+		const target = this.#epsilon * (1 - this.#tightening) * this.#tightening ** i;
+		return {
+			capacity,
+			...bloomSizing(capacity, target)
+		};
+	}
+	#open({ capacity, m, k }) {
+		const stage = {
+			bits: new BitSet(m),
+			m,
+			k,
+			capacity,
+			count: 0
+		};
+		this.#stages.push(stage);
+		if (k > this.#probes.length) this.#probes = new Uint32Array(k);
+		return stage;
+	}
+	#settingPairs(other) {
+		return [
+			[
+				"n",
+				this.#n,
+				other.#n
+			],
+			[
+				"epsilon",
+				this.#epsilon,
+				other.#epsilon
+			],
+			[
+				"growth",
+				this.#growth,
+				other.#growth
+			],
+			[
+				"tightening",
+				this.#tightening,
+				other.#tightening
+			],
+			[
+				"seed",
+				this.#seed,
+				other.#seed
+			]
+		];
+	}
+	#assertMergeable(other) {
+		for (const [name, a, b] of this.#settingPairs(other)) if (a !== b) throw new ScalableParamMismatchError(`cannot union scalable Bloom filters whose ${name} differs (${String(a)} vs ${String(b)})`);
+		this.#stages.forEach((s, i) => {
+			const t = other.#stages[i];
+			if (!t) return;
+			for (const field of [
+				"m",
+				"k",
+				"capacity"
+			]) if (s[field] !== t[field]) throw new ScalableParamMismatchError(`cannot union scalable Bloom filters whose stage ${String(i)} ${field} differs (${String(s[field])} vs ${String(t[field])})`);
+		});
+	}
+	#adopt(stages) {
+		this.#stages.length = 0;
+		let k = 0;
+		for (const s of stages) {
+			this.#stages.push(s);
+			if (s.k > k) k = s.k;
+		}
+		if (k > this.#probes.length) this.#probes = new Uint32Array(k);
+		return stages.reduce((_, s) => s);
+	}
+	#probe(stage) {
+		probeLanesInto(this.#hash.w0, this.#hash.w1, stage.k, stage.m, this.#probes);
+	}
+	#holds(stage) {
+		const { w0, w1 } = this.#hash;
+		for (let i = 0; i < stage.k; i++) if (!stage.bits.get(probeAt(w0, w1, i, stage.m))) return false;
+		return true;
+	}
+	#anyHolds() {
+		for (let i = this.#stages.length - 1; i >= 0; i--) {
+			const stage = this.#stages[i];
+			if (stage && this.#holds(stage)) return true;
+		}
+		return false;
+	}
+	add(key) {
+		hash128KeyInto(key, this.#seed, this.#hash);
+		if (this.#anyHolds()) return;
+		if (this.#newest.count >= this.#newest.capacity) {
+			const i = this.#stages.length;
+			const next = this.#geometry(i);
+			if (next.m > MAX_BITS) throw new RangeError(`stage ${String(i)} needs ${String(next.m)} bits, more than ${String(MAX_BITS)}; the filter cannot grow further`);
+			this.#newest = this.#open(next);
+		}
+		const stage = this.#newest;
+		this.#probe(stage);
+		for (let i = 0; i < stage.k; i++) stage.bits.set(this.#probes[i] ?? 0);
+		stage.count++;
+	}
+	has(key) {
+		hash128KeyInto(key, this.#seed, this.#hash);
+		return this.#anyHolds();
+	}
+	rate() {
+		let miss = 1;
+		for (const s of this.#stages) miss *= 1 - (s.bits.count() / s.m) ** s.k;
+		return 1 - miss;
+	}
+	union(other) {
+		this.#assertMergeable(other);
+		const [longer, shorter] = this.#stages.length >= other.#stages.length ? [this.#stages, other.#stages] : [other.#stages, this.#stages];
+		const stages = longer.map((stage, i) => {
+			const bits = new BitSet(stage.m);
+			bits.bytes.set(stage.bits.bytes);
+			const twin = shorter[i];
+			if (twin) {
+				const out = bits.bytes;
+				const bytes = twin.bits.bytes;
+				for (let j = 0; j < out.length; j++) out[j] = (out[j] ?? 0) | (bytes[j] ?? 0);
+			}
+			return {
+				bits,
+				m: stage.m,
+				k: stage.k,
+				capacity: stage.capacity,
+				count: Math.min(stage.capacity, stage.count + (twin?.count ?? 0))
+			};
+		});
+		return ScalableBloomFilter.#withStages({
+			n: this.#n,
+			epsilon: this.#epsilon,
+			growth: this.#growth,
+			tightening: this.#tightening,
+			seed: this.#seed
+		}, stages);
+	}
+	equals(other) {
+		if (this.#settingPairs(other).some(([, a, b]) => a !== b)) return false;
+		if (this.#stages.length !== other.#stages.length) return false;
+		return this.#stages.every((s, i) => {
+			const t = other.#stages[i];
+			if (!t) return false;
+			return s.m === t.m && s.k === t.k && s.capacity === t.capacity && s.count === t.count && bytesEqual(s.bits.bytes, t.bits.bytes);
+		});
+	}
+	static fromBytes(bytes) {
+		const { type, flags, body } = readHeader(bytes);
+		if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
+		if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
+		assertMinBodyLength(body.length, PARAMS_SIZE, "scalable");
+		assertParamsPadding(body, 36, PARAMS_SIZE, "scalable");
+		const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
+		const stageCount = view.getUint32(32, true);
+		if (stageCount === 0) throw new SerializationError("scalable: frame declares no stages");
+		const tableEnd = PARAMS_SIZE + ENTRY_SIZE * stageCount;
+		assertMinBodyLength(body.length, tableEnd, "scalable");
+		const entries = readTable(view, stageCount);
+		const expected = entries.reduce((sum, e) => sum + padded8(Math.ceil(e.m / 8)), tableEnd);
+		assertBodyLength(body.length, expected, "scalable");
+		const stages = readStages(body, entries, tableEnd);
+		try {
+			return ScalableBloomFilter.#withStages({
+				n: view.getUint32(0, true),
+				seed: view.getUint32(4, true),
+				epsilon: view.getFloat64(8, true),
+				growth: view.getFloat64(16, true),
+				tightening: view.getFloat64(24, true)
+			}, stages);
+		} catch (err) {
+			if (err instanceof ParamError) throw new SerializationError(`scalable: ${err.message}`);
+			throw err;
+		}
+	}
+	static #withStages(params, stages) {
+		restoring = stages;
+		try {
+			return new ScalableBloomFilter(params);
+		} finally {
+			restoring = void 0;
+		}
+	}
+	toBytes() {
+		const payload = this.#stages.reduce((sum, s) => sum + ENTRY_SIZE + padded8(s.bits.bytes.length), 0);
+		return writeFrame({
+			version: 5,
+			type: TYPE,
+			flags: 0
+		}, PARAMS_SIZE, payload, (body, view) => {
+			view.setUint32(0, this.#n, true);
+			view.setUint32(4, this.#seed, true);
+			view.setFloat64(8, this.#epsilon, true);
+			view.setFloat64(16, this.#growth, true);
+			view.setFloat64(24, this.#tightening, true);
+			view.setUint32(32, this.#stages.length, true);
+			let at = PARAMS_SIZE + ENTRY_SIZE * this.#stages.length;
+			this.#stages.forEach((s, i) => {
+				const entry = PARAMS_SIZE + ENTRY_SIZE * i;
+				view.setUint32(entry, s.m, true);
+				view.setUint16(entry + 4, s.k, true);
+				view.setUint32(entry + 8, s.capacity, true);
+				view.setUint32(entry + 12, s.count, true);
+				body.set(s.bits.bytes, at);
+				at += padded8(s.bits.bytes.length);
+			});
+		});
+	}
+	toJSON() {
+		return toJSONEnvelope(this.toBytes());
+	}
+	static fromJSON(value) {
+		return ScalableBloomFilter.fromBytes(fromJSONEnvelope(value));
+	}
+	get count() {
+		return this.#stages.reduce((sum, s) => sum + s.count, 0);
+	}
+	get capacity() {
+		return this.#stages.reduce((sum, s) => sum + s.capacity, 0);
+	}
+	get stages() {
+		return this.#stages.length;
+	}
+	get m() {
+		return this.#stages.reduce((sum, s) => sum + s.m, 0);
+	}
+	get bitsPerKey() {
+		return this.m / this.capacity;
+	}
+	get seed() {
+		return this.#seed;
+	}
+	get epsilon() {
+		return this.#epsilon;
+	}
+	get growth() {
+		return this.#growth;
+	}
+	get tightening() {
+		return this.#tightening;
+	}
+};
+//#endregion
+export { BadMagicError, ChecksumError, ParamError, ReservedBitsError, ScalableBloomFilter, ScalableParamMismatchError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "distillate",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Probabilistic data structures for JavaScript. Space-efficient, approximate-membership filters (Bloom, Blocked Bloom, Binary Fuse) with tunable error and a portable binary format; zero dependencies, universal.",
   "keywords": [
     "bloom-filter",
@@ -71,6 +71,22 @@
       "import": "./dist/frame/index.js",
       "require": "./dist/frame/index.cjs"
     },
+    "./scalable": {
+      "types": {
+        "import": "./dist/scalable/index.d.ts",
+        "require": "./dist/scalable/index.d.cts"
+      },
+      "import": "./dist/scalable/index.js",
+      "require": "./dist/scalable/index.cjs"
+    },
+    "./cuckoo": {
+      "types": {
+        "import": "./dist/cuckoo/index.d.ts",
+        "require": "./dist/cuckoo/index.d.cts"
+      },
+      "import": "./dist/cuckoo/index.js",
+      "require": "./dist/cuckoo/index.cjs"
+    },
     "./package.json": "./package.json"
   },
   "main": "./dist/index.cjs",