distillate 0.9.0 → 0.11.0

package/dist/scalable/index.js

@@ -0,0 +1,330 @@
+import { t as BitSet } from "../bitset-Cf2jwRJX.js";
+import { i as hash128KeyInto, l as probeLanesInto, s as probeAt } from "../hasher-3Cyn7Yg-.js";
+import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
+import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
+import { t as bloomSizing } from "../sizing-BtUrtvF_.js";
+//#region src/scalable/scalable.ts
+const MAX_BITS = 4294967295;
+const TYPE = 6;
+const PARAMS_SIZE = 40;
+const ENTRY_SIZE = 16;
+const padded8 = (bytes) => Math.ceil(bytes / 8) * 8;
+let restoring;
+var ScalableParamMismatchError = class extends Error {
+	name = "ScalableParamMismatchError";
+};
+function readTable(view, stageCount) {
+	const entries = [];
+	for (let i = 0; i < stageCount; i++) {
+		const at = PARAMS_SIZE + ENTRY_SIZE * i;
+		const entry = {
+			m: view.getUint32(at, true),
+			k: view.getUint16(at + 4, true),
+			capacity: view.getUint32(at + 8, true),
+			count: view.getUint32(at + 12, true)
+		};
+		if (view.getUint16(at + 6, true) !== 0) throw new SerializationError(`scalable: stage ${String(i)} table padding is not zero`);
+		if (entry.m === 0 || entry.k === 0 || entry.capacity === 0) throw new SerializationError(`scalable: stage ${String(i)} declares m=${String(entry.m)}, k=${String(entry.k)}, capacity=${String(entry.capacity)}; all must be positive`);
+		if (entry.count > entry.capacity) throw new SerializationError(`scalable: stage ${String(i)} holds ${String(entry.count)} keys, above its capacity of ${String(entry.capacity)}`);
+		entries.push(entry);
+	}
+	return entries;
+}
+function readStages(body, entries, start) {
+	let at = start;
+	return entries.map((entry, i) => {
+		const length = Math.ceil(entry.m / 8);
+		const bits = new BitSet(entry.m);
+		bits.bytes.set(body.subarray(at, at + length));
+		for (let j = at + length; j < at + padded8(length); j++) if (body[j] !== 0) throw new SerializationError(`scalable: stage ${String(i)} bit padding is not zero`);
+		at += padded8(length);
+		return {
+			...entry,
+			bits
+		};
+	});
+}
+var ScalableBloomFilter = class ScalableBloomFilter {
+	#n;
+	#epsilon;
+	#growth;
+	#tightening;
+	#seed;
+	#stages = [];
+	#hash = {
+		w0: 0,
+		w1: 0,
+		w2: 0,
+		w3: 0
+	};
+	#probes = new Uint32Array(0);
+	#newest;
+	static create(n, epsilon, options = {}) {
+		return new ScalableBloomFilter({
+			...options,
+			n,
+			epsilon
+		});
+	}
+	static from(keys, epsilon, options = {}) {
+		const arr = [...keys];
+		const f = ScalableBloomFilter.create(Math.max(1, arr.length), epsilon, options);
+		for (const key of arr) f.add(key);
+		return f;
+	}
+	constructor({ n, epsilon, growth = 2, tightening = .85, seed = 0 }) {
+		assertPositiveInt(n, "n");
+		assertUint32(n, "n");
+		assertProbability(epsilon, "epsilon");
+		if (!Number.isFinite(growth) || growth <= 1) throw new ParamError(`growth must be a finite number greater than 1, got ${String(growth)}`);
+		assertProbability(tightening, "tightening");
+		assertUint32(seed, "seed");
+		this.#n = n;
+		this.#epsilon = epsilon;
+		this.#growth = growth;
+		this.#tightening = tightening;
+		this.#seed = seed;
+		if (restoring) {
+			this.#newest = this.#adopt(restoring);
+			return;
+		}
+		const first = this.#geometry(0);
+		if (first.m > MAX_BITS) throw new ParamError(`the first stage needs ${String(first.m)} bits, more than ${String(MAX_BITS)}`);
+		this.#newest = this.#open(first);
+	}
+	#geometry(i) {
+		const capacity = Math.ceil(this.#n * this.#growth ** i);
+		const target = this.#epsilon * (1 - this.#tightening) * this.#tightening ** i;
+		return {
+			capacity,
+			...bloomSizing(capacity, target)
+		};
+	}
+	#open({ capacity, m, k }) {
+		const stage = {
+			bits: new BitSet(m),
+			m,
+			k,
+			capacity,
+			count: 0
+		};
+		this.#stages.push(stage);
+		if (k > this.#probes.length) this.#probes = new Uint32Array(k);
+		return stage;
+	}
+	#settingPairs(other) {
+		return [
+			[
+				"n",
+				this.#n,
+				other.#n
+			],
+			[
+				"epsilon",
+				this.#epsilon,
+				other.#epsilon
+			],
+			[
+				"growth",
+				this.#growth,
+				other.#growth
+			],
+			[
+				"tightening",
+				this.#tightening,
+				other.#tightening
+			],
+			[
+				"seed",
+				this.#seed,
+				other.#seed
+			]
+		];
+	}
+	#assertMergeable(other) {
+		for (const [name, a, b] of this.#settingPairs(other)) if (a !== b) throw new ScalableParamMismatchError(`cannot union scalable Bloom filters whose ${name} differs (${String(a)} vs ${String(b)})`);
+		this.#stages.forEach((s, i) => {
+			const t = other.#stages[i];
+			if (!t) return;
+			for (const field of [
+				"m",
+				"k",
+				"capacity"
+			]) if (s[field] !== t[field]) throw new ScalableParamMismatchError(`cannot union scalable Bloom filters whose stage ${String(i)} ${field} differs (${String(s[field])} vs ${String(t[field])})`);
+		});
+	}
+	#adopt(stages) {
+		this.#stages.length = 0;
+		this.#stages.push(...stages);
+		const k = Math.max(...stages.map((s) => s.k));
+		if (k > this.#probes.length) this.#probes = new Uint32Array(k);
+		return stages.reduce((_, s) => s);
+	}
+	#probe(stage) {
+		probeLanesInto(this.#hash.w0, this.#hash.w1, stage.k, stage.m, this.#probes);
+	}
+	#holds(stage) {
+		const { w0, w1 } = this.#hash;
+		for (let i = 0; i < stage.k; i++) if (!stage.bits.get(probeAt(w0, w1, i, stage.m))) return false;
+		return true;
+	}
+	#anyHolds() {
+		for (let i = this.#stages.length - 1; i >= 0; i--) {
+			const stage = this.#stages[i];
+			if (stage && this.#holds(stage)) return true;
+		}
+		return false;
+	}
+	add(key) {
+		hash128KeyInto(key, this.#seed, this.#hash);
+		if (this.#anyHolds()) return;
+		if (this.#newest.count >= this.#newest.capacity) {
+			const i = this.#stages.length;
+			const next = this.#geometry(i);
+			if (next.m > MAX_BITS) throw new RangeError(`stage ${String(i)} needs ${String(next.m)} bits, more than ${String(MAX_BITS)}; the filter cannot grow further`);
+			this.#newest = this.#open(next);
+		}
+		const stage = this.#newest;
+		this.#probe(stage);
+		for (let i = 0; i < stage.k; i++) stage.bits.set(this.#probes[i] ?? 0);
+		stage.count++;
+	}
+	has(key) {
+		hash128KeyInto(key, this.#seed, this.#hash);
+		return this.#anyHolds();
+	}
+	rate() {
+		let miss = 1;
+		for (const s of this.#stages) miss *= 1 - (s.bits.count() / s.m) ** s.k;
+		return 1 - miss;
+	}
+	union(other) {
+		this.#assertMergeable(other);
+		const result = new ScalableBloomFilter({
+			n: this.#n,
+			epsilon: this.#epsilon,
+			growth: this.#growth,
+			tightening: this.#tightening,
+			seed: this.#seed
+		});
+		const [longer, shorter] = this.#stages.length >= other.#stages.length ? [this.#stages, other.#stages] : [other.#stages, this.#stages];
+		const stages = longer.map((stage, i) => {
+			const bits = new BitSet(stage.m);
+			bits.bytes.set(stage.bits.bytes);
+			const twin = shorter[i];
+			if (twin) {
+				const out = bits.bytes;
+				const bytes = twin.bits.bytes;
+				for (let j = 0; j < out.length; j++) out[j] = (out[j] ?? 0) | (bytes[j] ?? 0);
+			}
+			return {
+				bits,
+				m: stage.m,
+				k: stage.k,
+				capacity: stage.capacity,
+				count: Math.min(stage.capacity, stage.count + (twin?.count ?? 0))
+			};
+		});
+		result.#newest = result.#adopt(stages);
+		return result;
+	}
+	equals(other) {
+		if (this.#settingPairs(other).some(([, a, b]) => a !== b)) return false;
+		if (this.#stages.length !== other.#stages.length) return false;
+		return this.#stages.every((s, i) => {
+			const t = other.#stages[i];
+			if (!t) return false;
+			return s.m === t.m && s.k === t.k && s.capacity === t.capacity && s.count === t.count && bytesEqual(s.bits.bytes, t.bits.bytes);
+		});
+	}
+	static fromBytes(bytes) {
+		const { type, flags, body } = readHeader(bytes);
+		if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
+		if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
+		assertMinBodyLength(body.length, PARAMS_SIZE, "scalable");
+		assertParamsPadding(body, 36, PARAMS_SIZE, "scalable");
+		const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
+		const stageCount = view.getUint32(32, true);
+		if (stageCount === 0) throw new SerializationError("scalable: frame declares no stages");
+		const tableEnd = PARAMS_SIZE + ENTRY_SIZE * stageCount;
+		assertMinBodyLength(body.length, tableEnd, "scalable");
+		const entries = readTable(view, stageCount);
+		const expected = entries.reduce((sum, e) => sum + padded8(Math.ceil(e.m / 8)), tableEnd);
+		assertBodyLength(body.length, expected, "scalable");
+		restoring = readStages(body, entries, tableEnd);
+		try {
+			return new ScalableBloomFilter({
+				n: view.getUint32(0, true),
+				seed: view.getUint32(4, true),
+				epsilon: view.getFloat64(8, true),
+				growth: view.getFloat64(16, true),
+				tightening: view.getFloat64(24, true)
+			});
+		} catch (err) {
+			if (err instanceof ParamError) throw new SerializationError(`scalable: ${err.message}`);
+			throw err;
+		} finally {
+			restoring = void 0;
+		}
+	}
+	toBytes() {
+		const payload = this.#stages.reduce((sum, s) => sum + ENTRY_SIZE + padded8(s.bits.bytes.length), 0);
+		return writeFrame({
+			version: 5,
+			type: TYPE,
+			flags: 0
+		}, PARAMS_SIZE, payload, (body, view) => {
+			view.setUint32(0, this.#n, true);
+			view.setUint32(4, this.#seed, true);
+			view.setFloat64(8, this.#epsilon, true);
+			view.setFloat64(16, this.#growth, true);
+			view.setFloat64(24, this.#tightening, true);
+			view.setUint32(32, this.#stages.length, true);
+			let at = PARAMS_SIZE + ENTRY_SIZE * this.#stages.length;
+			this.#stages.forEach((s, i) => {
+				const entry = PARAMS_SIZE + ENTRY_SIZE * i;
+				view.setUint32(entry, s.m, true);
+				view.setUint16(entry + 4, s.k, true);
+				view.setUint32(entry + 8, s.capacity, true);
+				view.setUint32(entry + 12, s.count, true);
+				body.set(s.bits.bytes, at);
+				at += padded8(s.bits.bytes.length);
+			});
+		});
+	}
+	toJSON() {
+		return toJSONEnvelope(this.toBytes());
+	}
+	static fromJSON(value) {
+		return ScalableBloomFilter.fromBytes(fromJSONEnvelope(value));
+	}
+	get count() {
+		return this.#stages.reduce((sum, s) => sum + s.count, 0);
+	}
+	get capacity() {
+		return this.#stages.reduce((sum, s) => sum + s.capacity, 0);
+	}
+	get stages() {
+		return this.#stages.length;
+	}
+	get m() {
+		return this.#stages.reduce((sum, s) => sum + s.m, 0);
+	}
+	get bitsPerKey() {
+		return this.m / this.capacity;
+	}
+	get seed() {
+		return this.#seed;
+	}
+	get epsilon() {
+		return this.#epsilon;
+	}
+	get growth() {
+		return this.#growth;
+	}
+	get tightening() {
+		return this.#tightening;
+	}
+};
+//#endregion
+export { BadMagicError, ChecksumError, ParamError, ReservedBitsError, ScalableBloomFilter, ScalableParamMismatchError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };

package/dist/{serialize-Bv1zWzrh.d.cts → serialize--xR6vGSW.d.cts}

@@ -1,6 +1,3 @@
-//#region src/core/bytes.d.ts
-type BytesLike = string | Uint8Array | ArrayBuffer;
-//#endregion
 //#region src/core/serialize.d.ts
 /**
  * Base class for every defect `fromBytes` and `fromJSON` reject. Thrown
@@ -38,22 +35,36 @@ declare class BadMagicError extends SerializationError {
 }
 /**
  * Thrown when a frame or JSON envelope declares a format version this release
- * does not read. A reader must be at least as new as the producer, so upgrade
- * `distillate` or re-serialize the data with the version you run.
+ * does not read. The remedy depends on which side is older. A frame newer than
+ * this release needs `distillate` upgraded on the reading side. A frame older
+ * than it is readable by no current release, since each reads exactly one
+ * format version; rebuild it from the source keys. Being at least as new as
+ * the producer is necessary, not sufficient.
  */
 declare class UnknownVersionError extends SerializationError {
   /** Discriminates this error from other `Error`s. */
   override readonly name = "UnknownVersionError";
 }
 /**
- * Thrown when a frame's flags nibble names a hash this release cannot
- * reproduce, so its stored bits are unreadable. Rebuild the filter from the
+ * Thrown when a frame's flags nibble names a hash or index mapping this
+ * release cannot reproduce, so its stored bits are unreadable. Rebuild the filter from the
  * source keys with the version you run.
  */
 declare class UnknownHashVariantError extends SerializationError {
   /** Discriminates this error from other `Error`s. */
   override readonly name = "UnknownHashVariantError";
 }
+/**
+ * Thrown when a frame sets a header bit this format reserves: the flags high
+ * nibble, the byte at offset 7, or the word at offset 12. Only a newer writer
+ * puts anything there, and reading its frame under this release's meaning
+ * would give a wrong answer with no error. Upgrade `distillate` on the reading
+ * side; a reader must be at least as new as the producer.
+ */
+declare class ReservedBitsError extends SerializationError {
+  /** Discriminates this error from other `Error`s. */
+  override readonly name = "ReservedBitsError";
+}
 /**
  * Thrown when a frame's CRC32 trailer does not match its contents, so the
  * bytes were corrupted after they were written. Discard them and re-fetch;
@@ -72,5 +83,30 @@ interface FilterJSON {
   /** Base64 of the `toBytes` frame. */
   data: string;
 }
+/** One frame read in place from a larger buffer by {@link readFrameAt}. */
+interface Frame {
+  /** Binary format version. */
+  version: number;
+  /** Structure type code; one this release does not implement still reads. */
+  type: number;
+  /** Header flags byte; the low nibble is the hash variant. */
+  flags: number;
+  /** The body, a view into the source buffer rather than a copy. */
+  body: Uint8Array;
+  /** Bytes the whole frame occupies, so `offset + byteLength` is the next one. */
+  byteLength: number;
+}
+/**
+ * Reads and validates the frame starting at `offset` in `bytes`, which may
+ * hold further frames after it: magic, version, declared length, CRC32 and
+ * reserved bits, without needing to know the structure type. Advance by the
+ * returned `byteLength` to walk a buffer of concatenated frames.
+ *
+ * @throws TruncatedError when the frame runs past the end of `bytes`.
+ * @throws BadMagicError, UnknownVersionError, ChecksumError or
+ *   ReservedBitsError when the frame at `offset` is not one this release reads.
+ * @throws RangeError when `offset` is not a non-negative integer.
+ */
+declare function readFrameAt(bytes: Uint8Array, offset: number): Frame;
 //#endregion
-export { TruncatedError as a, BytesLike as c, SerializationError as i, ChecksumError as n, UnknownHashVariantError as o, FilterJSON as r, UnknownVersionError as s, BadMagicError as t };
+export { ReservedBitsError as a, UnknownHashVariantError as c, Frame as i, UnknownVersionError as l, ChecksumError as n, SerializationError as o, FilterJSON as r, TruncatedError as s, BadMagicError as t, readFrameAt as u };

package/dist/{serialize-Bv1zWzrh.d.ts → serialize--xR6vGSW.d.ts}

@@ -1,6 +1,3 @@
-//#region src/core/bytes.d.ts
-type BytesLike = string | Uint8Array | ArrayBuffer;
-//#endregion
 //#region src/core/serialize.d.ts
 /**
  * Base class for every defect `fromBytes` and `fromJSON` reject. Thrown
@@ -38,22 +35,36 @@ declare class BadMagicError extends SerializationError {
 }
 /**
  * Thrown when a frame or JSON envelope declares a format version this release
- * does not read. A reader must be at least as new as the producer, so upgrade
- * `distillate` or re-serialize the data with the version you run.
+ * does not read. The remedy depends on which side is older. A frame newer than
+ * this release needs `distillate` upgraded on the reading side. A frame older
+ * than it is readable by no current release, since each reads exactly one
+ * format version; rebuild it from the source keys. Being at least as new as
+ * the producer is necessary, not sufficient.
  */
 declare class UnknownVersionError extends SerializationError {
   /** Discriminates this error from other `Error`s. */
   override readonly name = "UnknownVersionError";
 }
 /**
- * Thrown when a frame's flags nibble names a hash this release cannot
- * reproduce, so its stored bits are unreadable. Rebuild the filter from the
+ * Thrown when a frame's flags nibble names a hash or index mapping this
+ * release cannot reproduce, so its stored bits are unreadable. Rebuild the filter from the
  * source keys with the version you run.
  */
 declare class UnknownHashVariantError extends SerializationError {
   /** Discriminates this error from other `Error`s. */
   override readonly name = "UnknownHashVariantError";
 }
+/**
+ * Thrown when a frame sets a header bit this format reserves: the flags high
+ * nibble, the byte at offset 7, or the word at offset 12. Only a newer writer
+ * puts anything there, and reading its frame under this release's meaning
+ * would give a wrong answer with no error. Upgrade `distillate` on the reading
+ * side; a reader must be at least as new as the producer.
+ */
+declare class ReservedBitsError extends SerializationError {
+  /** Discriminates this error from other `Error`s. */
+  override readonly name = "ReservedBitsError";
+}
 /**
  * Thrown when a frame's CRC32 trailer does not match its contents, so the
  * bytes were corrupted after they were written. Discard them and re-fetch;
@@ -72,5 +83,30 @@ interface FilterJSON {
   /** Base64 of the `toBytes` frame. */
   data: string;
 }
+/** One frame read in place from a larger buffer by {@link readFrameAt}. */
+interface Frame {
+  /** Binary format version. */
+  version: number;
+  /** Structure type code; one this release does not implement still reads. */
+  type: number;
+  /** Header flags byte; the low nibble is the hash variant. */
+  flags: number;
+  /** The body, a view into the source buffer rather than a copy. */
+  body: Uint8Array;
+  /** Bytes the whole frame occupies, so `offset + byteLength` is the next one. */
+  byteLength: number;
+}
+/**
+ * Reads and validates the frame starting at `offset` in `bytes`, which may
+ * hold further frames after it: magic, version, declared length, CRC32 and
+ * reserved bits, without needing to know the structure type. Advance by the
+ * returned `byteLength` to walk a buffer of concatenated frames.
+ *
+ * @throws TruncatedError when the frame runs past the end of `bytes`.
+ * @throws BadMagicError, UnknownVersionError, ChecksumError or
+ *   ReservedBitsError when the frame at `offset` is not one this release reads.
+ * @throws RangeError when `offset` is not a non-negative integer.
+ */
+declare function readFrameAt(bytes: Uint8Array, offset: number): Frame;
 //#endregion
-export { TruncatedError as a, BytesLike as c, SerializationError as i, ChecksumError as n, UnknownHashVariantError as o, FilterJSON as r, UnknownVersionError as s, BadMagicError as t };
+export { ReservedBitsError as a, UnknownHashVariantError as c, Frame as i, UnknownVersionError as l, ChecksumError as n, SerializationError as o, FilterJSON as r, TruncatedError as s, BadMagicError as t, readFrameAt as u };