distillate 0.8.2 → 0.9.0

package/dist/hll/index.cjs

@@ -0,0 +1,362 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_serialize = require("../serialize-Ul2LKFJU.cjs");
+const require_params = require("../params-UTZbJ22c.cjs");
+const require_sizing = require("../sizing-BLzZk2zr.cjs");
+//#region src/hll/estimate.ts
+const ALPHA_INF = .5 / Math.LN2;
+function sigma(x) {
+	if (x === 1) return Infinity;
+	let y = 1;
+	let z = x;
+	let previous;
+	do {
+		x *= x;
+		previous = z;
+		z += x * y;
+		y += y;
+	} while (z !== previous);
+	return z;
+}
+function tau(x) {
+	if (x === 0 || x === 1) return 0;
+	let y = 1;
+	let z = 1 - x;
+	let previous;
+	do {
+		x = Math.sqrt(x);
+		previous = z;
+		y *= .5;
+		const gap = 1 - x;
+		z -= gap * gap * y;
+	} while (z !== previous);
+	return z / 3;
+}
+function estimate(hist, p) {
+	const m = 2 ** p;
+	const q = 64 - p;
+	let z = m * tau((m - (hist[q + 1] ?? 0)) / m);
+	for (let k = q; k >= 1; k--) z = .5 * (z + (hist[k] ?? 0));
+	z += m * sigma((hist[0] ?? 0) / m);
+	return ALPHA_INF * m * m / z;
+}
+//#endregion
+//#region src/hll/fold.ts
+function foldRho(index, rho, d) {
+	const mid = index & (1 << d) - 1;
+	if (mid === 0) return d + rho;
+	return Math.clz32(mid) - 32 + d + 1;
+}
+function foldDense(src, srcP, dst, dstP) {
+	const d = srcP - dstP;
+	for (let i = 0; i < 2 ** srcP; i++) {
+		const rho = src.get(i);
+		if (rho === 0) continue;
+		dst.raise(i >>> d, foldRho(i, rho, d));
+	}
+}
+//#endregion
+//#region src/hll/registers.ts
+const WIDTH = 6;
+const MASK = 63;
+var Registers = class {
+	#bytes;
+	constructor(p) {
+		this.#bytes = new Uint8Array(WIDTH * 2 ** p / 8);
+	}
+	get bytes() {
+		return this.#bytes;
+	}
+	max() {
+		const bytes = this.#bytes;
+		let max = 0;
+		for (let at = 0; at < bytes.length; at += 3) {
+			const word = (bytes[at] ?? 0) | (bytes[at + 1] ?? 0) << 8 | (bytes[at + 2] ?? 0) << 16;
+			const a = word & MASK;
+			const b = word >>> 6 & MASK;
+			const c = word >>> 12 & MASK;
+			const d = word >>> 18 & MASK;
+			if (a > max) max = a;
+			if (b > max) max = b;
+			if (c > max) max = c;
+			if (d > max) max = d;
+		}
+		return max;
+	}
+	get(i) {
+		const bit = i * WIDTH;
+		const at = bit >>> 3;
+		return ((this.#bytes[at] ?? 0) | (this.#bytes[at + 1] ?? 0) << 8) >>> (bit & 7) & MASK;
+	}
+	raise(i, v) {
+		if (v > this.get(i)) this.set(i, v);
+	}
+	set(i, v) {
+		const bit = i * WIDTH;
+		const at = bit >>> 3;
+		const off = bit & 7;
+		const word = ((this.#bytes[at] ?? 0) | (this.#bytes[at + 1] ?? 0) << 8) & ~(MASK << off) | (v & MASK) << off;
+		this.#bytes[at] = word & 255;
+		if (at + 1 < this.#bytes.length) this.#bytes[at + 1] = word >>> 8 & 255;
+	}
+};
+const RHO_BITS = 6;
+const RHO_MASK = (1 << RHO_BITS) - 1;
+function encodeSparse(index, rho) {
+	return index << RHO_BITS | rho;
+}
+function sparseIndex(entry) {
+	return entry >>> RHO_BITS;
+}
+function sparseRho(entry) {
+	return entry & RHO_MASK;
+}
+function foldSparse(buf, len, fromP, registers, toP) {
+	const d = fromP - toP;
+	for (let i = 0; i < len; i++) {
+		const entry = buf[i] ?? 0;
+		const at = sparseIndex(entry) >>> 25 - fromP;
+		registers.raise(at >>> d, foldRho(at, sparseRho(entry), d));
+	}
+}
+function refoldSparse(entry, fromP, toP) {
+	const index = sparseIndex(entry);
+	return encodeSparse(index, foldRho(index >>> 25 - fromP, sparseRho(entry), fromP - toP));
+}
+function compact(buf, len) {
+	if (len === 0) return 0;
+	buf.subarray(0, len).sort();
+	let out = 0;
+	for (let i = 0; i < len; i++) {
+		const entry = buf[i] ?? 0;
+		const next = buf[i + 1] ?? 0;
+		if (i + 1 < len && sparseIndex(next) === sparseIndex(entry)) continue;
+		buf[out++] = entry;
+	}
+	return out;
+}
+//#endregion
+//#region src/hll/hll.ts
+const ERROR_CONSTANT = 1.04;
+const TYPE = 5;
+const PARAMS_SIZE = 6;
+const DENSE = 0;
+const SPARSE = 1;
+const ENTRY_SIZE = 4;
+function assertPrecision(p) {
+	if (!Number.isInteger(p) || p < 4 || p > 18) throw new require_params.ParamError(`p must be an integer in [${String(4)}, ${String(18)}], got ${String(p)}`);
+}
+function sparseCapacity(registers) {
+	return registers.bytes.length >> 2;
+}
+var HyperLogLog = class HyperLogLog {
+	#registers;
+	#p;
+	#seed;
+	#scratch = {
+		w0: 0,
+		w1: 0,
+		w2: 0,
+		w3: 0
+	};
+	#sparse;
+	#len = 0;
+	constructor({ p, seed = 0 }) {
+		assertPrecision(p);
+		require_params.assertUint32(seed, "seed");
+		this.#registers = new Registers(p);
+		this.#p = p;
+		this.#seed = seed;
+		this.#sparse = new Int32Array(sparseCapacity(this.#registers));
+	}
+	static create(relativeError) {
+		return new HyperLogLog(require_sizing.hllSizing(relativeError));
+	}
+	static from(keys, relativeError) {
+		const sketch = HyperLogLog.create(relativeError);
+		for (const key of keys) sketch.add(key);
+		return sketch;
+	}
+	static fromBytes(bytes) {
+		const { type, flags, body } = require_serialize.readHeader(bytes);
+		if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
+		if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
+		require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "hll");
+		const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
+		const p = body[0] ?? 0;
+		const encoding = body[1] ?? 0;
+		const seed = dv.getUint32(2, true);
+		if (encoding !== DENSE && encoding !== SPARSE) throw new require_serialize.SerializationError(`unknown hll encoding ${String(encoding)}`);
+		const sketch = new HyperLogLog({
+			p,
+			seed
+		});
+		const payload = body.length - PARAMS_SIZE;
+		const maxRho = 65 - p;
+		if (encoding === DENSE) {
+			require_serialize.assertBodyLength(body.length, PARAMS_SIZE + sketch.#registers.bytes.length, "hll");
+			sketch.#goDense();
+			sketch.#registers.bytes.set(body.subarray(PARAMS_SIZE));
+			const largest = sketch.#registers.max();
+			if (largest > maxRho) throw new require_serialize.SerializationError(`hll: register holds ${String(largest)}, above the maximum ${String(maxRho)} at p ${String(p)}`);
+			return sketch;
+		}
+		const capacity = sparseCapacity(sketch.#registers);
+		if (payload % ENTRY_SIZE !== 0 || payload / ENTRY_SIZE > capacity) throw new require_serialize.TruncatedError(`hll: sparse payload of ${String(payload)} bytes is not up to ${String(capacity)} whole entries`);
+		const entries = payload / ENTRY_SIZE;
+		const buffer = new Int32Array(capacity);
+		for (let i = 0; i < entries; i++) {
+			const entry = dv.getUint32(PARAMS_SIZE + i * ENTRY_SIZE, true);
+			if (entry > 2147483647) throw new require_serialize.SerializationError(`hll: sparse entry ${String(entry)} does not fit the 31-bit encoding`);
+			const rho = sparseRho(entry);
+			if (rho > maxRho) throw new require_serialize.SerializationError(`hll: sparse entry holds rho ${String(rho)}, above the maximum ${String(maxRho)} at p ${String(p)}`);
+			buffer[i] = entry;
+		}
+		sketch.#sparse = buffer;
+		sketch.#len = entries;
+		return sketch;
+	}
+	toBytes() {
+		const sparse = this.#sparse;
+		if (sparse !== null) {
+			const entries = compact(sparse, this.#len);
+			this.#len = entries;
+			return require_serialize.writeFrame({
+				version: 4,
+				type: TYPE,
+				flags: 0
+			}, PARAMS_SIZE + entries * ENTRY_SIZE, (body, dv) => {
+				body[0] = this.#p;
+				body[1] = SPARSE;
+				dv.setUint32(2, this.#seed, true);
+				for (let i = 0; i < entries; i++) dv.setUint32(PARAMS_SIZE + i * ENTRY_SIZE, sparse[i] ?? 0, true);
+			});
+		}
+		const payload = this.#registers.bytes;
+		return require_serialize.writeFrame({
+			version: 4,
+			type: TYPE,
+			flags: 0
+		}, PARAMS_SIZE + payload.length, (body, dv) => {
+			body[0] = this.#p;
+			body[1] = DENSE;
+			dv.setUint32(2, this.#seed, true);
+			body.set(payload, PARAMS_SIZE);
+		});
+	}
+	toJSON() {
+		return require_serialize.toJSONEnvelope(this.toBytes());
+	}
+	static fromJSON(value) {
+		return HyperLogLog.fromBytes(require_serialize.fromJSONEnvelope(value));
+	}
+	get p() {
+		return this.#p;
+	}
+	get seed() {
+		return this.#seed;
+	}
+	get standardError() {
+		return ERROR_CONSTANT / Math.sqrt(2 ** this.#p);
+	}
+	add(key) {
+		const s = this.#scratch;
+		require_serialize.hash128KeyInto(key, this.#seed, s);
+		const p = this.#p;
+		const tail = s.w0 << p >>> 0;
+		const rho = tail !== 0 ? Math.clz32(tail) + 1 : 32 - p + Math.clz32(s.w1) + 1;
+		const sparse = this.#sparse;
+		if (sparse === null) {
+			this.#registers.raise(s.w0 >>> 32 - p, rho);
+			return;
+		}
+		sparse[this.#len++] = encodeSparse(s.w0 >>> 7, rho);
+		if (this.#len === sparse.length) this.#collapse(sparse);
+	}
+	#collapse(sparse) {
+		const distinct = compact(sparse, this.#len);
+		this.#len = distinct;
+		const slack = Math.max(1, sparse.length >> 2);
+		if (distinct > sparse.length - slack) this.#goDense();
+	}
+	union(other) {
+		if (other.#seed !== this.#seed) throw new require_params.ParamError(`cannot merge sketches with different seeds, got ${String(this.#seed)} and ${String(other.#seed)}`);
+		const p = Math.min(this.#p, other.#p);
+		const merged = new HyperLogLog({
+			p,
+			seed: this.#seed
+		});
+		merged.#drain(this);
+		merged.#drain(other);
+		return merged;
+	}
+	#drain(src) {
+		const sparse = src.#sparse;
+		if (sparse === null) {
+			this.#goDense();
+			foldDense(src.#registers, src.#p, this.#registers, this.#p);
+			return;
+		}
+		const distinct = compact(sparse, src.#len);
+		src.#len = distinct;
+		for (let i = 0; i < distinct; i++) this.#absorb(refoldSparse(sparse[i] ?? 0, src.#p, this.#p));
+	}
+	#absorb(entry) {
+		const sparse = this.#sparse;
+		if (sparse === null) {
+			const index = sparseIndex(entry) >>> 25 - this.#p;
+			this.#registers.raise(index, sparseRho(entry));
+			return;
+		}
+		sparse[this.#len++] = entry;
+		if (this.#len === sparse.length) this.#collapse(sparse);
+	}
+	#goDense() {
+		const sparse = this.#sparse;
+		if (sparse === null) return;
+		foldSparse(sparse, this.#len, this.#p, this.#registers, this.#p);
+		this.#sparse = null;
+		this.#len = 0;
+	}
+	#dense() {
+		const sparse = this.#sparse;
+		if (sparse === null) return this.#registers;
+		const out = new Registers(this.#p);
+		foldSparse(sparse, this.#len, this.#p, out, this.#p);
+		return out;
+	}
+	equals(other) {
+		if (other.#p !== this.#p || other.#seed !== this.#seed) return false;
+		const mine = this.#dense().bytes;
+		const theirs = other.#dense().bytes;
+		for (let i = 0; i < mine.length; i++) if (mine[i] !== theirs[i]) return false;
+		return true;
+	}
+	count() {
+		const sparse = this.#sparse;
+		if (sparse !== null) {
+			const distinct = compact(sparse, this.#len);
+			this.#len = distinct;
+			const buckets = 2 ** 25;
+			return Math.round(buckets * Math.log(buckets / (buckets - distinct)));
+		}
+		const p = this.#p;
+		const q = 64 - p;
+		const m = 2 ** p;
+		const hist = new Int32Array(q + 2);
+		for (let i = 0; i < m; i++) {
+			const value = this.#registers.get(i);
+			hist[value] = (hist[value] ?? 0) + 1;
+		}
+		return Math.round(estimate(hist, p));
+	}
+};
+//#endregion
+exports.BadMagicError = require_serialize.BadMagicError;
+exports.ChecksumError = require_serialize.ChecksumError;
+exports.HyperLogLog = HyperLogLog;
+exports.ParamError = require_params.ParamError;
+exports.SerializationError = require_serialize.SerializationError;
+exports.TruncatedError = require_serialize.TruncatedError;
+exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
+exports.UnknownVersionError = require_serialize.UnknownVersionError;
+exports.hllSizing = require_sizing.hllSizing;

package/dist/hll/index.d.cts

@@ -0,0 +1,119 @@
+import { a as TruncatedError, c as BytesLike, i as SerializationError, n as ChecksumError, o as UnknownHashVariantError, r as FilterJSON, s as UnknownVersionError, t as BadMagicError } from "../serialize-Bv1zWzrh.cjs";
+import { t as ParamError } from "../params-DnqJBqLS.cjs";
+import { i as hllSizing, n as HllSizing } from "../sizing-wYOW27eY.cjs";
+//#region src/hll/hll.d.ts
+/** Construction parameters for a {@link HyperLogLog} sketch. */
+interface HllParams {
+  /** Precision: the sketch holds `2 ** p` registers. */
+  p: number;
+  /** Hash seed; defaults to `0`. */
+  seed?: number;
+}
+/**
+ * A HyperLogLog sketch: estimates how many distinct keys it has seen, in space
+ * fixed by `p` rather than by the cardinality.
+ *
+ * @example
+ * ```ts
+ * const sketch = HyperLogLog.create(0.01);
+ * sketch.add("alice");
+ * ```
+ */
+declare class HyperLogLog {
+  #private;
+  /**
+   * Creates a sketch at an explicit precision.
+   *
+   * @param params - Precision and optional hash seed.
+   */
+  constructor({ p, seed }: HllParams);
+  /**
+   * Creates a sketch whose standard error meets `relativeError`.
+   *
+   * @param relativeError - Target relative error, e.g. `0.01` for 1%.
+   * @returns A new, empty sketch.
+   */
+  static create(relativeError: number): HyperLogLog;
+  /**
+   * Creates a sketch counting `keys`, sized to meet `relativeError`. The
+   * ergonomic entry point when the key set is already in hand.
+   *
+   * Needs no count of the keys first, unlike the filters: a sketch is sized by
+   * the error it targets, not by how many keys it will see.
+   *
+   * @param keys - The keys to record.
+   * @param relativeError - Target relative error, e.g. `0.01` for 1%.
+   * @returns A new sketch holding every key.
+   */
+  static from(keys: Iterable<BytesLike>, relativeError: number): HyperLogLog;
+  /**
+   * Restores a sketch from its {@link HyperLogLog.toBytes} serialization.
+   *
+   * @param bytes - The serialized sketch.
+   * @returns The reconstructed sketch.
+   */
+  static fromBytes(bytes: Uint8Array): HyperLogLog;
+  /**
+   * Serializes the sketch to a portable little-endian byte layout.
+   *
+   * @returns The serialized sketch, readable by
+   * {@link HyperLogLog.fromBytes}.
+   */
+  toBytes(): Uint8Array;
+  /**
+   * Serializes the sketch to a JSON-safe envelope wrapping
+   * {@link HyperLogLog.toBytes}.
+   *
+   * @returns The envelope, readable by {@link HyperLogLog.fromJSON}.
+   */
+  toJSON(): FilterJSON;
+  /**
+   * Restores a sketch from its {@link HyperLogLog.toJSON} envelope.
+   *
+   * @param value - The parsed envelope.
+   * @returns The reconstructed sketch.
+   */
+  static fromJSON(value: unknown): HyperLogLog;
+  /** Precision: the sketch holds `2 ** p` registers. */
+  get p(): number;
+  /** Hash seed the sketch was built with. */
+  get seed(): number;
+  /** Analytic relative standard error, `1.04 / sqrt(2 ** p)`. */
+  get standardError(): number;
+  /**
+   * Records a key. Adding a key already seen leaves the sketch unchanged, which
+   * is what lets it count distinct values without storing them.
+   *
+   * @param key - The key to record.
+   */
+  add(key: BytesLike): void;
+  /**
+   * Merges two sketches, giving a sketch that counts the keys either has seen.
+   *
+   * Both operands are left untouched.
+   *
+   * @param other - The sketch to merge with; must share this one's seed.
+   * @returns A new sketch holding both key sets.
+   * @throws {@link ParamError} if the seeds differ, since the two would have
+   * sent the same key to different registers and the merge would mean nothing.
+   */
+  union(other: HyperLogLog): HyperLogLog;
+  /**
+   * Whether two sketches hold the same registers at the same precision and
+   * seed. Which representation each is in does not enter into it.
+   *
+   * @param other - The sketch to compare against.
+   * @returns `true` if the two would answer every query identically.
+   */
+  equals(other: HyperLogLog): boolean;
+  /**
+   * Counts how many distinct keys have been added. Below a few thousand
+   * distinct keys the answer is exact; above that, and once the sketch has
+   * gone dense, it estimates within {@link standardError}.
+   *
+   * @returns The cardinality, a whole number; `0` for an empty sketch.
+   */
+  count(): number;
+}
+//#endregion
+export { BadMagicError, ChecksumError, type FilterJSON, type HllParams, type HllSizing, HyperLogLog, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, hllSizing };

package/dist/hll/index.d.ts

@@ -0,0 +1,119 @@
+import { a as TruncatedError, c as BytesLike, i as SerializationError, n as ChecksumError, o as UnknownHashVariantError, r as FilterJSON, s as UnknownVersionError, t as BadMagicError } from "../serialize-Bv1zWzrh.js";
+import { t as ParamError } from "../params-DnqJBqLS.js";
+import { i as hllSizing, n as HllSizing } from "../sizing-wYOW27eY.js";
+//#region src/hll/hll.d.ts
+/** Construction parameters for a {@link HyperLogLog} sketch. */
+interface HllParams {
+  /** Precision: the sketch holds `2 ** p` registers. */
+  p: number;
+  /** Hash seed; defaults to `0`. */
+  seed?: number;
+}
+/**
+ * A HyperLogLog sketch: estimates how many distinct keys it has seen, in space
+ * fixed by `p` rather than by the cardinality.
+ *
+ * @example
+ * ```ts
+ * const sketch = HyperLogLog.create(0.01);
+ * sketch.add("alice");
+ * ```
+ */
+declare class HyperLogLog {
+  #private;
+  /**
+   * Creates a sketch at an explicit precision.
+   *
+   * @param params - Precision and optional hash seed.
+   */
+  constructor({ p, seed }: HllParams);
+  /**
+   * Creates a sketch whose standard error meets `relativeError`.
+   *
+   * @param relativeError - Target relative error, e.g. `0.01` for 1%.
+   * @returns A new, empty sketch.
+   */
+  static create(relativeError: number): HyperLogLog;
+  /**
+   * Creates a sketch counting `keys`, sized to meet `relativeError`. The
+   * ergonomic entry point when the key set is already in hand.
+   *
+   * Needs no count of the keys first, unlike the filters: a sketch is sized by
+   * the error it targets, not by how many keys it will see.
+   *
+   * @param keys - The keys to record.
+   * @param relativeError - Target relative error, e.g. `0.01` for 1%.
+   * @returns A new sketch holding every key.
+   */
+  static from(keys: Iterable<BytesLike>, relativeError: number): HyperLogLog;
+  /**
+   * Restores a sketch from its {@link HyperLogLog.toBytes} serialization.
+   *
+   * @param bytes - The serialized sketch.
+   * @returns The reconstructed sketch.
+   */
+  static fromBytes(bytes: Uint8Array): HyperLogLog;
+  /**
+   * Serializes the sketch to a portable little-endian byte layout.
+   *
+   * @returns The serialized sketch, readable by
+   * {@link HyperLogLog.fromBytes}.
+   */
+  toBytes(): Uint8Array;
+  /**
+   * Serializes the sketch to a JSON-safe envelope wrapping
+   * {@link HyperLogLog.toBytes}.
+   *
+   * @returns The envelope, readable by {@link HyperLogLog.fromJSON}.
+   */
+  toJSON(): FilterJSON;
+  /**
+   * Restores a sketch from its {@link HyperLogLog.toJSON} envelope.
+   *
+   * @param value - The parsed envelope.
+   * @returns The reconstructed sketch.
+   */
+  static fromJSON(value: unknown): HyperLogLog;
+  /** Precision: the sketch holds `2 ** p` registers. */
+  get p(): number;
+  /** Hash seed the sketch was built with. */
+  get seed(): number;
+  /** Analytic relative standard error, `1.04 / sqrt(2 ** p)`. */
+  get standardError(): number;
+  /**
+   * Records a key. Adding a key already seen leaves the sketch unchanged, which
+   * is what lets it count distinct values without storing them.
+   *
+   * @param key - The key to record.
+   */
+  add(key: BytesLike): void;
+  /**
+   * Merges two sketches, giving a sketch that counts the keys either has seen.
+   *
+   * Both operands are left untouched.
+   *
+   * @param other - The sketch to merge with; must share this one's seed.
+   * @returns A new sketch holding both key sets.
+   * @throws {@link ParamError} if the seeds differ, since the two would have
+   * sent the same key to different registers and the merge would mean nothing.
+   */
+  union(other: HyperLogLog): HyperLogLog;
+  /**
+   * Whether two sketches hold the same registers at the same precision and
+   * seed. Which representation each is in does not enter into it.
+   *
+   * @param other - The sketch to compare against.
+   * @returns `true` if the two would answer every query identically.
+   */
+  equals(other: HyperLogLog): boolean;
+  /**
+   * Counts how many distinct keys have been added. Below a few thousand
+   * distinct keys the answer is exact; above that, and once the sketch has
+   * gone dense, it estimates within {@link standardError}.
+   *
+   * @returns The cardinality, a whole number; `0` for an empty sketch.
+   */
+  count(): number;
+}
+//#endregion
+export { BadMagicError, ChecksumError, type FilterJSON, type HllParams, type HllSizing, HyperLogLog, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, hllSizing };