distillate 0.4.0 → 0.6.0

package/README.md

@@ -24,17 +24,17 @@ npm install distillate
 # or: pnpm add distillate / bun add distillate / deno add npm:distillate
 ```
 
-Requires Node 20+ (or any modern Bun/Deno/browser/edge runtime).
+Requires Node 22+ (or any modern Bun/Deno/browser/edge runtime).
 
 ## Runtime support
 
 `distillate` targets ES2022 with zero runtime dependencies and no `eval`, so it runs on every modern JavaScript runtime:
 
-- **Node.js** 20, 22, 24 (LTS and current)
+- **Node.js** 22, 24 (LTS and current)
 - **Bun** and **Deno**
 - Browsers and Cloudflare/Vercel edge
 
-Every push runs a CI smoke matrix that imports the built package on Node 20/22/24, Bun, and Deno, so cross-runtime support is verified, not assumed.
+Every push runs a CI smoke matrix that imports the built package on Node 22/24, Bun, and Deno, so cross-runtime support is verified, not assumed.
 
 ## Structures
 
@@ -43,7 +43,7 @@ Each structure ships as its own subpath, so you only bundle what you import.
 | Import               | Structure     | Mutable? | Use for                                              |
 | -------------------- | ------------- | -------- | ---------------------------------------------------- |
 | `distillate/bloom`   | Classic Bloom | yes      | Familiar default, migration from `bloom-filters`     |
-| `distillate/blocked` | Blocked Bloom | yes      | Streaming inserts, speed-first, cache-friendly       |
+| `distillate/blocked` | Blocked Bloom | yes      | Faster lookups and a lower FPR for ~15% more space   |
 | `distillate/fuse`    | Binary Fuse   | no       | Static set built once and queried a lot; least space |
 
 ### Classic Bloom (`distillate/bloom`)
@@ -72,11 +72,11 @@ filter.add("alice");
 filter.has("alice"); // true
 ```
 
-Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Confines every lookup to one cache line, trading ~20-30% more space for cache-friendly throughput.
+Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Confines every lookup to a single cache line, so it is consistently faster than Classic across sizes (measured on Apple M5: ~7% faster at 100k keys, widening to ~1.4x once the filter outgrows CPU cache at tens of millions), and its sizing gives a lower false-positive rate. The cost is ~15% more space. Reach for it when lookup throughput matters; prefer Classic when space is tight.
 
 ### Binary Fuse (`distillate/fuse`)
 
-A **static** filter: built once from the full key set, then immutable. The most space-efficient option (~9 bits/key at ~0.39% FPR for 8-bit; ~19 bits/key at ~1/65536 for 16-bit).
+A **static** filter: built once from the full key set, then immutable. The most space-efficient option (~9 bits/key at ~0.39% FPR for 8-bit; ~19 bits/key at ~1/65536 for 16-bit), and it queries at ~11 M ops/s (Apple M5, 100k keys).
 
 ```ts
 import { BinaryFuse8, BinaryFuse16 } from "distillate/fuse";
@@ -98,12 +98,12 @@ Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same
 
 | Classic Bloom  | bits/key | measured FPR | `has` throughput |
 | -------------- | -------- | ------------ | ---------------- |
-| **distillate** | 9.59     | 1.03%        | ~7.0 M ops/s     |
+| **distillate** | 9.59     | 1.03%        | ~21 M ops/s      |
 | bloom-filters  | 9.59     | 0.99%        | ~0.29 M ops/s    |
 
-Same space, same accuracy, **~24x the lookup throughput** of [`bloom-filters`](https://www.npmjs.com/package/bloom-filters) (the package distillate replaces), while hashing UTF-8 bytes with MurmurHash3 so filters stay portable and cross-language readable.
+Same space, same accuracy, **~72x the lookup throughput** of [`bloom-filters`](https://www.npmjs.com/package/bloom-filters) (the package distillate replaces), while hashing UTF-8 bytes with MurmurHash3 so filters stay portable and cross-language readable.
 
-These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [distillate-bench](https://github.com/akshay-xp/distillate-bench) repo: [RESULTS.md](https://github.com/akshay-xp/distillate-bench/blob/main/RESULTS.md), [METHODOLOGY.md](https://github.com/akshay-xp/distillate-bench/blob/main/METHODOLOGY.md).
+These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [`apps/bench`](https://github.com/akshay-xp/distillate/tree/main/apps/bench) workspace: [RESULTS.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/RESULTS.md), [METHODOLOGY.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/METHODOLOGY.md).
 
 ## Docs
 

package/dist/blocked/index.cjs

@@ -1,15 +1,42 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_serialize = require("../serialize-DPdiySxq.cjs");
-const require_params = require("../params-J8p3bKq5.cjs");
+const require_serialize = require("../serialize-BIIKUHH6.cjs");
+const require_params = require("../params--8CNXYWu.cjs");
 //#region src/blocked/blocked.ts
 const TYPE = 2;
 const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
-const scratchHash = {
-	h1lo: 0,
-	h1hi: 0,
-	h2lo: 0,
-	h2hi: 0
-};
+const scratch2 = /* @__PURE__ */ new Uint32Array(2);
+const BLOCK_BITS = 256;
+const LANE_BITS = 32;
+const LANES = 8;
+/**
+* Modeled false-positive rate of a split-block filter at `bitsPerKey`. A block
+* holding `j` keys has FPR `(1 - (1 - 1/32)^j)^8` (8 lanes of 32 bits, one probe
+* each); the filter's rate averages that over the Poisson block load
+* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
+* is not linear in `log10(1/epsilon)`.
+*/
+function blockedFprAt(bitsPerKey) {
+	const lambda = BLOCK_BITS / bitsPerKey;
+	let fpr = 0;
+	let p = Math.exp(-lambda);
+	for (let j = 0;; j++) {
+		if (j > 0) p *= lambda / j;
+		fpr += p * (1 - (1 - 1 / LANE_BITS) ** j) ** LANES;
+		if (j > lambda && p < 1e-15) break;
+	}
+	return fpr;
+}
+const MAX_BITS_PER_KEY = 128;
+/**
+* Minimal integer bits-per-key whose modeled split-block FPR is at or below
+* `epsilon`. Throws {@link ParamError} when even the densest supported filter
+* cannot reach the target, so callers get a typed rejection instead of a
+* silently under-provisioned filter.
+*/
+function blockedBitsPerKey(epsilon) {
+	for (let bpk = 1; bpk <= MAX_BITS_PER_KEY; bpk++) if (blockedFprAt(bpk) <= epsilon) return bpk;
+	throw new require_params.ParamError(`epsilon ${String(epsilon)} is below the blocked-filter floor; use a classic or fuse filter`);
+}
 /** Thrown when an operation requires two filters built with identical parameters. */
 var BlockedBloomParamMismatchError = class extends Error {
 	/** Discriminates this error from other `Error`s. */
@@ -17,7 +44,7 @@ var BlockedBloomParamMismatchError = class extends Error {
 };
 /**
 * A blocked (split-block) Bloom filter: confines every lookup to a single cache
-* line, trading ~20-30% more space for cache-friendly throughput.
+* line, trading ~15% more space for higher lookup throughput and a lower FPR.
 *
 * @example
 * ```ts
@@ -33,11 +60,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	#n;
 	#words = /* @__PURE__ */ new Uint32Array(8);
 	#bits = /* @__PURE__ */ new Uint32Array(8);
-	static #ANCHORS = [
-		[2, 10.5],
-		[3, 16.9],
-		[4, 26.4]
-	];
 	/**
 	* Creates a filter sized for `n` expected keys at a target false-positive rate.
 	*
@@ -48,35 +70,62 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	static create(n, epsilon) {
 		require_params.assertPositiveInt(n, "n");
 		require_params.assertProbability(epsilon, "epsilon");
-		const t = Math.log10(1 / epsilon);
-		const a = BlockedBloomFilter.#ANCHORS;
-		let seg = a.findIndex((p) => t <= p[0]);
-		if (seg < 1) seg = seg === -1 ? a.length - 1 : 1;
-		const [t0, b0] = a[seg - 1] ?? [0, 0];
-		const [t1, b1] = a[seg] ?? [0, 0];
-		const bitsPerKey = b0 + (b1 - b0) / (t1 - t0) * (t - t0);
 		return new BlockedBloomFilter({
-			bitsPerKey: Math.max(1, Math.ceil(bitsPerKey)),
+			bitsPerKey: blockedBitsPerKey(epsilon),
 			capacity: n
 		});
 	}
 	/**
+	* Builds a filter from `keys`, sized for their count at the target
+	* false-positive rate. The ergonomic entry point when the key set is already
+	* in hand; use {@link BlockedBloomFilter.create} to size for a count known
+	* ahead.
+	*
+	* @param keys - The keys to insert.
+	* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
+	* @returns A new filter containing every key.
+	*/
+	static from(keys, epsilon) {
+		const arr = [...keys];
+		const f = BlockedBloomFilter.create(Math.max(1, arr.length), epsilon);
+		for (const k of arr) f.add(k);
+		return f;
+	}
+	/**
 	* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
 	* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
 	*/
 	constructor({ bitsPerKey, capacity, seed = 0 }) {
 		require_params.assertPositiveFinite(bitsPerKey, "bitsPerKey");
 		require_params.assertPositiveInt(capacity, "capacity");
+		require_params.assertUint32(capacity, "capacity");
 		require_params.assertUint32(seed, "seed");
 		this.#numBlocks = Math.max(1, Math.ceil(bitsPerKey * capacity / 256));
 		this.#lanes = new Uint32Array(this.#numBlocks * 8);
 		this.#seed = seed;
 		this.#n = capacity;
 	}
+	static #fromNumBlocks(numBlocks, seed, n) {
+		const f = new BlockedBloomFilter({
+			bitsPerKey: 256,
+			capacity: numBlocks,
+			seed
+		});
+		f.#n = n;
+		return f;
+	}
 	/** Actual bits allocated per key (`total bits / capacity`). */
 	get bitsPerKey() {
 		return this.#numBlocks * 256 / this.#n;
 	}
+	/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
+	get numBlocks() {
+		return this.#numBlocks;
+	}
+	/** Hash seed; the other field `union` requires to match. */
+	get seed() {
+		return this.#seed;
+	}
 	/** Number of bits currently set across all lanes. */
 	get length() {
 		let bits = 0;
@@ -104,20 +153,17 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	* @returns The reconstructed filter.
 	*/
 	static fromBytes(bytes) {
-		const { type, body } = require_serialize.readHeader(bytes);
+		const { type, flags, body } = require_serialize.readHeader(bytes);
 		if (type !== TYPE) throw new require_serialize.SerializationError(`expected AMQF type ${String(TYPE)}, got ${String(type)}`);
+		if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
 		require_serialize.assertMinBodyLength(body.length, 12, "blocked");
 		const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
 		const numBlocks = dv.getUint32(0, true);
 		const seed = dv.getUint32(4, true);
 		const n = dv.getUint32(8, true);
+		if (numBlocks === 0 || n === 0) throw new require_serialize.SerializationError(`blocked frame declares numBlocks=${String(numBlocks)}, n=${String(n)}; both must be positive`);
 		require_serialize.assertBodyLength(body.length, 12 + numBlocks * 32, "blocked");
-		const f = new BlockedBloomFilter({
-			bitsPerKey: 256,
-			capacity: numBlocks,
-			seed
-		});
-		f.#n = n;
+		const f = BlockedBloomFilter.#fromNumBlocks(numBlocks, seed, n);
 		new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
 		return f;
 	}
@@ -128,17 +174,44 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	*/
 	toBytes() {
 		const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
-		const body = new Uint8Array(12 + lanes.length);
-		const dv = new DataView(body.buffer);
-		dv.setUint32(0, this.#numBlocks, true);
-		dv.setUint32(4, this.#seed, true);
-		dv.setUint32(8, this.#n, true);
-		body.set(lanes, 12);
-		return require_serialize.writeHeader({
-			version: 2,
+		return require_serialize.writeFrame({
+			version: 3,
 			type: TYPE,
 			flags: 0
-		}, body);
+		}, 12 + lanes.length, (body, dv) => {
+			dv.setUint32(0, this.#numBlocks, true);
+			dv.setUint32(4, this.#seed, true);
+			dv.setUint32(8, this.#n, true);
+			body.set(lanes, 12);
+		});
+	}
+	/**
+	* Tests structural equality: `true` when `other` serializes to identical
+	* bytes, meaning identical parameters and set bits.
+	*
+	* @param other - The filter to compare against.
+	* @returns `true` if the two filters are byte-for-byte identical.
+	*/
+	equals(other) {
+		return require_serialize.bytesEqual(this.toBytes(), other.toBytes());
+	}
+	/**
+	* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
+	* {@link BlockedBloomFilter.toBytes}.
+	*
+	* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
+	*/
+	toJSON() {
+		return require_serialize.toJSONEnvelope(this.toBytes());
+	}
+	/**
+	* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
+	*
+	* @param value - The JSON envelope.
+	* @returns The reconstructed filter.
+	*/
+	static fromJSON(value) {
+		return BlockedBloomFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
 	}
 	/**
 	* Returns a new filter containing the union of this filter and `other`.
@@ -149,11 +222,7 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	*/
 	union(other) {
 		if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
-		const r = new BlockedBloomFilter({
-			bitsPerKey: this.bitsPerKey,
-			capacity: this.#n,
-			seed: this.#seed
-		});
+		const r = BlockedBloomFilter.#fromNumBlocks(this.#numBlocks, this.#seed, this.#n);
 		for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
 		return r;
 	}
@@ -185,9 +254,9 @@ var BlockedBloomFilter = class BlockedBloomFilter {
 	}
 };
 function fillBlock(key, numBlocks, seed, outWords, outBits) {
-	require_serialize.hash128KeyInto(key, seed, scratchHash);
-	const block = require_serialize.reduce((scratchHash.h1lo ^ scratchHash.h1hi) >>> 0, numBlocks);
-	const x = (scratchHash.h2lo ^ scratchHash.h2hi) >>> 0;
+	require_serialize.hash32x2Into(key, seed, scratch2);
+	const block = require_serialize.reduce(scratch2[0] ?? 0, numBlocks);
+	const x = scratch2[1] ?? 0;
 	const base = block * 8;
 	for (let i = 0; i < 8; i++) {
 		outWords[i] = base + i;

package/dist/blocked/index.d.cts

@@ -1,4 +1,4 @@
-import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
+import { n as BytesLike, t as FilterJSON } from "../serialize-DRKh6QOr.cjs";
 import { t as ParamError } from "../params-DnqJBqLS.cjs";
 //#region src/blocked/blocked.d.ts
 /** Thrown when an operation requires two filters built with identical parameters. */
@@ -17,7 +17,7 @@ interface BlockedBloomParams {
 }
 /**
  * A blocked (split-block) Bloom filter: confines every lookup to a single cache
- * line, trading ~20-30% more space for cache-friendly throughput.
+ * line, trading ~15% more space for higher lookup throughput and a lower FPR.
  *
  * @example
  * ```ts
@@ -36,6 +36,17 @@ declare class BlockedBloomFilter {
    * @returns A new, empty filter.
    */
   static create(n: number, epsilon: number): BlockedBloomFilter;
+  /**
+   * Builds a filter from `keys`, sized for their count at the target
+   * false-positive rate. The ergonomic entry point when the key set is already
+   * in hand; use {@link BlockedBloomFilter.create} to size for a count known
+   * ahead.
+   *
+   * @param keys - The keys to insert.
+   * @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
+   * @returns A new filter containing every key.
+   */
+  static from(keys: Iterable<BytesLike>, epsilon: number): BlockedBloomFilter;
   /**
    * Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
    * {@link BlockedBloomFilter.create} unless restoring a specific configuration.
@@ -43,6 +54,10 @@ declare class BlockedBloomFilter {
   constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
   /** Actual bits allocated per key (`total bits / capacity`). */
   get bitsPerKey(): number;
+  /** Number of 256-bit blocks; one of the two fields `union` requires to match. */
+  get numBlocks(): number;
+  /** Hash seed; the other field `union` requires to match. */
+  get seed(): number;
   /** Number of bits currently set across all lanes. */
   get length(): number;
   /**
@@ -67,6 +82,28 @@ declare class BlockedBloomFilter {
    * @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
    */
   toBytes(): Uint8Array;
+  /**
+   * Tests structural equality: `true` when `other` serializes to identical
+   * bytes, meaning identical parameters and set bits.
+   *
+   * @param other - The filter to compare against.
+   * @returns `true` if the two filters are byte-for-byte identical.
+   */
+  equals(other: BlockedBloomFilter): boolean;
+  /**
+   * Serializes the filter to a JSON-friendly envelope wrapping the base64 of
+   * {@link BlockedBloomFilter.toBytes}.
+   *
+   * @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
+   */
+  toJSON(): FilterJSON;
+  /**
+   * Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
+   *
+   * @param value - The JSON envelope.
+   * @returns The reconstructed filter.
+   */
+  static fromJSON(value: unknown): BlockedBloomFilter;
   /**
    * Returns a new filter containing the union of this filter and `other`.
    *
@@ -90,4 +127,4 @@ declare class BlockedBloomFilter {
   has(key: BytesLike): boolean;
 }
 //#endregion
-export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ParamError };
+export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError };

package/dist/blocked/index.d.ts

@@ -1,4 +1,4 @@
-import { t as BytesLike } from "../bytes-DCuYtUVS.js";
+import { n as BytesLike, t as FilterJSON } from "../serialize-DRKh6QOr.js";
 import { t as ParamError } from "../params-DnqJBqLS.js";
 //#region src/blocked/blocked.d.ts
 /** Thrown when an operation requires two filters built with identical parameters. */
@@ -17,7 +17,7 @@ interface BlockedBloomParams {
 }
 /**
  * A blocked (split-block) Bloom filter: confines every lookup to a single cache
- * line, trading ~20-30% more space for cache-friendly throughput.
+ * line, trading ~15% more space for higher lookup throughput and a lower FPR.
  *
  * @example
  * ```ts
@@ -36,6 +36,17 @@ declare class BlockedBloomFilter {
    * @returns A new, empty filter.
    */
   static create(n: number, epsilon: number): BlockedBloomFilter;
+  /**
+   * Builds a filter from `keys`, sized for their count at the target
+   * false-positive rate. The ergonomic entry point when the key set is already
+   * in hand; use {@link BlockedBloomFilter.create} to size for a count known
+   * ahead.
+   *
+   * @param keys - The keys to insert.
+   * @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
+   * @returns A new filter containing every key.
+   */
+  static from(keys: Iterable<BytesLike>, epsilon: number): BlockedBloomFilter;
   /**
    * Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
    * {@link BlockedBloomFilter.create} unless restoring a specific configuration.
@@ -43,6 +54,10 @@ declare class BlockedBloomFilter {
   constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
   /** Actual bits allocated per key (`total bits / capacity`). */
   get bitsPerKey(): number;
+  /** Number of 256-bit blocks; one of the two fields `union` requires to match. */
+  get numBlocks(): number;
+  /** Hash seed; the other field `union` requires to match. */
+  get seed(): number;
   /** Number of bits currently set across all lanes. */
   get length(): number;
   /**
@@ -67,6 +82,28 @@ declare class BlockedBloomFilter {
    * @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
    */
   toBytes(): Uint8Array;
+  /**
+   * Tests structural equality: `true` when `other` serializes to identical
+   * bytes, meaning identical parameters and set bits.
+   *
+   * @param other - The filter to compare against.
+   * @returns `true` if the two filters are byte-for-byte identical.
+   */
+  equals(other: BlockedBloomFilter): boolean;
+  /**
+   * Serializes the filter to a JSON-friendly envelope wrapping the base64 of
+   * {@link BlockedBloomFilter.toBytes}.
+   *
+   * @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
+   */
+  toJSON(): FilterJSON;
+  /**
+   * Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
+   *
+   * @param value - The JSON envelope.
+   * @returns The reconstructed filter.
+   */
+  static fromJSON(value: unknown): BlockedBloomFilter;
   /**
    * Returns a new filter containing the union of this filter and `other`.
    *
@@ -90,4 +127,4 @@ declare class BlockedBloomFilter {
   has(key: BytesLike): boolean;
 }
 //#endregion
-export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ParamError };
+export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError };