distillate 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/blocked/index.cjs +111 -38
- package/dist/blocked/index.d.cts +40 -3
- package/dist/blocked/index.d.ts +40 -3
- package/dist/blocked/index.js +111 -38
- package/dist/bloom/index.cjs +69 -27
- package/dist/bloom/index.d.cts +34 -2
- package/dist/bloom/index.d.ts +34 -2
- package/dist/bloom/index.js +69 -27
- package/dist/fuse/index.cjs +63 -19
- package/dist/fuse/index.d.cts +34 -2
- package/dist/fuse/index.d.ts +34 -2
- package/dist/fuse/index.js +63 -19
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/{params-J8p3bKq5.cjs → params--8CNXYWu.cjs} +10 -0
- package/dist/{params-ChTRNxM9.js → params-BrWuBC2A.js} +5 -1
- package/dist/serialize-BIIKUHH6.cjs +515 -0
- package/dist/serialize-BnwtzcMw.js +414 -0
- package/dist/serialize-DRKh6QOr.d.cts +15 -0
- package/dist/serialize-DRKh6QOr.d.ts +15 -0
- package/package.json +5 -28
- package/dist/bytes-DCuYtUVS.d.cts +0 -4
- package/dist/bytes-DCuYtUVS.d.ts +0 -4
- package/dist/serialize-BQ2UzZqq.js +0 -354
- package/dist/serialize-fa-pEUGq.cjs +0 -437
package/README.md
CHANGED
|
@@ -24,17 +24,17 @@ npm install distillate
|
|
|
24
24
|
# or: pnpm add distillate / bun add distillate / deno add npm:distillate
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
-
Requires Node
|
|
27
|
+
Requires Node 22+ (or any modern Bun/Deno/browser/edge runtime).
|
|
28
28
|
|
|
29
29
|
## Runtime support
|
|
30
30
|
|
|
31
31
|
`distillate` targets ES2022 with zero runtime dependencies and no `eval`, so it runs on every modern JavaScript runtime:
|
|
32
32
|
|
|
33
|
-
- **Node.js**
|
|
33
|
+
- **Node.js** 22, 24 (LTS and current)
|
|
34
34
|
- **Bun** and **Deno**
|
|
35
35
|
- Browsers and Cloudflare/Vercel edge
|
|
36
36
|
|
|
37
|
-
Every push runs a CI smoke matrix that imports the built package on Node
|
|
37
|
+
Every push runs a CI smoke matrix that imports the built package on Node 22/24, Bun, and Deno, so cross-runtime support is verified, not assumed.
|
|
38
38
|
|
|
39
39
|
## Structures
|
|
40
40
|
|
|
@@ -43,7 +43,7 @@ Each structure ships as its own subpath, so you only bundle what you import.
|
|
|
43
43
|
| Import | Structure | Mutable? | Use for |
|
|
44
44
|
| -------------------- | ------------- | -------- | ---------------------------------------------------- |
|
|
45
45
|
| `distillate/bloom` | Classic Bloom | yes | Familiar default, migration from `bloom-filters` |
|
|
46
|
-
| `distillate/blocked` | Blocked Bloom | yes |
|
|
46
|
+
| `distillate/blocked` | Blocked Bloom | yes | Faster lookups and a lower FPR for ~15% more space |
|
|
47
47
|
| `distillate/fuse` | Binary Fuse | no | Static set built once and queried a lot; least space |
|
|
48
48
|
|
|
49
49
|
### Classic Bloom (`distillate/bloom`)
|
|
@@ -72,11 +72,11 @@ filter.add("alice");
|
|
|
72
72
|
filter.has("alice"); // true
|
|
73
73
|
```
|
|
74
74
|
|
|
75
|
-
Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Confines every lookup to a single cache line,
|
|
75
|
+
Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Confines every lookup to a single cache line, so it is consistently faster than Classic across sizes (measured on Apple M5: ~7% faster at 100k keys, widening to ~1.4x once the filter outgrows CPU cache at tens of millions), and its sizing gives a lower false-positive rate. The cost is ~15% more space. Reach for it when lookup throughput matters; prefer Classic when space is tight.
|
|
76
76
|
|
|
77
77
|
### Binary Fuse (`distillate/fuse`)
|
|
78
78
|
|
|
79
|
-
A **static** filter: built once from the full key set, then immutable. The most space-efficient option (~9 bits/key at ~0.39% FPR for 8-bit; ~19 bits/key at ~1/65536 for 16-bit).
|
|
79
|
+
A **static** filter: built once from the full key set, then immutable. The most space-efficient option (~9 bits/key at ~0.39% FPR for 8-bit; ~19 bits/key at ~1/65536 for 16-bit), and it queries at ~11 M ops/s (Apple M5, 100k keys).
|
|
80
80
|
|
|
81
81
|
```ts
|
|
82
82
|
import { BinaryFuse8, BinaryFuse16 } from "distillate/fuse";
|
|
@@ -98,12 +98,12 @@ Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same
|
|
|
98
98
|
|
|
99
99
|
| Classic Bloom | bits/key | measured FPR | `has` throughput |
|
|
100
100
|
| -------------- | -------- | ------------ | ---------------- |
|
|
101
|
-
| **distillate** | 9.59 | 1.03% | ~
|
|
101
|
+
| **distillate** | 9.59 | 1.03% | ~21 M ops/s |
|
|
102
102
|
| bloom-filters | 9.59 | 0.99% | ~0.29 M ops/s |
|
|
103
103
|
|
|
104
|
-
Same space, same accuracy, **~
|
|
104
|
+
Same space, same accuracy, **~72x the lookup throughput** of [`bloom-filters`](https://www.npmjs.com/package/bloom-filters) (the package distillate replaces), while hashing UTF-8 bytes with MurmurHash3 so filters stay portable and cross-language readable.
|
|
105
105
|
|
|
106
|
-
These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [
|
|
106
|
+
These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [`apps/bench`](https://github.com/akshay-xp/distillate/tree/main/apps/bench) workspace: [RESULTS.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/RESULTS.md), [METHODOLOGY.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/METHODOLOGY.md).
|
|
107
107
|
|
|
108
108
|
## Docs
|
|
109
109
|
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_serialize = require("../serialize-
|
|
3
|
-
const require_params = require("../params
|
|
2
|
+
const require_serialize = require("../serialize-BIIKUHH6.cjs");
|
|
3
|
+
const require_params = require("../params--8CNXYWu.cjs");
|
|
4
4
|
//#region src/blocked/blocked.ts
|
|
5
5
|
const TYPE = 2;
|
|
6
6
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
7
7
|
const scratch2 = /* @__PURE__ */ new Uint32Array(2);
|
|
8
|
+
const BLOCK_BITS = 256;
|
|
9
|
+
const LANE_BITS = 32;
|
|
10
|
+
const LANES = 8;
|
|
11
|
+
/**
|
|
12
|
+
* Modeled false-positive rate of a split-block filter at `bitsPerKey`. A block
|
|
13
|
+
* holding `j` keys has FPR `(1 - (1 - 1/32)^j)^8` (8 lanes of 32 bits, one probe
|
|
14
|
+
* each); the filter's rate averages that over the Poisson block load
|
|
15
|
+
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
16
|
+
* is not linear in `log10(1/epsilon)`.
|
|
17
|
+
*/
|
|
18
|
+
function blockedFprAt(bitsPerKey) {
|
|
19
|
+
const lambda = BLOCK_BITS / bitsPerKey;
|
|
20
|
+
let fpr = 0;
|
|
21
|
+
let p = Math.exp(-lambda);
|
|
22
|
+
for (let j = 0;; j++) {
|
|
23
|
+
if (j > 0) p *= lambda / j;
|
|
24
|
+
fpr += p * (1 - (1 - 1 / LANE_BITS) ** j) ** LANES;
|
|
25
|
+
if (j > lambda && p < 1e-15) break;
|
|
26
|
+
}
|
|
27
|
+
return fpr;
|
|
28
|
+
}
|
|
29
|
+
const MAX_BITS_PER_KEY = 128;
|
|
30
|
+
/**
|
|
31
|
+
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
32
|
+
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
33
|
+
* cannot reach the target, so callers get a typed rejection instead of a
|
|
34
|
+
* silently under-provisioned filter.
|
|
35
|
+
*/
|
|
36
|
+
function blockedBitsPerKey(epsilon) {
|
|
37
|
+
for (let bpk = 1; bpk <= MAX_BITS_PER_KEY; bpk++) if (blockedFprAt(bpk) <= epsilon) return bpk;
|
|
38
|
+
throw new require_params.ParamError(`epsilon ${String(epsilon)} is below the blocked-filter floor; use a classic or fuse filter`);
|
|
39
|
+
}
|
|
8
40
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
9
41
|
var BlockedBloomParamMismatchError = class extends Error {
|
|
10
42
|
/** Discriminates this error from other `Error`s. */
|
|
@@ -12,7 +44,7 @@ var BlockedBloomParamMismatchError = class extends Error {
|
|
|
12
44
|
};
|
|
13
45
|
/**
|
|
14
46
|
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
15
|
-
* line, trading ~
|
|
47
|
+
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
16
48
|
*
|
|
17
49
|
* @example
|
|
18
50
|
* ```ts
|
|
@@ -28,11 +60,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
28
60
|
#n;
|
|
29
61
|
#words = /* @__PURE__ */ new Uint32Array(8);
|
|
30
62
|
#bits = /* @__PURE__ */ new Uint32Array(8);
|
|
31
|
-
static #ANCHORS = [
|
|
32
|
-
[2, 10.5],
|
|
33
|
-
[3, 16.9],
|
|
34
|
-
[4, 26.4]
|
|
35
|
-
];
|
|
36
63
|
/**
|
|
37
64
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
38
65
|
*
|
|
@@ -43,35 +70,62 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
43
70
|
static create(n, epsilon) {
|
|
44
71
|
require_params.assertPositiveInt(n, "n");
|
|
45
72
|
require_params.assertProbability(epsilon, "epsilon");
|
|
46
|
-
const t = Math.log10(1 / epsilon);
|
|
47
|
-
const a = BlockedBloomFilter.#ANCHORS;
|
|
48
|
-
let seg = a.findIndex((p) => t <= p[0]);
|
|
49
|
-
if (seg < 1) seg = seg === -1 ? a.length - 1 : 1;
|
|
50
|
-
const [t0, b0] = a[seg - 1] ?? [0, 0];
|
|
51
|
-
const [t1, b1] = a[seg] ?? [0, 0];
|
|
52
|
-
const bitsPerKey = b0 + (b1 - b0) / (t1 - t0) * (t - t0);
|
|
53
73
|
return new BlockedBloomFilter({
|
|
54
|
-
bitsPerKey:
|
|
74
|
+
bitsPerKey: blockedBitsPerKey(epsilon),
|
|
55
75
|
capacity: n
|
|
56
76
|
});
|
|
57
77
|
}
|
|
58
78
|
/**
|
|
79
|
+
* Builds a filter from `keys`, sized for their count at the target
|
|
80
|
+
* false-positive rate. The ergonomic entry point when the key set is already
|
|
81
|
+
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
82
|
+
* ahead.
|
|
83
|
+
*
|
|
84
|
+
* @param keys - The keys to insert.
|
|
85
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
86
|
+
* @returns A new filter containing every key.
|
|
87
|
+
*/
|
|
88
|
+
static from(keys, epsilon) {
|
|
89
|
+
const arr = [...keys];
|
|
90
|
+
const f = BlockedBloomFilter.create(Math.max(1, arr.length), epsilon);
|
|
91
|
+
for (const k of arr) f.add(k);
|
|
92
|
+
return f;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
59
95
|
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
60
96
|
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
61
97
|
*/
|
|
62
98
|
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
63
99
|
require_params.assertPositiveFinite(bitsPerKey, "bitsPerKey");
|
|
64
100
|
require_params.assertPositiveInt(capacity, "capacity");
|
|
101
|
+
require_params.assertUint32(capacity, "capacity");
|
|
65
102
|
require_params.assertUint32(seed, "seed");
|
|
66
103
|
this.#numBlocks = Math.max(1, Math.ceil(bitsPerKey * capacity / 256));
|
|
67
104
|
this.#lanes = new Uint32Array(this.#numBlocks * 8);
|
|
68
105
|
this.#seed = seed;
|
|
69
106
|
this.#n = capacity;
|
|
70
107
|
}
|
|
108
|
+
static #fromNumBlocks(numBlocks, seed, n) {
|
|
109
|
+
const f = new BlockedBloomFilter({
|
|
110
|
+
bitsPerKey: 256,
|
|
111
|
+
capacity: numBlocks,
|
|
112
|
+
seed
|
|
113
|
+
});
|
|
114
|
+
f.#n = n;
|
|
115
|
+
return f;
|
|
116
|
+
}
|
|
71
117
|
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
72
118
|
get bitsPerKey() {
|
|
73
119
|
return this.#numBlocks * 256 / this.#n;
|
|
74
120
|
}
|
|
121
|
+
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
122
|
+
get numBlocks() {
|
|
123
|
+
return this.#numBlocks;
|
|
124
|
+
}
|
|
125
|
+
/** Hash seed; the other field `union` requires to match. */
|
|
126
|
+
get seed() {
|
|
127
|
+
return this.#seed;
|
|
128
|
+
}
|
|
75
129
|
/** Number of bits currently set across all lanes. */
|
|
76
130
|
get length() {
|
|
77
131
|
let bits = 0;
|
|
@@ -101,19 +155,15 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
101
155
|
static fromBytes(bytes) {
|
|
102
156
|
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
103
157
|
if (type !== TYPE) throw new require_serialize.SerializationError(`expected AMQF type ${String(TYPE)}, got ${String(type)}`);
|
|
104
|
-
if ((flags & 15) !==
|
|
158
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
105
159
|
require_serialize.assertMinBodyLength(body.length, 12, "blocked");
|
|
106
160
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
107
161
|
const numBlocks = dv.getUint32(0, true);
|
|
108
162
|
const seed = dv.getUint32(4, true);
|
|
109
163
|
const n = dv.getUint32(8, true);
|
|
164
|
+
if (numBlocks === 0 || n === 0) throw new require_serialize.SerializationError(`blocked frame declares numBlocks=${String(numBlocks)}, n=${String(n)}; both must be positive`);
|
|
110
165
|
require_serialize.assertBodyLength(body.length, 12 + numBlocks * 32, "blocked");
|
|
111
|
-
const f =
|
|
112
|
-
bitsPerKey: 256,
|
|
113
|
-
capacity: numBlocks,
|
|
114
|
-
seed
|
|
115
|
-
});
|
|
116
|
-
f.#n = n;
|
|
166
|
+
const f = BlockedBloomFilter.#fromNumBlocks(numBlocks, seed, n);
|
|
117
167
|
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
118
168
|
return f;
|
|
119
169
|
}
|
|
@@ -124,17 +174,44 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
124
174
|
*/
|
|
125
175
|
toBytes() {
|
|
126
176
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
dv.setUint32(0, this.#numBlocks, true);
|
|
130
|
-
dv.setUint32(4, this.#seed, true);
|
|
131
|
-
dv.setUint32(8, this.#n, true);
|
|
132
|
-
body.set(lanes, 12);
|
|
133
|
-
return require_serialize.writeHeader({
|
|
134
|
-
version: 2,
|
|
177
|
+
return require_serialize.writeFrame({
|
|
178
|
+
version: 3,
|
|
135
179
|
type: TYPE,
|
|
136
|
-
flags:
|
|
137
|
-
}, body)
|
|
180
|
+
flags: 0
|
|
181
|
+
}, 12 + lanes.length, (body, dv) => {
|
|
182
|
+
dv.setUint32(0, this.#numBlocks, true);
|
|
183
|
+
dv.setUint32(4, this.#seed, true);
|
|
184
|
+
dv.setUint32(8, this.#n, true);
|
|
185
|
+
body.set(lanes, 12);
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
190
|
+
* bytes, meaning identical parameters and set bits.
|
|
191
|
+
*
|
|
192
|
+
* @param other - The filter to compare against.
|
|
193
|
+
* @returns `true` if the two filters are byte-for-byte identical.
|
|
194
|
+
*/
|
|
195
|
+
equals(other) {
|
|
196
|
+
return require_serialize.bytesEqual(this.toBytes(), other.toBytes());
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
200
|
+
* {@link BlockedBloomFilter.toBytes}.
|
|
201
|
+
*
|
|
202
|
+
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
203
|
+
*/
|
|
204
|
+
toJSON() {
|
|
205
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
209
|
+
*
|
|
210
|
+
* @param value - The JSON envelope.
|
|
211
|
+
* @returns The reconstructed filter.
|
|
212
|
+
*/
|
|
213
|
+
static fromJSON(value) {
|
|
214
|
+
return BlockedBloomFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
138
215
|
}
|
|
139
216
|
/**
|
|
140
217
|
* Returns a new filter containing the union of this filter and `other`.
|
|
@@ -145,11 +222,7 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
145
222
|
*/
|
|
146
223
|
union(other) {
|
|
147
224
|
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
148
|
-
const r =
|
|
149
|
-
bitsPerKey: this.bitsPerKey,
|
|
150
|
-
capacity: this.#n,
|
|
151
|
-
seed: this.#seed
|
|
152
|
-
});
|
|
225
|
+
const r = BlockedBloomFilter.#fromNumBlocks(this.#numBlocks, this.#seed, this.#n);
|
|
153
226
|
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
154
227
|
return r;
|
|
155
228
|
}
|
package/dist/blocked/index.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { n as BytesLike, t as FilterJSON } from "../serialize-DRKh6QOr.cjs";
|
|
2
2
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
3
3
|
//#region src/blocked/blocked.d.ts
|
|
4
4
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
@@ -17,7 +17,7 @@ interface BlockedBloomParams {
|
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
19
|
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
20
|
-
* line, trading ~
|
|
20
|
+
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
21
21
|
*
|
|
22
22
|
* @example
|
|
23
23
|
* ```ts
|
|
@@ -36,6 +36,17 @@ declare class BlockedBloomFilter {
|
|
|
36
36
|
* @returns A new, empty filter.
|
|
37
37
|
*/
|
|
38
38
|
static create(n: number, epsilon: number): BlockedBloomFilter;
|
|
39
|
+
/**
|
|
40
|
+
* Builds a filter from `keys`, sized for their count at the target
|
|
41
|
+
* false-positive rate. The ergonomic entry point when the key set is already
|
|
42
|
+
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
43
|
+
* ahead.
|
|
44
|
+
*
|
|
45
|
+
* @param keys - The keys to insert.
|
|
46
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
47
|
+
* @returns A new filter containing every key.
|
|
48
|
+
*/
|
|
49
|
+
static from(keys: Iterable<BytesLike>, epsilon: number): BlockedBloomFilter;
|
|
39
50
|
/**
|
|
40
51
|
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
41
52
|
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
@@ -43,6 +54,10 @@ declare class BlockedBloomFilter {
|
|
|
43
54
|
constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
|
|
44
55
|
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
45
56
|
get bitsPerKey(): number;
|
|
57
|
+
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
58
|
+
get numBlocks(): number;
|
|
59
|
+
/** Hash seed; the other field `union` requires to match. */
|
|
60
|
+
get seed(): number;
|
|
46
61
|
/** Number of bits currently set across all lanes. */
|
|
47
62
|
get length(): number;
|
|
48
63
|
/**
|
|
@@ -67,6 +82,28 @@ declare class BlockedBloomFilter {
|
|
|
67
82
|
* @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
|
|
68
83
|
*/
|
|
69
84
|
toBytes(): Uint8Array;
|
|
85
|
+
/**
|
|
86
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
87
|
+
* bytes, meaning identical parameters and set bits.
|
|
88
|
+
*
|
|
89
|
+
* @param other - The filter to compare against.
|
|
90
|
+
* @returns `true` if the two filters are byte-for-byte identical.
|
|
91
|
+
*/
|
|
92
|
+
equals(other: BlockedBloomFilter): boolean;
|
|
93
|
+
/**
|
|
94
|
+
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
95
|
+
* {@link BlockedBloomFilter.toBytes}.
|
|
96
|
+
*
|
|
97
|
+
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
98
|
+
*/
|
|
99
|
+
toJSON(): FilterJSON;
|
|
100
|
+
/**
|
|
101
|
+
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
102
|
+
*
|
|
103
|
+
* @param value - The JSON envelope.
|
|
104
|
+
* @returns The reconstructed filter.
|
|
105
|
+
*/
|
|
106
|
+
static fromJSON(value: unknown): BlockedBloomFilter;
|
|
70
107
|
/**
|
|
71
108
|
* Returns a new filter containing the union of this filter and `other`.
|
|
72
109
|
*
|
|
@@ -90,4 +127,4 @@ declare class BlockedBloomFilter {
|
|
|
90
127
|
has(key: BytesLike): boolean;
|
|
91
128
|
}
|
|
92
129
|
//#endregion
|
|
93
|
-
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ParamError };
|
|
130
|
+
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError };
|
package/dist/blocked/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { n as BytesLike, t as FilterJSON } from "../serialize-DRKh6QOr.js";
|
|
2
2
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
3
3
|
//#region src/blocked/blocked.d.ts
|
|
4
4
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
@@ -17,7 +17,7 @@ interface BlockedBloomParams {
|
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
19
|
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
20
|
-
* line, trading ~
|
|
20
|
+
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
21
21
|
*
|
|
22
22
|
* @example
|
|
23
23
|
* ```ts
|
|
@@ -36,6 +36,17 @@ declare class BlockedBloomFilter {
|
|
|
36
36
|
* @returns A new, empty filter.
|
|
37
37
|
*/
|
|
38
38
|
static create(n: number, epsilon: number): BlockedBloomFilter;
|
|
39
|
+
/**
|
|
40
|
+
* Builds a filter from `keys`, sized for their count at the target
|
|
41
|
+
* false-positive rate. The ergonomic entry point when the key set is already
|
|
42
|
+
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
43
|
+
* ahead.
|
|
44
|
+
*
|
|
45
|
+
* @param keys - The keys to insert.
|
|
46
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
47
|
+
* @returns A new filter containing every key.
|
|
48
|
+
*/
|
|
49
|
+
static from(keys: Iterable<BytesLike>, epsilon: number): BlockedBloomFilter;
|
|
39
50
|
/**
|
|
40
51
|
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
41
52
|
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
@@ -43,6 +54,10 @@ declare class BlockedBloomFilter {
|
|
|
43
54
|
constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
|
|
44
55
|
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
45
56
|
get bitsPerKey(): number;
|
|
57
|
+
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
58
|
+
get numBlocks(): number;
|
|
59
|
+
/** Hash seed; the other field `union` requires to match. */
|
|
60
|
+
get seed(): number;
|
|
46
61
|
/** Number of bits currently set across all lanes. */
|
|
47
62
|
get length(): number;
|
|
48
63
|
/**
|
|
@@ -67,6 +82,28 @@ declare class BlockedBloomFilter {
|
|
|
67
82
|
* @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
|
|
68
83
|
*/
|
|
69
84
|
toBytes(): Uint8Array;
|
|
85
|
+
/**
|
|
86
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
87
|
+
* bytes, meaning identical parameters and set bits.
|
|
88
|
+
*
|
|
89
|
+
* @param other - The filter to compare against.
|
|
90
|
+
* @returns `true` if the two filters are byte-for-byte identical.
|
|
91
|
+
*/
|
|
92
|
+
equals(other: BlockedBloomFilter): boolean;
|
|
93
|
+
/**
|
|
94
|
+
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
95
|
+
* {@link BlockedBloomFilter.toBytes}.
|
|
96
|
+
*
|
|
97
|
+
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
98
|
+
*/
|
|
99
|
+
toJSON(): FilterJSON;
|
|
100
|
+
/**
|
|
101
|
+
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
102
|
+
*
|
|
103
|
+
* @param value - The JSON envelope.
|
|
104
|
+
* @returns The reconstructed filter.
|
|
105
|
+
*/
|
|
106
|
+
static fromJSON(value: unknown): BlockedBloomFilter;
|
|
70
107
|
/**
|
|
71
108
|
* Returns a new filter containing the union of this filter and `other`.
|
|
72
109
|
*
|
|
@@ -90,4 +127,4 @@ declare class BlockedBloomFilter {
|
|
|
90
127
|
has(key: BytesLike): boolean;
|
|
91
128
|
}
|
|
92
129
|
//#endregion
|
|
93
|
-
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ParamError };
|
|
130
|
+
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError };
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,9 +1,41 @@
|
|
|
1
|
-
import { a as
|
|
2
|
-
import {
|
|
1
|
+
import { _ as reduce, a as bytesEqual, c as toJSONEnvelope, i as assertMinBodyLength, l as writeFrame, m as hash32x2Into, n as UnknownHashVariantError, o as fromJSONEnvelope, r as assertBodyLength, s as readHeader, t as SerializationError } from "../serialize-BnwtzcMw.js";
|
|
2
|
+
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-BrWuBC2A.js";
|
|
3
3
|
//#region src/blocked/blocked.ts
|
|
4
4
|
const TYPE = 2;
|
|
5
5
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
6
6
|
const scratch2 = /* @__PURE__ */ new Uint32Array(2);
|
|
7
|
+
const BLOCK_BITS = 256;
|
|
8
|
+
const LANE_BITS = 32;
|
|
9
|
+
const LANES = 8;
|
|
10
|
+
/**
|
|
11
|
+
* Modeled false-positive rate of a split-block filter at `bitsPerKey`. A block
|
|
12
|
+
* holding `j` keys has FPR `(1 - (1 - 1/32)^j)^8` (8 lanes of 32 bits, one probe
|
|
13
|
+
* each); the filter's rate averages that over the Poisson block load
|
|
14
|
+
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
15
|
+
* is not linear in `log10(1/epsilon)`.
|
|
16
|
+
*/
|
|
17
|
+
function blockedFprAt(bitsPerKey) {
|
|
18
|
+
const lambda = BLOCK_BITS / bitsPerKey;
|
|
19
|
+
let fpr = 0;
|
|
20
|
+
let p = Math.exp(-lambda);
|
|
21
|
+
for (let j = 0;; j++) {
|
|
22
|
+
if (j > 0) p *= lambda / j;
|
|
23
|
+
fpr += p * (1 - (1 - 1 / LANE_BITS) ** j) ** LANES;
|
|
24
|
+
if (j > lambda && p < 1e-15) break;
|
|
25
|
+
}
|
|
26
|
+
return fpr;
|
|
27
|
+
}
|
|
28
|
+
const MAX_BITS_PER_KEY = 128;
|
|
29
|
+
/**
|
|
30
|
+
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
31
|
+
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
32
|
+
* cannot reach the target, so callers get a typed rejection instead of a
|
|
33
|
+
* silently under-provisioned filter.
|
|
34
|
+
*/
|
|
35
|
+
function blockedBitsPerKey(epsilon) {
|
|
36
|
+
for (let bpk = 1; bpk <= MAX_BITS_PER_KEY; bpk++) if (blockedFprAt(bpk) <= epsilon) return bpk;
|
|
37
|
+
throw new ParamError(`epsilon ${String(epsilon)} is below the blocked-filter floor; use a classic or fuse filter`);
|
|
38
|
+
}
|
|
7
39
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
8
40
|
var BlockedBloomParamMismatchError = class extends Error {
|
|
9
41
|
/** Discriminates this error from other `Error`s. */
|
|
@@ -11,7 +43,7 @@ var BlockedBloomParamMismatchError = class extends Error {
|
|
|
11
43
|
};
|
|
12
44
|
/**
|
|
13
45
|
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
14
|
-
* line, trading ~
|
|
46
|
+
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
15
47
|
*
|
|
16
48
|
* @example
|
|
17
49
|
* ```ts
|
|
@@ -27,11 +59,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
27
59
|
#n;
|
|
28
60
|
#words = /* @__PURE__ */ new Uint32Array(8);
|
|
29
61
|
#bits = /* @__PURE__ */ new Uint32Array(8);
|
|
30
|
-
static #ANCHORS = [
|
|
31
|
-
[2, 10.5],
|
|
32
|
-
[3, 16.9],
|
|
33
|
-
[4, 26.4]
|
|
34
|
-
];
|
|
35
62
|
/**
|
|
36
63
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
37
64
|
*
|
|
@@ -42,35 +69,62 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
42
69
|
static create(n, epsilon) {
|
|
43
70
|
assertPositiveInt(n, "n");
|
|
44
71
|
assertProbability(epsilon, "epsilon");
|
|
45
|
-
const t = Math.log10(1 / epsilon);
|
|
46
|
-
const a = BlockedBloomFilter.#ANCHORS;
|
|
47
|
-
let seg = a.findIndex((p) => t <= p[0]);
|
|
48
|
-
if (seg < 1) seg = seg === -1 ? a.length - 1 : 1;
|
|
49
|
-
const [t0, b0] = a[seg - 1] ?? [0, 0];
|
|
50
|
-
const [t1, b1] = a[seg] ?? [0, 0];
|
|
51
|
-
const bitsPerKey = b0 + (b1 - b0) / (t1 - t0) * (t - t0);
|
|
52
72
|
return new BlockedBloomFilter({
|
|
53
|
-
bitsPerKey:
|
|
73
|
+
bitsPerKey: blockedBitsPerKey(epsilon),
|
|
54
74
|
capacity: n
|
|
55
75
|
});
|
|
56
76
|
}
|
|
57
77
|
/**
|
|
78
|
+
* Builds a filter from `keys`, sized for their count at the target
|
|
79
|
+
* false-positive rate. The ergonomic entry point when the key set is already
|
|
80
|
+
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
81
|
+
* ahead.
|
|
82
|
+
*
|
|
83
|
+
* @param keys - The keys to insert.
|
|
84
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
85
|
+
* @returns A new filter containing every key.
|
|
86
|
+
*/
|
|
87
|
+
static from(keys, epsilon) {
|
|
88
|
+
const arr = [...keys];
|
|
89
|
+
const f = BlockedBloomFilter.create(Math.max(1, arr.length), epsilon);
|
|
90
|
+
for (const k of arr) f.add(k);
|
|
91
|
+
return f;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
58
94
|
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
59
95
|
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
60
96
|
*/
|
|
61
97
|
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
62
98
|
assertPositiveFinite(bitsPerKey, "bitsPerKey");
|
|
63
99
|
assertPositiveInt(capacity, "capacity");
|
|
100
|
+
assertUint32(capacity, "capacity");
|
|
64
101
|
assertUint32(seed, "seed");
|
|
65
102
|
this.#numBlocks = Math.max(1, Math.ceil(bitsPerKey * capacity / 256));
|
|
66
103
|
this.#lanes = new Uint32Array(this.#numBlocks * 8);
|
|
67
104
|
this.#seed = seed;
|
|
68
105
|
this.#n = capacity;
|
|
69
106
|
}
|
|
107
|
+
static #fromNumBlocks(numBlocks, seed, n) {
|
|
108
|
+
const f = new BlockedBloomFilter({
|
|
109
|
+
bitsPerKey: 256,
|
|
110
|
+
capacity: numBlocks,
|
|
111
|
+
seed
|
|
112
|
+
});
|
|
113
|
+
f.#n = n;
|
|
114
|
+
return f;
|
|
115
|
+
}
|
|
70
116
|
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
71
117
|
get bitsPerKey() {
|
|
72
118
|
return this.#numBlocks * 256 / this.#n;
|
|
73
119
|
}
|
|
120
|
+
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
121
|
+
get numBlocks() {
|
|
122
|
+
return this.#numBlocks;
|
|
123
|
+
}
|
|
124
|
+
/** Hash seed; the other field `union` requires to match. */
|
|
125
|
+
get seed() {
|
|
126
|
+
return this.#seed;
|
|
127
|
+
}
|
|
74
128
|
/** Number of bits currently set across all lanes. */
|
|
75
129
|
get length() {
|
|
76
130
|
let bits = 0;
|
|
@@ -100,19 +154,15 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
100
154
|
static fromBytes(bytes) {
|
|
101
155
|
const { type, flags, body } = readHeader(bytes);
|
|
102
156
|
if (type !== TYPE) throw new SerializationError(`expected AMQF type ${String(TYPE)}, got ${String(type)}`);
|
|
103
|
-
if ((flags & 15) !==
|
|
157
|
+
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
104
158
|
assertMinBodyLength(body.length, 12, "blocked");
|
|
105
159
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
106
160
|
const numBlocks = dv.getUint32(0, true);
|
|
107
161
|
const seed = dv.getUint32(4, true);
|
|
108
162
|
const n = dv.getUint32(8, true);
|
|
163
|
+
if (numBlocks === 0 || n === 0) throw new SerializationError(`blocked frame declares numBlocks=${String(numBlocks)}, n=${String(n)}; both must be positive`);
|
|
109
164
|
assertBodyLength(body.length, 12 + numBlocks * 32, "blocked");
|
|
110
|
-
const f =
|
|
111
|
-
bitsPerKey: 256,
|
|
112
|
-
capacity: numBlocks,
|
|
113
|
-
seed
|
|
114
|
-
});
|
|
115
|
-
f.#n = n;
|
|
165
|
+
const f = BlockedBloomFilter.#fromNumBlocks(numBlocks, seed, n);
|
|
116
166
|
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
117
167
|
return f;
|
|
118
168
|
}
|
|
@@ -123,17 +173,44 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
123
173
|
*/
|
|
124
174
|
toBytes() {
|
|
125
175
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
dv.setUint32(0, this.#numBlocks, true);
|
|
129
|
-
dv.setUint32(4, this.#seed, true);
|
|
130
|
-
dv.setUint32(8, this.#n, true);
|
|
131
|
-
body.set(lanes, 12);
|
|
132
|
-
return writeHeader({
|
|
133
|
-
version: 2,
|
|
176
|
+
return writeFrame({
|
|
177
|
+
version: 3,
|
|
134
178
|
type: TYPE,
|
|
135
|
-
flags:
|
|
136
|
-
}, body)
|
|
179
|
+
flags: 0
|
|
180
|
+
}, 12 + lanes.length, (body, dv) => {
|
|
181
|
+
dv.setUint32(0, this.#numBlocks, true);
|
|
182
|
+
dv.setUint32(4, this.#seed, true);
|
|
183
|
+
dv.setUint32(8, this.#n, true);
|
|
184
|
+
body.set(lanes, 12);
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
189
|
+
* bytes, meaning identical parameters and set bits.
|
|
190
|
+
*
|
|
191
|
+
* @param other - The filter to compare against.
|
|
192
|
+
* @returns `true` if the two filters are byte-for-byte identical.
|
|
193
|
+
*/
|
|
194
|
+
equals(other) {
|
|
195
|
+
return bytesEqual(this.toBytes(), other.toBytes());
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
199
|
+
* {@link BlockedBloomFilter.toBytes}.
|
|
200
|
+
*
|
|
201
|
+
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
202
|
+
*/
|
|
203
|
+
toJSON() {
|
|
204
|
+
return toJSONEnvelope(this.toBytes());
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
208
|
+
*
|
|
209
|
+
* @param value - The JSON envelope.
|
|
210
|
+
* @returns The reconstructed filter.
|
|
211
|
+
*/
|
|
212
|
+
static fromJSON(value) {
|
|
213
|
+
return BlockedBloomFilter.fromBytes(fromJSONEnvelope(value));
|
|
137
214
|
}
|
|
138
215
|
/**
|
|
139
216
|
* Returns a new filter containing the union of this filter and `other`.
|
|
@@ -144,11 +221,7 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
144
221
|
*/
|
|
145
222
|
union(other) {
|
|
146
223
|
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
147
|
-
const r =
|
|
148
|
-
bitsPerKey: this.bitsPerKey,
|
|
149
|
-
capacity: this.#n,
|
|
150
|
-
seed: this.#seed
|
|
151
|
-
});
|
|
224
|
+
const r = BlockedBloomFilter.#fromNumBlocks(this.#numBlocks, this.#seed, this.#n);
|
|
152
225
|
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
153
226
|
return r;
|
|
154
227
|
}
|