distillate 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -19
- package/dist/blocked/index.cjs +11 -112
- package/dist/blocked/index.d.cts +2 -2
- package/dist/blocked/index.d.ts +2 -2
- package/dist/blocked/index.js +6 -113
- package/dist/bloom/index.cjs +8 -97
- package/dist/bloom/index.d.cts +2 -2
- package/dist/bloom/index.d.ts +2 -2
- package/dist/bloom/index.js +3 -98
- package/dist/fuse/index.cjs +10 -111
- package/dist/fuse/index.d.cts +2 -2
- package/dist/fuse/index.d.ts +2 -2
- package/dist/fuse/index.js +5 -112
- package/dist/index.cjs +1 -2
- package/dist/index.js +1 -2
- package/dist/{params-BrWuBC2A.js → params-CeajSrwv.js} +0 -7
- package/dist/{params--8CNXYWu.cjs → params-UTZbJ22c.cjs} +0 -7
- package/dist/{serialize-BnwtzcMw.js → serialize-BqIcsR2J.js} +3 -29
- package/dist/{serialize-BIIKUHH6.cjs → serialize-CXnRWItH.cjs} +26 -28
- package/dist/serialize-ChyWpB9F.d.cts +73 -0
- package/dist/serialize-ChyWpB9F.d.ts +73 -0
- package/package.json +2 -3
- package/dist/serialize-DRKh6QOr.d.cts +0 -15
- package/dist/serialize-DRKh6QOr.d.ts +0 -15
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://www.npmjs.com/package/distillate)
|
|
5
5
|
[](./LICENSE)
|
|
6
6
|
|
|
7
|
-
Probabilistic data structures for JavaScript: space-efficient, approximate answers with tunable error and zero false negatives. TypeScript-first, zero dependencies, and the right structure per workload. It opens with a family of membership filters
|
|
7
|
+
Probabilistic data structures for JavaScript: space-efficient, approximate answers with tunable error and zero false negatives. TypeScript-first, zero dependencies, and the right structure per workload. It opens with a family of membership filters and is built to grow into other sketches.
|
|
8
8
|
|
|
9
9
|
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0`, and more structures (Cuckoo, Scalable Bloom) are on the way. Pin a version if you depend on it.
|
|
10
10
|
|
|
@@ -12,10 +12,12 @@ Probabilistic data structures for JavaScript: space-efficient, approximate answe
|
|
|
12
12
|
|
|
13
13
|
An approximate-membership query (AMQ) filter answers "is this in the set?" with a tunable false-positive rate and **zero false negatives**, in a fraction of the space of storing the set itself.
|
|
14
14
|
|
|
15
|
-
- **Runs anywhere**: Node, Bun, Deno, browsers,
|
|
16
|
-
- **Correct**: no false negatives
|
|
17
|
-
- **Small**: per-structure subpath imports,
|
|
18
|
-
- **Portable**: a versioned
|
|
15
|
+
- **Runs anywhere**: Node, Bun, Deno, browsers, Cloudflare/Vercel edge.
|
|
16
|
+
- **Correct**: no false negatives, property-tested.
|
|
17
|
+
- **Small**: per-structure subpath imports, zero runtime dependencies.
|
|
18
|
+
- **Portable**: a versioned binary format (`toBytes` / `fromBytes`).
|
|
19
|
+
|
|
20
|
+
Why these promises, and what the incumbents get wrong: [overview.md](./docs/overview.md).
|
|
19
21
|
|
|
20
22
|
## Install
|
|
21
23
|
|
|
@@ -40,11 +42,11 @@ Every push runs a CI smoke matrix that imports the built package on Node 22/24,
|
|
|
40
42
|
|
|
41
43
|
Each structure ships as its own subpath, so you only bundle what you import.
|
|
42
44
|
|
|
43
|
-
| Import | Structure | Mutable? | Use for
|
|
44
|
-
| -------------------- | ------------- | -------- |
|
|
45
|
-
| `distillate/bloom` | Classic Bloom | yes | Familiar default, migration from `bloom-filters`
|
|
46
|
-
| `distillate/blocked` | Blocked Bloom | yes | Faster lookups and a lower FPR for
|
|
47
|
-
| `distillate/fuse` | Binary Fuse | no | Static set built once and queried a lot; least space
|
|
45
|
+
| Import | Structure | Mutable? | Use for |
|
|
46
|
+
| -------------------- | ------------- | -------- | -------------------------------------------------------- |
|
|
47
|
+
| `distillate/bloom` | Classic Bloom | yes | Familiar default, migration from `bloom-filters` |
|
|
48
|
+
| `distillate/blocked` | Blocked Bloom | yes | Faster lookups and a lower FPR for a small space premium |
|
|
49
|
+
| `distillate/fuse` | Binary Fuse | no | Static set built once and queried a lot; least space |
|
|
48
50
|
|
|
49
51
|
### Classic Bloom (`distillate/bloom`)
|
|
50
52
|
|
|
@@ -72,11 +74,11 @@ filter.add("alice");
|
|
|
72
74
|
filter.has("alice"); // true
|
|
73
75
|
```
|
|
74
76
|
|
|
75
|
-
Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`).
|
|
77
|
+
Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Reach for it when lookup throughput matters; prefer Classic when space is tight. Cache-line rationale, speed ratios, and the space penalty: [Blocked Bloom guide](https://distillate.akxp.net/guides/blocked/).
|
|
76
78
|
|
|
77
79
|
### Binary Fuse (`distillate/fuse`)
|
|
78
80
|
|
|
79
|
-
A **static** filter: built once from the full key set, then immutable. The most space-efficient option
|
|
81
|
+
A **static** filter: built once from the full key set, then immutable. The most space-efficient option in the lineup. Bits-per-key, FPR, and query throughput: [Binary Fuse guide](https://distillate.akxp.net/guides/fuse/).
|
|
80
82
|
|
|
81
83
|
```ts
|
|
82
84
|
import { BinaryFuse8, BinaryFuse16 } from "distillate/fuse";
|
|
@@ -94,14 +96,14 @@ Also: `toBytes` / `fromBytes`. No `add` / `delete`; rebuild `from` the new set t
|
|
|
94
96
|
|
|
95
97
|
## Performance
|
|
96
98
|
|
|
97
|
-
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (
|
|
99
|
+
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (cross-library harness, node v24.14.1, Apple M5):
|
|
98
100
|
|
|
99
101
|
| Classic Bloom | bits/key | measured FPR | `has` throughput |
|
|
100
102
|
| -------------- | -------- | ------------ | ---------------- |
|
|
101
|
-
| **distillate** | 9.59 | 1.
|
|
103
|
+
| **distillate** | 9.59 | 1.01% | ~21.8 M ops/s |
|
|
102
104
|
| bloom-filters | 9.59 | 0.99% | ~0.29 M ops/s |
|
|
103
105
|
|
|
104
|
-
Same space, same accuracy, **~
|
|
106
|
+
Same space, same accuracy, **~75x the lookup throughput** of [`bloom-filters`](https://www.npmjs.com/package/bloom-filters) (the package distillate replaces), while hashing UTF-8 bytes with MurmurHash3 so filters stay portable and cross-language readable.
|
|
105
107
|
|
|
106
108
|
These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [`apps/bench`](https://github.com/akshay-xp/distillate/tree/main/apps/bench) workspace: [RESULTS.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/RESULTS.md), [METHODOLOGY.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/METHODOLOGY.md).
|
|
107
109
|
|
|
@@ -110,10 +112,11 @@ These are a point-in-time snapshot on one machine. The full report (blocked/fuse
|
|
|
110
112
|
Design notes, the structure decision matrix, hashing, and the binary format live in [`docs/`](./docs):
|
|
111
113
|
|
|
112
114
|
- [overview](./docs/overview.md): what and why
|
|
113
|
-
- [
|
|
114
|
-
- [
|
|
115
|
-
- [versioning](
|
|
116
|
-
- [
|
|
115
|
+
- [choosing a structure](https://distillate.akxp.net/guides/choosing-a-structure/): decision matrix and the full lineup
|
|
116
|
+
- [serialization](https://distillate.akxp.net/reference/serialization/): the binary format spec
|
|
117
|
+
- [versioning](https://distillate.akxp.net/reference/versioning/): SemVer policy and supported-runtime baseline
|
|
118
|
+
- [architecture](./docs/architecture.md), [hashing](./docs/hashing.md): contributor notes, on GitHub
|
|
119
|
+
- [API reference](https://distillate.akxp.net/api/readme/): generated from TSDoc at site build time (per entry point)
|
|
117
120
|
|
|
118
121
|
## License
|
|
119
122
|
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,20 +1,13 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_serialize = require("../serialize-
|
|
3
|
-
const require_params = require("../params
|
|
2
|
+
const require_serialize = require("../serialize-CXnRWItH.cjs");
|
|
3
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
4
4
|
//#region src/blocked/blocked.ts
|
|
5
5
|
const TYPE = 2;
|
|
6
6
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
7
|
-
const scratch2 =
|
|
7
|
+
const scratch2 = new Uint32Array(2);
|
|
8
8
|
const BLOCK_BITS = 256;
|
|
9
9
|
const LANE_BITS = 32;
|
|
10
10
|
const LANES = 8;
|
|
11
|
-
/**
|
|
12
|
-
* Modeled false-positive rate of a split-block filter at `bitsPerKey`. A block
|
|
13
|
-
* holding `j` keys has FPR `(1 - (1 - 1/32)^j)^8` (8 lanes of 32 bits, one probe
|
|
14
|
-
* each); the filter's rate averages that over the Poisson block load
|
|
15
|
-
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
16
|
-
* is not linear in `log10(1/epsilon)`.
|
|
17
|
-
*/
|
|
18
11
|
function blockedFprAt(bitsPerKey) {
|
|
19
12
|
const lambda = BLOCK_BITS / bitsPerKey;
|
|
20
13
|
let fpr = 0;
|
|
@@ -27,46 +20,20 @@ function blockedFprAt(bitsPerKey) {
|
|
|
27
20
|
return fpr;
|
|
28
21
|
}
|
|
29
22
|
const MAX_BITS_PER_KEY = 128;
|
|
30
|
-
/**
|
|
31
|
-
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
32
|
-
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
33
|
-
* cannot reach the target, so callers get a typed rejection instead of a
|
|
34
|
-
* silently under-provisioned filter.
|
|
35
|
-
*/
|
|
36
23
|
function blockedBitsPerKey(epsilon) {
|
|
37
24
|
for (let bpk = 1; bpk <= MAX_BITS_PER_KEY; bpk++) if (blockedFprAt(bpk) <= epsilon) return bpk;
|
|
38
25
|
throw new require_params.ParamError(`epsilon ${String(epsilon)} is below the blocked-filter floor; use a classic or fuse filter`);
|
|
39
26
|
}
|
|
40
|
-
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
41
27
|
var BlockedBloomParamMismatchError = class extends Error {
|
|
42
|
-
/** Discriminates this error from other `Error`s. */
|
|
43
28
|
name = "BlockedBloomParamMismatchError";
|
|
44
29
|
};
|
|
45
|
-
/**
|
|
46
|
-
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
47
|
-
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
48
|
-
*
|
|
49
|
-
* @example
|
|
50
|
-
* ```ts
|
|
51
|
-
* const filter = BlockedBloomFilter.create(100_000, 0.01);
|
|
52
|
-
* filter.add("alice");
|
|
53
|
-
* filter.has("alice"); // true
|
|
54
|
-
* ```
|
|
55
|
-
*/
|
|
56
30
|
var BlockedBloomFilter = class BlockedBloomFilter {
|
|
57
31
|
#lanes;
|
|
58
32
|
#numBlocks;
|
|
59
33
|
#seed;
|
|
60
34
|
#n;
|
|
61
|
-
#words =
|
|
62
|
-
#bits =
|
|
63
|
-
/**
|
|
64
|
-
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
65
|
-
*
|
|
66
|
-
* @param n - Expected number of keys.
|
|
67
|
-
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
68
|
-
* @returns A new, empty filter.
|
|
69
|
-
*/
|
|
35
|
+
#words = new Uint32Array(8);
|
|
36
|
+
#bits = new Uint32Array(8);
|
|
70
37
|
static create(n, epsilon) {
|
|
71
38
|
require_params.assertPositiveInt(n, "n");
|
|
72
39
|
require_params.assertProbability(epsilon, "epsilon");
|
|
@@ -75,26 +42,12 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
75
42
|
capacity: n
|
|
76
43
|
});
|
|
77
44
|
}
|
|
78
|
-
/**
|
|
79
|
-
* Builds a filter from `keys`, sized for their count at the target
|
|
80
|
-
* false-positive rate. The ergonomic entry point when the key set is already
|
|
81
|
-
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
82
|
-
* ahead.
|
|
83
|
-
*
|
|
84
|
-
* @param keys - The keys to insert.
|
|
85
|
-
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
86
|
-
* @returns A new filter containing every key.
|
|
87
|
-
*/
|
|
88
45
|
static from(keys, epsilon) {
|
|
89
46
|
const arr = [...keys];
|
|
90
47
|
const f = BlockedBloomFilter.create(Math.max(1, arr.length), epsilon);
|
|
91
48
|
for (const k of arr) f.add(k);
|
|
92
49
|
return f;
|
|
93
50
|
}
|
|
94
|
-
/**
|
|
95
|
-
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
96
|
-
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
97
|
-
*/
|
|
98
51
|
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
99
52
|
require_params.assertPositiveFinite(bitsPerKey, "bitsPerKey");
|
|
100
53
|
require_params.assertPositiveInt(capacity, "capacity");
|
|
@@ -114,19 +67,15 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
114
67
|
f.#n = n;
|
|
115
68
|
return f;
|
|
116
69
|
}
|
|
117
|
-
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
118
70
|
get bitsPerKey() {
|
|
119
71
|
return this.#numBlocks * 256 / this.#n;
|
|
120
72
|
}
|
|
121
|
-
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
122
73
|
get numBlocks() {
|
|
123
74
|
return this.#numBlocks;
|
|
124
75
|
}
|
|
125
|
-
/** Hash seed; the other field `union` requires to match. */
|
|
126
76
|
get seed() {
|
|
127
77
|
return this.#seed;
|
|
128
78
|
}
|
|
129
|
-
/** Number of bits currently set across all lanes. */
|
|
130
79
|
get length() {
|
|
131
80
|
let bits = 0;
|
|
132
81
|
for (let w of this.#lanes) while (w) {
|
|
@@ -135,23 +84,9 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
135
84
|
}
|
|
136
85
|
return bits;
|
|
137
86
|
}
|
|
138
|
-
/**
|
|
139
|
-
* Estimates the current false-positive rate from the actual fill,
|
|
140
|
-
* `(length / totalBits) ** 8`. A split-block query checks exactly 8 lane-bits,
|
|
141
|
-
* so the exponent is 8 rather than a classic probe count `k`. This reflects
|
|
142
|
-
* how full the filter is right now, not the design target.
|
|
143
|
-
*
|
|
144
|
-
* @returns The estimated false-positive rate, `0` for an empty filter.
|
|
145
|
-
*/
|
|
146
87
|
rate() {
|
|
147
88
|
return (this.length / (this.#numBlocks * 256)) ** 8;
|
|
148
89
|
}
|
|
149
|
-
/**
|
|
150
|
-
* Restores a filter from its {@link BlockedBloomFilter.toBytes} serialization.
|
|
151
|
-
*
|
|
152
|
-
* @param bytes - The serialized filter.
|
|
153
|
-
* @returns The reconstructed filter.
|
|
154
|
-
*/
|
|
155
90
|
static fromBytes(bytes) {
|
|
156
91
|
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
157
92
|
if (type !== TYPE) throw new require_serialize.SerializationError(`expected AMQF type ${String(TYPE)}, got ${String(type)}`);
|
|
@@ -167,11 +102,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
167
102
|
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
168
103
|
return f;
|
|
169
104
|
}
|
|
170
|
-
/**
|
|
171
|
-
* Serializes the filter to a portable little-endian byte layout.
|
|
172
|
-
*
|
|
173
|
-
* @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
|
|
174
|
-
*/
|
|
175
105
|
toBytes() {
|
|
176
106
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
177
107
|
return require_serialize.writeFrame({
|
|
@@ -185,52 +115,21 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
185
115
|
body.set(lanes, 12);
|
|
186
116
|
});
|
|
187
117
|
}
|
|
188
|
-
/**
|
|
189
|
-
* Tests structural equality: `true` when `other` serializes to identical
|
|
190
|
-
* bytes, meaning identical parameters and set bits.
|
|
191
|
-
*
|
|
192
|
-
* @param other - The filter to compare against.
|
|
193
|
-
* @returns `true` if the two filters are byte-for-byte identical.
|
|
194
|
-
*/
|
|
195
118
|
equals(other) {
|
|
196
119
|
return require_serialize.bytesEqual(this.toBytes(), other.toBytes());
|
|
197
120
|
}
|
|
198
|
-
/**
|
|
199
|
-
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
200
|
-
* {@link BlockedBloomFilter.toBytes}.
|
|
201
|
-
*
|
|
202
|
-
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
203
|
-
*/
|
|
204
121
|
toJSON() {
|
|
205
122
|
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
206
123
|
}
|
|
207
|
-
/**
|
|
208
|
-
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
209
|
-
*
|
|
210
|
-
* @param value - The JSON envelope.
|
|
211
|
-
* @returns The reconstructed filter.
|
|
212
|
-
*/
|
|
213
124
|
static fromJSON(value) {
|
|
214
125
|
return BlockedBloomFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
215
126
|
}
|
|
216
|
-
/**
|
|
217
|
-
* Returns a new filter containing the union of this filter and `other`.
|
|
218
|
-
*
|
|
219
|
-
* @param other - A filter built with identical parameters.
|
|
220
|
-
* @returns A new filter reporting membership for keys in either input.
|
|
221
|
-
* @throws {@link BlockedBloomParamMismatchError} if the parameters differ.
|
|
222
|
-
*/
|
|
223
127
|
union(other) {
|
|
224
128
|
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
225
129
|
const r = BlockedBloomFilter.#fromNumBlocks(this.#numBlocks, this.#seed, this.#n);
|
|
226
130
|
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
227
131
|
return r;
|
|
228
132
|
}
|
|
229
|
-
/**
|
|
230
|
-
* Adds a key to the set.
|
|
231
|
-
*
|
|
232
|
-
* @param key - The key to insert, as a string or bytes.
|
|
233
|
-
*/
|
|
234
133
|
add(key) {
|
|
235
134
|
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
236
135
|
for (let i = 0; i < 8; i++) {
|
|
@@ -238,12 +137,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
238
137
|
this.#lanes[w] = (this.#lanes[w] ?? 0) | (this.#bits[i] ?? 0);
|
|
239
138
|
}
|
|
240
139
|
}
|
|
241
|
-
/**
|
|
242
|
-
* Tests whether a key is in the set.
|
|
243
|
-
*
|
|
244
|
-
* @param key - The key to test.
|
|
245
|
-
* @returns `true` if present (possibly a false positive); `false` guarantees absence.
|
|
246
|
-
*/
|
|
247
140
|
has(key) {
|
|
248
141
|
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
249
142
|
for (let i = 0; i < 8; i++) {
|
|
@@ -264,8 +157,14 @@ function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
|
264
157
|
}
|
|
265
158
|
}
|
|
266
159
|
//#endregion
|
|
160
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
267
161
|
exports.BlockedBloomFilter = BlockedBloomFilter;
|
|
268
162
|
exports.BlockedBloomParamMismatchError = BlockedBloomParamMismatchError;
|
|
163
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
269
164
|
exports.ParamError = require_params.ParamError;
|
|
165
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
166
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
167
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
168
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
270
169
|
exports.blockedBitsPerKey = blockedBitsPerKey;
|
|
271
170
|
exports.blockedFprAt = blockedFprAt;
|
package/dist/blocked/index.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { a as TruncatedError, c as BytesLike, i as SerializationError, n as ChecksumError, o as UnknownHashVariantError, r as FilterJSON, s as UnknownVersionError, t as BadMagicError } from "../serialize-ChyWpB9F.cjs";
|
|
2
2
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
3
3
|
//#region src/blocked/blocked.d.ts
|
|
4
4
|
/**
|
|
@@ -142,4 +142,4 @@ declare class BlockedBloomFilter {
|
|
|
142
142
|
has(key: BytesLike): boolean;
|
|
143
143
|
}
|
|
144
144
|
//#endregion
|
|
145
|
-
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError, blockedBitsPerKey, blockedFprAt };
|
|
145
|
+
export { BadMagicError, BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ChecksumError, type FilterJSON, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, blockedBitsPerKey, blockedFprAt };
|
package/dist/blocked/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { a as TruncatedError, c as BytesLike, i as SerializationError, n as ChecksumError, o as UnknownHashVariantError, r as FilterJSON, s as UnknownVersionError, t as BadMagicError } from "../serialize-ChyWpB9F.js";
|
|
2
2
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
3
3
|
//#region src/blocked/blocked.d.ts
|
|
4
4
|
/**
|
|
@@ -142,4 +142,4 @@ declare class BlockedBloomFilter {
|
|
|
142
142
|
has(key: BytesLike): boolean;
|
|
143
143
|
}
|
|
144
144
|
//#endregion
|
|
145
|
-
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, type FilterJSON, ParamError, blockedBitsPerKey, blockedFprAt };
|
|
145
|
+
export { BadMagicError, BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams, ChecksumError, type FilterJSON, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, blockedBitsPerKey, blockedFprAt };
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,19 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-
|
|
1
|
+
import { a as UnknownHashVariantError, c as assertMinBodyLength, d as readHeader, f as toJSONEnvelope, i as TruncatedError, l as bytesEqual, n as ChecksumError, o as UnknownVersionError, p as writeFrame, r as SerializationError, s as assertBodyLength, t as BadMagicError, u as fromJSONEnvelope, v as hash32x2Into, x as reduce } from "../serialize-BqIcsR2J.js";
|
|
2
|
+
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
3
3
|
//#region src/blocked/blocked.ts
|
|
4
4
|
const TYPE = 2;
|
|
5
5
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
6
|
-
const scratch2 =
|
|
6
|
+
const scratch2 = new Uint32Array(2);
|
|
7
7
|
const BLOCK_BITS = 256;
|
|
8
8
|
const LANE_BITS = 32;
|
|
9
9
|
const LANES = 8;
|
|
10
|
-
/**
|
|
11
|
-
* Modeled false-positive rate of a split-block filter at `bitsPerKey`. A block
|
|
12
|
-
* holding `j` keys has FPR `(1 - (1 - 1/32)^j)^8` (8 lanes of 32 bits, one probe
|
|
13
|
-
* each); the filter's rate averages that over the Poisson block load
|
|
14
|
-
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
15
|
-
* is not linear in `log10(1/epsilon)`.
|
|
16
|
-
*/
|
|
17
10
|
function blockedFprAt(bitsPerKey) {
|
|
18
11
|
const lambda = BLOCK_BITS / bitsPerKey;
|
|
19
12
|
let fpr = 0;
|
|
@@ -26,46 +19,20 @@ function blockedFprAt(bitsPerKey) {
|
|
|
26
19
|
return fpr;
|
|
27
20
|
}
|
|
28
21
|
const MAX_BITS_PER_KEY = 128;
|
|
29
|
-
/**
|
|
30
|
-
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
31
|
-
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
32
|
-
* cannot reach the target, so callers get a typed rejection instead of a
|
|
33
|
-
* silently under-provisioned filter.
|
|
34
|
-
*/
|
|
35
22
|
function blockedBitsPerKey(epsilon) {
|
|
36
23
|
for (let bpk = 1; bpk <= MAX_BITS_PER_KEY; bpk++) if (blockedFprAt(bpk) <= epsilon) return bpk;
|
|
37
24
|
throw new ParamError(`epsilon ${String(epsilon)} is below the blocked-filter floor; use a classic or fuse filter`);
|
|
38
25
|
}
|
|
39
|
-
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
40
26
|
var BlockedBloomParamMismatchError = class extends Error {
|
|
41
|
-
/** Discriminates this error from other `Error`s. */
|
|
42
27
|
name = "BlockedBloomParamMismatchError";
|
|
43
28
|
};
|
|
44
|
-
/**
|
|
45
|
-
* A blocked (split-block) Bloom filter: confines every lookup to a single cache
|
|
46
|
-
* line, trading ~15% more space for higher lookup throughput and a lower FPR.
|
|
47
|
-
*
|
|
48
|
-
* @example
|
|
49
|
-
* ```ts
|
|
50
|
-
* const filter = BlockedBloomFilter.create(100_000, 0.01);
|
|
51
|
-
* filter.add("alice");
|
|
52
|
-
* filter.has("alice"); // true
|
|
53
|
-
* ```
|
|
54
|
-
*/
|
|
55
29
|
var BlockedBloomFilter = class BlockedBloomFilter {
|
|
56
30
|
#lanes;
|
|
57
31
|
#numBlocks;
|
|
58
32
|
#seed;
|
|
59
33
|
#n;
|
|
60
|
-
#words =
|
|
61
|
-
#bits =
|
|
62
|
-
/**
|
|
63
|
-
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
64
|
-
*
|
|
65
|
-
* @param n - Expected number of keys.
|
|
66
|
-
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
67
|
-
* @returns A new, empty filter.
|
|
68
|
-
*/
|
|
34
|
+
#words = new Uint32Array(8);
|
|
35
|
+
#bits = new Uint32Array(8);
|
|
69
36
|
static create(n, epsilon) {
|
|
70
37
|
assertPositiveInt(n, "n");
|
|
71
38
|
assertProbability(epsilon, "epsilon");
|
|
@@ -74,26 +41,12 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
74
41
|
capacity: n
|
|
75
42
|
});
|
|
76
43
|
}
|
|
77
|
-
/**
|
|
78
|
-
* Builds a filter from `keys`, sized for their count at the target
|
|
79
|
-
* false-positive rate. The ergonomic entry point when the key set is already
|
|
80
|
-
* in hand; use {@link BlockedBloomFilter.create} to size for a count known
|
|
81
|
-
* ahead.
|
|
82
|
-
*
|
|
83
|
-
* @param keys - The keys to insert.
|
|
84
|
-
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
85
|
-
* @returns A new filter containing every key.
|
|
86
|
-
*/
|
|
87
44
|
static from(keys, epsilon) {
|
|
88
45
|
const arr = [...keys];
|
|
89
46
|
const f = BlockedBloomFilter.create(Math.max(1, arr.length), epsilon);
|
|
90
47
|
for (const k of arr) f.add(k);
|
|
91
48
|
return f;
|
|
92
49
|
}
|
|
93
|
-
/**
|
|
94
|
-
* Constructs a filter from low-level {@link BlockedBloomParams}. Prefer
|
|
95
|
-
* {@link BlockedBloomFilter.create} unless restoring a specific configuration.
|
|
96
|
-
*/
|
|
97
50
|
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
98
51
|
assertPositiveFinite(bitsPerKey, "bitsPerKey");
|
|
99
52
|
assertPositiveInt(capacity, "capacity");
|
|
@@ -113,19 +66,15 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
113
66
|
f.#n = n;
|
|
114
67
|
return f;
|
|
115
68
|
}
|
|
116
|
-
/** Actual bits allocated per key (`total bits / capacity`). */
|
|
117
69
|
get bitsPerKey() {
|
|
118
70
|
return this.#numBlocks * 256 / this.#n;
|
|
119
71
|
}
|
|
120
|
-
/** Number of 256-bit blocks; one of the two fields `union` requires to match. */
|
|
121
72
|
get numBlocks() {
|
|
122
73
|
return this.#numBlocks;
|
|
123
74
|
}
|
|
124
|
-
/** Hash seed; the other field `union` requires to match. */
|
|
125
75
|
get seed() {
|
|
126
76
|
return this.#seed;
|
|
127
77
|
}
|
|
128
|
-
/** Number of bits currently set across all lanes. */
|
|
129
78
|
get length() {
|
|
130
79
|
let bits = 0;
|
|
131
80
|
for (let w of this.#lanes) while (w) {
|
|
@@ -134,23 +83,9 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
134
83
|
}
|
|
135
84
|
return bits;
|
|
136
85
|
}
|
|
137
|
-
/**
|
|
138
|
-
* Estimates the current false-positive rate from the actual fill,
|
|
139
|
-
* `(length / totalBits) ** 8`. A split-block query checks exactly 8 lane-bits,
|
|
140
|
-
* so the exponent is 8 rather than a classic probe count `k`. This reflects
|
|
141
|
-
* how full the filter is right now, not the design target.
|
|
142
|
-
*
|
|
143
|
-
* @returns The estimated false-positive rate, `0` for an empty filter.
|
|
144
|
-
*/
|
|
145
86
|
rate() {
|
|
146
87
|
return (this.length / (this.#numBlocks * 256)) ** 8;
|
|
147
88
|
}
|
|
148
|
-
/**
|
|
149
|
-
* Restores a filter from its {@link BlockedBloomFilter.toBytes} serialization.
|
|
150
|
-
*
|
|
151
|
-
* @param bytes - The serialized filter.
|
|
152
|
-
* @returns The reconstructed filter.
|
|
153
|
-
*/
|
|
154
89
|
static fromBytes(bytes) {
|
|
155
90
|
const { type, flags, body } = readHeader(bytes);
|
|
156
91
|
if (type !== TYPE) throw new SerializationError(`expected AMQF type ${String(TYPE)}, got ${String(type)}`);
|
|
@@ -166,11 +101,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
166
101
|
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
167
102
|
return f;
|
|
168
103
|
}
|
|
169
|
-
/**
|
|
170
|
-
* Serializes the filter to a portable little-endian byte layout.
|
|
171
|
-
*
|
|
172
|
-
* @returns The serialized filter, readable by {@link BlockedBloomFilter.fromBytes}.
|
|
173
|
-
*/
|
|
174
104
|
toBytes() {
|
|
175
105
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
176
106
|
return writeFrame({
|
|
@@ -184,52 +114,21 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
184
114
|
body.set(lanes, 12);
|
|
185
115
|
});
|
|
186
116
|
}
|
|
187
|
-
/**
|
|
188
|
-
* Tests structural equality: `true` when `other` serializes to identical
|
|
189
|
-
* bytes, meaning identical parameters and set bits.
|
|
190
|
-
*
|
|
191
|
-
* @param other - The filter to compare against.
|
|
192
|
-
* @returns `true` if the two filters are byte-for-byte identical.
|
|
193
|
-
*/
|
|
194
117
|
equals(other) {
|
|
195
118
|
return bytesEqual(this.toBytes(), other.toBytes());
|
|
196
119
|
}
|
|
197
|
-
/**
|
|
198
|
-
* Serializes the filter to a JSON-friendly envelope wrapping the base64 of
|
|
199
|
-
* {@link BlockedBloomFilter.toBytes}.
|
|
200
|
-
*
|
|
201
|
-
* @returns The envelope, readable by {@link BlockedBloomFilter.fromJSON}.
|
|
202
|
-
*/
|
|
203
120
|
toJSON() {
|
|
204
121
|
return toJSONEnvelope(this.toBytes());
|
|
205
122
|
}
|
|
206
|
-
/**
|
|
207
|
-
* Restores a filter from its {@link BlockedBloomFilter.toJSON} envelope.
|
|
208
|
-
*
|
|
209
|
-
* @param value - The JSON envelope.
|
|
210
|
-
* @returns The reconstructed filter.
|
|
211
|
-
*/
|
|
212
123
|
static fromJSON(value) {
|
|
213
124
|
return BlockedBloomFilter.fromBytes(fromJSONEnvelope(value));
|
|
214
125
|
}
|
|
215
|
-
/**
|
|
216
|
-
* Returns a new filter containing the union of this filter and `other`.
|
|
217
|
-
*
|
|
218
|
-
* @param other - A filter built with identical parameters.
|
|
219
|
-
* @returns A new filter reporting membership for keys in either input.
|
|
220
|
-
* @throws {@link BlockedBloomParamMismatchError} if the parameters differ.
|
|
221
|
-
*/
|
|
222
126
|
union(other) {
|
|
223
127
|
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
224
128
|
const r = BlockedBloomFilter.#fromNumBlocks(this.#numBlocks, this.#seed, this.#n);
|
|
225
129
|
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
226
130
|
return r;
|
|
227
131
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Adds a key to the set.
|
|
230
|
-
*
|
|
231
|
-
* @param key - The key to insert, as a string or bytes.
|
|
232
|
-
*/
|
|
233
132
|
add(key) {
|
|
234
133
|
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
235
134
|
for (let i = 0; i < 8; i++) {
|
|
@@ -237,12 +136,6 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
237
136
|
this.#lanes[w] = (this.#lanes[w] ?? 0) | (this.#bits[i] ?? 0);
|
|
238
137
|
}
|
|
239
138
|
}
|
|
240
|
-
/**
|
|
241
|
-
* Tests whether a key is in the set.
|
|
242
|
-
*
|
|
243
|
-
* @param key - The key to test.
|
|
244
|
-
* @returns `true` if present (possibly a false positive); `false` guarantees absence.
|
|
245
|
-
*/
|
|
246
139
|
has(key) {
|
|
247
140
|
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
248
141
|
for (let i = 0; i < 8; i++) {
|
|
@@ -263,4 +156,4 @@ function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
|
263
156
|
}
|
|
264
157
|
}
|
|
265
158
|
//#endregion
|
|
266
|
-
export { BlockedBloomFilter, BlockedBloomParamMismatchError, ParamError, blockedBitsPerKey, blockedFprAt };
|
|
159
|
+
export { BadMagicError, BlockedBloomFilter, BlockedBloomParamMismatchError, ChecksumError, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, blockedBitsPerKey, blockedFprAt };
|