distillate 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -6
- package/dist/blocked/index.cjs +15 -10
- package/dist/blocked/index.d.cts +7 -6
- package/dist/blocked/index.d.ts +7 -6
- package/dist/blocked/index.js +13 -9
- package/dist/bloom/index.cjs +35 -34
- package/dist/bloom/index.d.cts +22 -17
- package/dist/bloom/index.d.ts +22 -17
- package/dist/bloom/index.js +32 -32
- package/dist/bytes-DCuYtUVS.d.cts +4 -0
- package/dist/bytes-DCuYtUVS.d.ts +4 -0
- package/dist/frame/index.cjs +9 -0
- package/dist/frame/index.d.cts +2 -0
- package/dist/frame/index.d.ts +2 -0
- package/dist/frame/index.js +2 -0
- package/dist/fuse/index.cjs +27 -23
- package/dist/fuse/index.d.cts +7 -6
- package/dist/fuse/index.d.ts +7 -6
- package/dist/fuse/index.js +15 -12
- package/dist/hasher-D8LCBTOM.cjs +276 -0
- package/dist/hasher-DVWXS_vJ.js +229 -0
- package/dist/hll/index.cjs +366 -0
- package/dist/hll/index.d.cts +120 -0
- package/dist/hll/index.d.ts +120 -0
- package/dist/hll/index.js +356 -0
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +2 -3
- package/dist/index.d.ts +2 -3
- package/dist/index.js +1 -1
- package/dist/serialize--xR6vGSW.d.cts +112 -0
- package/dist/serialize--xR6vGSW.d.ts +112 -0
- package/dist/serialize-CHikZ4GH.cjs +296 -0
- package/dist/serialize-CMSYFym-.js +201 -0
- package/dist/sizing-BLzZk2zr.cjs +28 -0
- package/dist/sizing-BtUrtvF_.js +17 -0
- package/dist/sizing-wYOW27eY.d.cts +22 -0
- package/dist/sizing-wYOW27eY.d.ts +22 -0
- package/package.json +26 -10
- package/dist/serialize-BqIcsR2J.js +0 -388
- package/dist/serialize-CXnRWItH.cjs +0 -513
- package/dist/serialize-ChyWpB9F.d.cts +0 -73
- package/dist/serialize-ChyWpB9F.d.ts +0 -73
package/README.md
CHANGED
|
@@ -42,11 +42,16 @@ Every push runs a CI smoke matrix that imports the built package on Node 22/24,
|
|
|
42
42
|
|
|
43
43
|
Each structure ships as its own subpath, so you only bundle what you import.
|
|
44
44
|
|
|
45
|
-
| Import | Structure |
|
|
46
|
-
| -------------------- | ------------- |
|
|
47
|
-
| `distillate/bloom` | Classic Bloom |
|
|
48
|
-
| `distillate/blocked` | Blocked Bloom |
|
|
49
|
-
| `distillate/fuse` | Binary Fuse |
|
|
45
|
+
| Import | Structure | Answers | Use for |
|
|
46
|
+
| -------------------- | ------------- | ------------------ | -------------------------------------------------------- |
|
|
47
|
+
| `distillate/bloom` | Classic Bloom | seen this key? | Familiar default, migration from `bloom-filters` |
|
|
48
|
+
| `distillate/blocked` | Blocked Bloom | seen this key? | Faster lookups and a lower FPR for a small space premium |
|
|
49
|
+
| `distillate/fuse` | Binary Fuse | seen this key? | Static set built once and queried a lot; least space |
|
|
50
|
+
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
51
|
+
|
|
52
|
+
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
53
|
+
key set. HyperLogLog is not a filter: it counts distinct keys and cannot report
|
|
54
|
+
whether it saw any particular one.
|
|
50
55
|
|
|
51
56
|
### Classic Bloom (`distillate/bloom`)
|
|
52
57
|
|
|
@@ -86,7 +91,7 @@ import { BinaryFuse8, BinaryFuse16 } from "distillate/fuse";
|
|
|
86
91
|
const filter = BinaryFuse8.from(["alice", "bob", "carol"]);
|
|
87
92
|
filter.has("alice"); // true
|
|
88
93
|
filter.size; // 3
|
|
89
|
-
filter.bitsPerKey; // 64
|
|
94
|
+
filter.bitsPerKey; // 64
|
|
90
95
|
|
|
91
96
|
// Lower false-positive rate, twice the space:
|
|
92
97
|
const precise = BinaryFuse16.from(["alice", "bob", "carol"]);
|
|
@@ -96,6 +101,29 @@ const precise = BinaryFuse16.from(["alice", "bob", "carol"]);
|
|
|
96
101
|
|
|
97
102
|
Also: `toBytes` / `fromBytes`. No `add` / `delete`; rebuild `from` the new set to change membership.
|
|
98
103
|
|
|
104
|
+
### HyperLogLog (`distillate/hll`)
|
|
105
|
+
|
|
106
|
+
A **sketch**, not a filter: it counts distinct keys in space fixed by precision rather than by the answer, and cannot report whether it saw any particular key. 12 KiB counts a thousand distinct keys or a billion, at about 0.8% relative error. Full API and sizing: [HyperLogLog guide](https://distillate.akxp.net/guides/hll/).
|
|
107
|
+
|
|
108
|
+
```ts
|
|
109
|
+
import { HyperLogLog } from "distillate/hll";
|
|
110
|
+
|
|
111
|
+
const sketch = HyperLogLog.create(0.01); // target relative error
|
|
112
|
+
sketch.add("alice");
|
|
113
|
+
sketch.add("bob");
|
|
114
|
+
sketch.add("alice"); // already counted
|
|
115
|
+
|
|
116
|
+
sketch.count(); // 2
|
|
117
|
+
|
|
118
|
+
// Merge per-shard sketches without double-counting the overlap:
|
|
119
|
+
const other = HyperLogLog.from(["bob", "carol"], 0.01);
|
|
120
|
+
sketch.union(other).count(); // 3
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Below a few thousand distinct keys the sketch counts rather than estimates, so small answers are exact. It switches to fixed-size registers on its own once that stops paying, with nothing to configure.
|
|
124
|
+
|
|
125
|
+
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `standardError`, `hllSizing(relativeError)`, and a low-level `new HyperLogLog({ p, seed })`.
|
|
126
|
+
|
|
99
127
|
## Performance
|
|
100
128
|
|
|
101
129
|
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (cross-library harness, node v24.14.1, Apple M5):
|
|
@@ -107,6 +135,25 @@ Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same
|
|
|
107
135
|
|
|
108
136
|
Same space, same accuracy, **~75x the lookup throughput** of [`bloom-filters`](https://www.npmjs.com/package/bloom-filters) (the package distillate replaces), while hashing UTF-8 bytes with MurmurHash3 so filters stay portable and cross-language readable.
|
|
109
137
|
|
|
138
|
+
Cardinality is a separate head-to-head, at a **matched register count** (`m = 2 ** p`, `p = 14`, the configuration Redis uses) over the same keys, swept from 1k to 10M distinct:
|
|
139
|
+
|
|
140
|
+
| p = 14, m = 16384 | distillate | bloom-filters |
|
|
141
|
+
| -------------------- | --------------- | ---------------- |
|
|
142
|
+
| rel. error, n=1k | 0.00% | 49.63% |
|
|
143
|
+
| rel. error, n=100k | 0.82% | 0.78% |
|
|
144
|
+
| rel. error, n=10M | 0.15% | 0.41% |
|
|
145
|
+
| serialized, n=10M | 12,314 B binary | 40,247 B JSON |
|
|
146
|
+
| build 10M keys | 0.63 s | 1,165 s (19 min) |
|
|
147
|
+
| sustained add, n=10M | ~16 M ops/s | ~9 k ops/s |
|
|
148
|
+
|
|
149
|
+
Both sketches carry the same theoretical error at this precision (`1.04 / sqrt(m)`, about 0.81%), and once `n` is well past `m` both stay inside it. The differences are elsewhere.
|
|
150
|
+
|
|
151
|
+
**Small counts.** `bloom-filters` has no working small-range correction, so below roughly `2.5 * m` it is off by about half: counting a few thousand distinct keys in a 16k-register sketch, it answers 504 for 1,000. distillate is exact there because it is still holding sparse entries, not because its estimator is better.
|
|
152
|
+
|
|
153
|
+
**Memory does not grow.** From 10k distinct keys to 10M, distillate stays at exactly 12,314 bytes. The incumbent's JSON grows from 32,904 to 40,247 bytes over the same range, because larger register values take more digits to spell out.
|
|
154
|
+
|
|
155
|
+
**Building the sketch.** `bloom-filters` holds a flat 9k adds/sec at every size, so its cost is purely linear: 10.6 seconds for 100k distinct keys, 109 seconds for 1M, and 1,165 seconds for 10M. distillate counts the same 10M in 0.63 seconds, and its rate climbs with `n` rather than staying flat, reaching about 16 M ops/s once the registers are dense.
|
|
156
|
+
|
|
110
157
|
These are a point-in-time snapshot on one machine. The full report (blocked/fuse, 1M capacity, the `bloomfilter` micro-package) and exactly how it is measured live in the [`apps/bench`](https://github.com/akshay-xp/distillate/tree/main/apps/bench) workspace: [RESULTS.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/RESULTS.md), [METHODOLOGY.md](https://github.com/akshay-xp/distillate/blob/main/apps/bench/METHODOLOGY.md).
|
|
111
158
|
|
|
112
159
|
## Docs
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
2
|
+
const require_hasher = require("../hasher-D8LCBTOM.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
3
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
4
5
|
//#region src/blocked/blocked.ts
|
|
5
6
|
const TYPE = 2;
|
|
7
|
+
const PARAMS_SIZE = 16;
|
|
8
|
+
const PARAMS_FIELDS_END = 12;
|
|
6
9
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
7
10
|
const scratch2 = new Uint32Array(2);
|
|
8
11
|
const BLOCK_BITS = 256;
|
|
@@ -89,30 +92,31 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
89
92
|
}
|
|
90
93
|
static fromBytes(bytes) {
|
|
91
94
|
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
92
|
-
if (type !== TYPE) throw new require_serialize.SerializationError(`expected
|
|
95
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
93
96
|
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
94
|
-
require_serialize.assertMinBodyLength(body.length,
|
|
97
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "blocked");
|
|
98
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "blocked");
|
|
95
99
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
96
100
|
const numBlocks = dv.getUint32(0, true);
|
|
97
101
|
const seed = dv.getUint32(4, true);
|
|
98
102
|
const n = dv.getUint32(8, true);
|
|
99
103
|
if (numBlocks === 0 || n === 0) throw new require_serialize.SerializationError(`blocked frame declares numBlocks=${String(numBlocks)}, n=${String(n)}; both must be positive`);
|
|
100
|
-
require_serialize.assertBodyLength(body.length,
|
|
104
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + numBlocks * 32, "blocked");
|
|
101
105
|
const f = BlockedBloomFilter.#fromNumBlocks(numBlocks, seed, n);
|
|
102
|
-
new Uint8Array(f.#lanes.buffer).set(body.subarray(
|
|
106
|
+
new Uint8Array(f.#lanes.buffer).set(body.subarray(PARAMS_SIZE));
|
|
103
107
|
return f;
|
|
104
108
|
}
|
|
105
109
|
toBytes() {
|
|
106
110
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
107
111
|
return require_serialize.writeFrame({
|
|
108
|
-
version:
|
|
112
|
+
version: 5,
|
|
109
113
|
type: TYPE,
|
|
110
114
|
flags: 0
|
|
111
|
-
},
|
|
115
|
+
}, PARAMS_SIZE, lanes.length, (body, dv) => {
|
|
112
116
|
dv.setUint32(0, this.#numBlocks, true);
|
|
113
117
|
dv.setUint32(4, this.#seed, true);
|
|
114
118
|
dv.setUint32(8, this.#n, true);
|
|
115
|
-
body.set(lanes,
|
|
119
|
+
body.set(lanes, PARAMS_SIZE);
|
|
116
120
|
});
|
|
117
121
|
}
|
|
118
122
|
equals(other) {
|
|
@@ -147,8 +151,8 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
147
151
|
}
|
|
148
152
|
};
|
|
149
153
|
function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
150
|
-
|
|
151
|
-
const block =
|
|
154
|
+
require_hasher.hash32x2Into(key, seed, scratch2);
|
|
155
|
+
const block = require_hasher.reduce(scratch2[0] ?? 0, numBlocks);
|
|
152
156
|
const x = scratch2[1] ?? 0;
|
|
153
157
|
const base = block * 8;
|
|
154
158
|
for (let i = 0; i < 8; i++) {
|
|
@@ -162,6 +166,7 @@ exports.BlockedBloomFilter = BlockedBloomFilter;
|
|
|
162
166
|
exports.BlockedBloomParamMismatchError = BlockedBloomParamMismatchError;
|
|
163
167
|
exports.ChecksumError = require_serialize.ChecksumError;
|
|
164
168
|
exports.ParamError = require_params.ParamError;
|
|
169
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
165
170
|
exports.SerializationError = require_serialize.SerializationError;
|
|
166
171
|
exports.TruncatedError = require_serialize.TruncatedError;
|
|
167
172
|
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
package/dist/blocked/index.d.cts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
2
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
3
4
|
//#region src/blocked/blocked.d.ts
|
|
4
5
|
/**
|
|
@@ -8,16 +9,16 @@ import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
|
8
9
|
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
9
10
|
* is not linear in `log10(1/epsilon)`.
|
|
10
11
|
*/
|
|
11
|
-
declare function blockedFprAt(bitsPerKey: number): number;
|
|
12
|
+
export declare function blockedFprAt(bitsPerKey: number): number;
|
|
12
13
|
/**
|
|
13
14
|
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
14
15
|
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
15
16
|
* cannot reach the target, so callers get a typed rejection instead of a
|
|
16
17
|
* silently under-provisioned filter.
|
|
17
18
|
*/
|
|
18
|
-
declare function blockedBitsPerKey(epsilon: number): number;
|
|
19
|
+
export declare function blockedBitsPerKey(epsilon: number): number;
|
|
19
20
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
20
|
-
declare class BlockedBloomParamMismatchError extends Error {
|
|
21
|
+
export declare class BlockedBloomParamMismatchError extends Error {
|
|
21
22
|
/** Discriminates this error from other `Error`s. */
|
|
22
23
|
override readonly name = "BlockedBloomParamMismatchError";
|
|
23
24
|
}
|
|
@@ -41,7 +42,7 @@ interface BlockedBloomParams {
|
|
|
41
42
|
* filter.has("alice"); // true
|
|
42
43
|
* ```
|
|
43
44
|
*/
|
|
44
|
-
declare class BlockedBloomFilter {
|
|
45
|
+
export declare class BlockedBloomFilter {
|
|
45
46
|
#private;
|
|
46
47
|
/**
|
|
47
48
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
@@ -142,4 +143,4 @@ declare class BlockedBloomFilter {
|
|
|
142
143
|
has(key: BytesLike): boolean;
|
|
143
144
|
}
|
|
144
145
|
//#endregion
|
|
145
|
-
export { BadMagicError,
|
|
146
|
+
export { BadMagicError, type BlockedBloomParams, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/blocked/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
2
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
3
4
|
//#region src/blocked/blocked.d.ts
|
|
4
5
|
/**
|
|
@@ -8,16 +9,16 @@ import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
|
8
9
|
* `lambda = 256 / bitsPerKey`. This clustering average is why the blocked curve
|
|
9
10
|
* is not linear in `log10(1/epsilon)`.
|
|
10
11
|
*/
|
|
11
|
-
declare function blockedFprAt(bitsPerKey: number): number;
|
|
12
|
+
export declare function blockedFprAt(bitsPerKey: number): number;
|
|
12
13
|
/**
|
|
13
14
|
* Minimal integer bits-per-key whose modeled split-block FPR is at or below
|
|
14
15
|
* `epsilon`. Throws {@link ParamError} when even the densest supported filter
|
|
15
16
|
* cannot reach the target, so callers get a typed rejection instead of a
|
|
16
17
|
* silently under-provisioned filter.
|
|
17
18
|
*/
|
|
18
|
-
declare function blockedBitsPerKey(epsilon: number): number;
|
|
19
|
+
export declare function blockedBitsPerKey(epsilon: number): number;
|
|
19
20
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
20
|
-
declare class BlockedBloomParamMismatchError extends Error {
|
|
21
|
+
export declare class BlockedBloomParamMismatchError extends Error {
|
|
21
22
|
/** Discriminates this error from other `Error`s. */
|
|
22
23
|
override readonly name = "BlockedBloomParamMismatchError";
|
|
23
24
|
}
|
|
@@ -41,7 +42,7 @@ interface BlockedBloomParams {
|
|
|
41
42
|
* filter.has("alice"); // true
|
|
42
43
|
* ```
|
|
43
44
|
*/
|
|
44
|
-
declare class BlockedBloomFilter {
|
|
45
|
+
export declare class BlockedBloomFilter {
|
|
45
46
|
#private;
|
|
46
47
|
/**
|
|
47
48
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
@@ -142,4 +143,4 @@ declare class BlockedBloomFilter {
|
|
|
142
143
|
has(key: BytesLike): boolean;
|
|
143
144
|
}
|
|
144
145
|
//#endregion
|
|
145
|
-
export { BadMagicError,
|
|
146
|
+
export { BadMagicError, type BlockedBloomParams, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
import { a as
|
|
1
|
+
import { a as hash32x2Into, c as reduce } from "../hasher-DVWXS_vJ.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
2
3
|
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
3
4
|
//#region src/blocked/blocked.ts
|
|
4
5
|
const TYPE = 2;
|
|
6
|
+
const PARAMS_SIZE = 16;
|
|
7
|
+
const PARAMS_FIELDS_END = 12;
|
|
5
8
|
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
6
9
|
const scratch2 = new Uint32Array(2);
|
|
7
10
|
const BLOCK_BITS = 256;
|
|
@@ -88,30 +91,31 @@ var BlockedBloomFilter = class BlockedBloomFilter {
|
|
|
88
91
|
}
|
|
89
92
|
static fromBytes(bytes) {
|
|
90
93
|
const { type, flags, body } = readHeader(bytes);
|
|
91
|
-
if (type !== TYPE) throw new SerializationError(`expected
|
|
94
|
+
if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
92
95
|
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
93
|
-
assertMinBodyLength(body.length,
|
|
96
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "blocked");
|
|
97
|
+
assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "blocked");
|
|
94
98
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
95
99
|
const numBlocks = dv.getUint32(0, true);
|
|
96
100
|
const seed = dv.getUint32(4, true);
|
|
97
101
|
const n = dv.getUint32(8, true);
|
|
98
102
|
if (numBlocks === 0 || n === 0) throw new SerializationError(`blocked frame declares numBlocks=${String(numBlocks)}, n=${String(n)}; both must be positive`);
|
|
99
|
-
assertBodyLength(body.length,
|
|
103
|
+
assertBodyLength(body.length, PARAMS_SIZE + numBlocks * 32, "blocked");
|
|
100
104
|
const f = BlockedBloomFilter.#fromNumBlocks(numBlocks, seed, n);
|
|
101
|
-
new Uint8Array(f.#lanes.buffer).set(body.subarray(
|
|
105
|
+
new Uint8Array(f.#lanes.buffer).set(body.subarray(PARAMS_SIZE));
|
|
102
106
|
return f;
|
|
103
107
|
}
|
|
104
108
|
toBytes() {
|
|
105
109
|
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
106
110
|
return writeFrame({
|
|
107
|
-
version:
|
|
111
|
+
version: 5,
|
|
108
112
|
type: TYPE,
|
|
109
113
|
flags: 0
|
|
110
|
-
},
|
|
114
|
+
}, PARAMS_SIZE, lanes.length, (body, dv) => {
|
|
111
115
|
dv.setUint32(0, this.#numBlocks, true);
|
|
112
116
|
dv.setUint32(4, this.#seed, true);
|
|
113
117
|
dv.setUint32(8, this.#n, true);
|
|
114
|
-
body.set(lanes,
|
|
118
|
+
body.set(lanes, PARAMS_SIZE);
|
|
115
119
|
});
|
|
116
120
|
}
|
|
117
121
|
equals(other) {
|
|
@@ -156,4 +160,4 @@ function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
|
156
160
|
}
|
|
157
161
|
}
|
|
158
162
|
//#endregion
|
|
159
|
-
export { BadMagicError, BlockedBloomFilter, BlockedBloomParamMismatchError, ChecksumError, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, blockedBitsPerKey, blockedFprAt };
|
|
163
|
+
export { BadMagicError, BlockedBloomFilter, BlockedBloomParamMismatchError, ChecksumError, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, blockedBitsPerKey, blockedFprAt };
|
package/dist/bloom/index.cjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
2
|
+
const require_hasher = require("../hasher-D8LCBTOM.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
3
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
const require_sizing = require("../sizing-BLzZk2zr.cjs");
|
|
4
6
|
//#region src/core/bitset.ts
|
|
5
7
|
var BitSetRangeError = class extends RangeError {
|
|
6
8
|
name = "BitSetRangeError";
|
|
@@ -32,17 +34,10 @@ var BitSet = class {
|
|
|
32
34
|
}
|
|
33
35
|
};
|
|
34
36
|
//#endregion
|
|
35
|
-
//#region src/core/sizing.ts
|
|
36
|
-
function bloomSizing(n, epsilon) {
|
|
37
|
-
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
38
|
-
return {
|
|
39
|
-
m,
|
|
40
|
-
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
//#endregion
|
|
44
37
|
//#region src/bloom/bloom.ts
|
|
45
38
|
const TYPE = 1;
|
|
39
|
+
const PARAMS_SIZE = 16;
|
|
40
|
+
const PARAMS_FIELDS_END = 14;
|
|
46
41
|
var BloomParamMismatchError = class extends Error {
|
|
47
42
|
name = "BloomParamMismatchError";
|
|
48
43
|
};
|
|
@@ -57,7 +52,10 @@ var BloomFilter = class BloomFilter {
|
|
|
57
52
|
require_params.assertPositiveInt(n, "n");
|
|
58
53
|
require_params.assertUint32(n, "n");
|
|
59
54
|
require_params.assertProbability(epsilon, "epsilon");
|
|
60
|
-
return BloomFilter
|
|
55
|
+
return new BloomFilter({
|
|
56
|
+
...require_sizing.bloomSizing(n, epsilon),
|
|
57
|
+
n
|
|
58
|
+
});
|
|
61
59
|
}
|
|
62
60
|
static from(keys, epsilon) {
|
|
63
61
|
const arr = [...keys];
|
|
@@ -67,40 +65,41 @@ var BloomFilter = class BloomFilter {
|
|
|
67
65
|
}
|
|
68
66
|
static fromBytes(bytes) {
|
|
69
67
|
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
70
|
-
if (type !== TYPE) throw new require_serialize.SerializationError(`expected
|
|
68
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
71
69
|
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
72
|
-
require_serialize.assertMinBodyLength(body.length,
|
|
70
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "bloom");
|
|
71
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "bloom");
|
|
73
72
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
74
73
|
const m = dv.getUint32(0, true);
|
|
75
74
|
const k = dv.getUint16(4, true);
|
|
76
75
|
const seed = dv.getUint32(6, true);
|
|
77
76
|
const n = dv.getUint32(10, true);
|
|
78
|
-
require_serialize.assertBodyLength(body.length,
|
|
79
|
-
const f = BloomFilter
|
|
77
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + Math.ceil(m / 8), "bloom");
|
|
78
|
+
const f = new BloomFilter({
|
|
80
79
|
m,
|
|
81
80
|
k,
|
|
82
|
-
seed
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
seed,
|
|
82
|
+
n
|
|
83
|
+
});
|
|
84
|
+
f.#bits.bytes.set(body.subarray(PARAMS_SIZE));
|
|
85
85
|
return f;
|
|
86
86
|
}
|
|
87
|
-
constructor({ m, k, seed = 0 }) {
|
|
87
|
+
constructor({ m, k, seed = 0, n }) {
|
|
88
88
|
require_params.assertPositiveInt(m, "m");
|
|
89
89
|
require_params.assertUint32(m, "m");
|
|
90
90
|
require_params.assertPositiveInt(k, "k");
|
|
91
91
|
require_params.assertUint16(k, "k");
|
|
92
92
|
require_params.assertUint32(seed, "seed");
|
|
93
|
+
if (n !== void 0) {
|
|
94
|
+
require_params.assertPositiveInt(n, "n");
|
|
95
|
+
require_params.assertUint32(n, "n");
|
|
96
|
+
}
|
|
93
97
|
this.#bits = new BitSet(m);
|
|
94
98
|
this.#m = m;
|
|
95
99
|
this.#k = k;
|
|
96
100
|
this.#seed = seed;
|
|
97
101
|
this.#scratch = new Uint32Array(k);
|
|
98
|
-
this.#n = Math.round(m * Math.LN2 / k);
|
|
99
|
-
}
|
|
100
|
-
static #withN(params, n) {
|
|
101
|
-
const f = new BloomFilter(params);
|
|
102
|
-
f.#n = n;
|
|
103
|
-
return f;
|
|
102
|
+
this.#n = n ?? Math.max(1, Math.round(m * Math.LN2 / k));
|
|
104
103
|
}
|
|
105
104
|
get m() {
|
|
106
105
|
return this.#m;
|
|
@@ -123,15 +122,15 @@ var BloomFilter = class BloomFilter {
|
|
|
123
122
|
toBytes() {
|
|
124
123
|
const payload = this.#bits.bytes;
|
|
125
124
|
return require_serialize.writeFrame({
|
|
126
|
-
version:
|
|
125
|
+
version: 5,
|
|
127
126
|
type: TYPE,
|
|
128
127
|
flags: 0
|
|
129
|
-
},
|
|
128
|
+
}, PARAMS_SIZE, payload.length, (body, dv) => {
|
|
130
129
|
dv.setUint32(0, this.#m, true);
|
|
131
130
|
dv.setUint16(4, this.#k, true);
|
|
132
131
|
dv.setUint32(6, this.#seed, true);
|
|
133
132
|
dv.setUint32(10, this.#n, true);
|
|
134
|
-
body.set(payload,
|
|
133
|
+
body.set(payload, PARAMS_SIZE);
|
|
135
134
|
});
|
|
136
135
|
}
|
|
137
136
|
equals(other) {
|
|
@@ -149,20 +148,21 @@ var BloomFilter = class BloomFilter {
|
|
|
149
148
|
const b = other.#bits.bytes;
|
|
150
149
|
const merged = new Uint8Array(a.length);
|
|
151
150
|
for (let i = 0; i < a.length; i++) merged[i] = (a[i] ?? 0) | (b[i] ?? 0);
|
|
152
|
-
const r = BloomFilter
|
|
151
|
+
const r = new BloomFilter({
|
|
153
152
|
m: this.#m,
|
|
154
153
|
k: this.#k,
|
|
155
|
-
seed: this.#seed
|
|
156
|
-
|
|
154
|
+
seed: this.#seed,
|
|
155
|
+
n: this.#n
|
|
156
|
+
});
|
|
157
157
|
r.#bits.bytes.set(merged);
|
|
158
158
|
return r;
|
|
159
159
|
}
|
|
160
160
|
add(key) {
|
|
161
|
-
|
|
161
|
+
require_hasher.probeInto(key, this.#k, this.#m, this.#seed, this.#scratch);
|
|
162
162
|
for (let i = 0; i < this.#k; i++) this.#bits.set(this.#scratch[i] ?? 0);
|
|
163
163
|
}
|
|
164
164
|
has(key) {
|
|
165
|
-
|
|
165
|
+
require_hasher.probeInto(key, this.#k, this.#m, this.#seed, this.#scratch);
|
|
166
166
|
for (let i = 0; i < this.#k; i++) if (!this.#bits.get(this.#scratch[i] ?? 0)) return false;
|
|
167
167
|
return true;
|
|
168
168
|
}
|
|
@@ -173,8 +173,9 @@ exports.BloomFilter = BloomFilter;
|
|
|
173
173
|
exports.BloomParamMismatchError = BloomParamMismatchError;
|
|
174
174
|
exports.ChecksumError = require_serialize.ChecksumError;
|
|
175
175
|
exports.ParamError = require_params.ParamError;
|
|
176
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
176
177
|
exports.SerializationError = require_serialize.SerializationError;
|
|
177
178
|
exports.TruncatedError = require_serialize.TruncatedError;
|
|
178
179
|
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
179
180
|
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
180
|
-
exports.bloomSizing = bloomSizing;
|
|
181
|
+
exports.bloomSizing = require_sizing.bloomSizing;
|
package/dist/bloom/index.d.cts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
2
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
+
import { r as bloomSizing, t as BloomSizing } from "../sizing-wYOW27eY.cjs";
|
|
3
5
|
//#region src/bloom/bloom.d.ts
|
|
4
6
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
5
|
-
declare class BloomParamMismatchError extends Error {
|
|
7
|
+
export declare class BloomParamMismatchError extends Error {
|
|
6
8
|
/** Discriminates this error from other `Error`s. */
|
|
7
9
|
override readonly name = "BloomParamMismatchError";
|
|
8
10
|
}
|
|
@@ -14,6 +16,12 @@ interface BloomParams {
|
|
|
14
16
|
k: number;
|
|
15
17
|
/** Hash seed; defaults to `0`. */
|
|
16
18
|
seed?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Expected number of keys. Omitted, it is derived as `m * ln2 / k`, the
|
|
21
|
+
* count this geometry is optimal for, which need not be the count the
|
|
22
|
+
* geometry was solved for.
|
|
23
|
+
*/
|
|
24
|
+
n?: number;
|
|
17
25
|
}
|
|
18
26
|
/**
|
|
19
27
|
* A classic Bloom filter: a space-efficient set with a tunable false-positive
|
|
@@ -27,7 +35,7 @@ interface BloomParams {
|
|
|
27
35
|
* filter.has("bob"); // false (or a ~1% false positive)
|
|
28
36
|
* ```
|
|
29
37
|
*/
|
|
30
|
-
declare class BloomFilter {
|
|
38
|
+
export declare class BloomFilter {
|
|
31
39
|
#private;
|
|
32
40
|
/**
|
|
33
41
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
@@ -58,7 +66,7 @@ declare class BloomFilter {
|
|
|
58
66
|
* Constructs a filter from low-level {@link BloomParams}. Prefer
|
|
59
67
|
* {@link BloomFilter.create} unless restoring a specific configuration.
|
|
60
68
|
*/
|
|
61
|
-
constructor({ m, k, seed }: BloomParams);
|
|
69
|
+
constructor({ m, k, seed, n }: BloomParams);
|
|
62
70
|
/** Number of bits in the filter. */
|
|
63
71
|
get m(): number;
|
|
64
72
|
/** Number of hash probes per key. */
|
|
@@ -67,7 +75,15 @@ declare class BloomFilter {
|
|
|
67
75
|
get seed(): number;
|
|
68
76
|
/** Number of bits currently set. */
|
|
69
77
|
get length(): number;
|
|
70
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Analytic design bits-per-key, `m / n`.
|
|
80
|
+
*
|
|
81
|
+
* A filter built without an explicit `n` derives one as `m * ln2 / k`, the
|
|
82
|
+
* count its geometry is optimal for. That need not be the count the
|
|
83
|
+
* geometry was solved for, so the same `m` and `k` can report a different
|
|
84
|
+
* cost depending on how the filter was constructed. `create` and
|
|
85
|
+
* `fromBytes` always carry the real `n`.
|
|
86
|
+
*/
|
|
71
87
|
get bitsPerKey(): number;
|
|
72
88
|
/**
|
|
73
89
|
* Estimates the current false-positive rate from the actual fill,
|
|
@@ -128,15 +144,4 @@ declare class BloomFilter {
|
|
|
128
144
|
has(key: BytesLike): boolean;
|
|
129
145
|
}
|
|
130
146
|
//#endregion
|
|
131
|
-
|
|
132
|
-
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
133
|
-
interface BloomSizing {
|
|
134
|
-
/** Number of bits in the filter. */
|
|
135
|
-
m: number;
|
|
136
|
-
/** Number of hash probes per key. */
|
|
137
|
-
k: number;
|
|
138
|
-
}
|
|
139
|
-
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
140
|
-
declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
141
|
-
//#endregion
|
|
142
|
-
export { BadMagicError, BloomFilter, BloomParamMismatchError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, bloomSizing };
|
|
147
|
+
export { BadMagicError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, bloomSizing };
|
package/dist/bloom/index.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
2
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
+
import { r as bloomSizing, t as BloomSizing } from "../sizing-wYOW27eY.js";
|
|
3
5
|
//#region src/bloom/bloom.d.ts
|
|
4
6
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
5
|
-
declare class BloomParamMismatchError extends Error {
|
|
7
|
+
export declare class BloomParamMismatchError extends Error {
|
|
6
8
|
/** Discriminates this error from other `Error`s. */
|
|
7
9
|
override readonly name = "BloomParamMismatchError";
|
|
8
10
|
}
|
|
@@ -14,6 +16,12 @@ interface BloomParams {
|
|
|
14
16
|
k: number;
|
|
15
17
|
/** Hash seed; defaults to `0`. */
|
|
16
18
|
seed?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Expected number of keys. Omitted, it is derived as `m * ln2 / k`, the
|
|
21
|
+
* count this geometry is optimal for, which need not be the count the
|
|
22
|
+
* geometry was solved for.
|
|
23
|
+
*/
|
|
24
|
+
n?: number;
|
|
17
25
|
}
|
|
18
26
|
/**
|
|
19
27
|
* A classic Bloom filter: a space-efficient set with a tunable false-positive
|
|
@@ -27,7 +35,7 @@ interface BloomParams {
|
|
|
27
35
|
* filter.has("bob"); // false (or a ~1% false positive)
|
|
28
36
|
* ```
|
|
29
37
|
*/
|
|
30
|
-
declare class BloomFilter {
|
|
38
|
+
export declare class BloomFilter {
|
|
31
39
|
#private;
|
|
32
40
|
/**
|
|
33
41
|
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
@@ -58,7 +66,7 @@ declare class BloomFilter {
|
|
|
58
66
|
* Constructs a filter from low-level {@link BloomParams}. Prefer
|
|
59
67
|
* {@link BloomFilter.create} unless restoring a specific configuration.
|
|
60
68
|
*/
|
|
61
|
-
constructor({ m, k, seed }: BloomParams);
|
|
69
|
+
constructor({ m, k, seed, n }: BloomParams);
|
|
62
70
|
/** Number of bits in the filter. */
|
|
63
71
|
get m(): number;
|
|
64
72
|
/** Number of hash probes per key. */
|
|
@@ -67,7 +75,15 @@ declare class BloomFilter {
|
|
|
67
75
|
get seed(): number;
|
|
68
76
|
/** Number of bits currently set. */
|
|
69
77
|
get length(): number;
|
|
70
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Analytic design bits-per-key, `m / n`.
|
|
80
|
+
*
|
|
81
|
+
* A filter built without an explicit `n` derives one as `m * ln2 / k`, the
|
|
82
|
+
* count its geometry is optimal for. That need not be the count the
|
|
83
|
+
* geometry was solved for, so the same `m` and `k` can report a different
|
|
84
|
+
* cost depending on how the filter was constructed. `create` and
|
|
85
|
+
* `fromBytes` always carry the real `n`.
|
|
86
|
+
*/
|
|
71
87
|
get bitsPerKey(): number;
|
|
72
88
|
/**
|
|
73
89
|
* Estimates the current false-positive rate from the actual fill,
|
|
@@ -128,15 +144,4 @@ declare class BloomFilter {
|
|
|
128
144
|
has(key: BytesLike): boolean;
|
|
129
145
|
}
|
|
130
146
|
//#endregion
|
|
131
|
-
|
|
132
|
-
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
133
|
-
interface BloomSizing {
|
|
134
|
-
/** Number of bits in the filter. */
|
|
135
|
-
m: number;
|
|
136
|
-
/** Number of hash probes per key. */
|
|
137
|
-
k: number;
|
|
138
|
-
}
|
|
139
|
-
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
140
|
-
declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
141
|
-
//#endregion
|
|
142
|
-
export { BadMagicError, BloomFilter, BloomParamMismatchError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, bloomSizing };
|
|
147
|
+
export { BadMagicError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, bloomSizing };
|