distillate 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -7
- package/dist/bitset-Cf2jwRJX.js +32 -0
- package/dist/bitset-QYGqxnEf.cjs +37 -0
- package/dist/blocked/index.cjs +1 -1
- package/dist/blocked/index.js +1 -1
- package/dist/bloom/index.cjs +3 -33
- package/dist/bloom/index.js +2 -32
- package/dist/cuckoo/index.cjs +278 -0
- package/dist/cuckoo/index.d.cts +164 -0
- package/dist/cuckoo/index.d.ts +164 -0
- package/dist/cuckoo/index.js +267 -0
- package/dist/fuse/index.cjs +1 -1
- package/dist/fuse/index.js +1 -1
- package/dist/{hasher-DVWXS_vJ.js → hasher-3Cyn7Yg-.js} +8 -7
- package/dist/{hasher-D8LCBTOM.cjs → hasher-MhY7aQ8F.cjs} +19 -6
- package/dist/hll/index.cjs +1 -1
- package/dist/hll/index.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +347 -0
- package/dist/scalable/index.d.cts +173 -0
- package/dist/scalable/index.d.ts +173 -0
- package/dist/scalable/index.js +337 -0
- package/package.json +17 -1
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
Probabilistic data structures for JavaScript: space-efficient, approximate answers with tunable error and zero false negatives. TypeScript-first, zero dependencies, and the right structure per workload. It opens with a family of membership filters and is built to grow into other sketches.
|
|
8
8
|
|
|
9
|
-
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0
|
|
9
|
+
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0`. Pin a version if you depend on it.
|
|
10
10
|
|
|
11
11
|
## Why
|
|
12
12
|
|
|
@@ -42,12 +42,14 @@ Every push runs a CI smoke matrix that imports the built package on Node 22/24,
|
|
|
42
42
|
|
|
43
43
|
Each structure ships as its own subpath, so you only bundle what you import.
|
|
44
44
|
|
|
45
|
-
| Import
|
|
46
|
-
|
|
|
47
|
-
| `distillate/bloom`
|
|
48
|
-
| `distillate/
|
|
49
|
-
| `distillate/
|
|
50
|
-
| `distillate/
|
|
45
|
+
| Import | Structure | Answers | Use for |
|
|
46
|
+
| --------------------- | -------------- | ------------------ | -------------------------------------------------------- |
|
|
47
|
+
| `distillate/bloom` | Classic Bloom | seen this key? | Familiar default, migration from `bloom-filters` |
|
|
48
|
+
| `distillate/scalable` | Scalable Bloom | seen this key? | Key count unknown or growing; keeps its FPR bound |
|
|
49
|
+
| `distillate/blocked` | Blocked Bloom | seen this key? | Faster lookups and a lower FPR for a small space premium |
|
|
50
|
+
| `distillate/fuse` | Binary Fuse | seen this key? | Static set built once and queried a lot; least space |
|
|
51
|
+
| `distillate/cuckoo` | Cuckoo | seen this key? | Keys come and go; the one filter with delete |
|
|
52
|
+
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
51
53
|
|
|
52
54
|
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
53
55
|
key set. HyperLogLog is not a filter: it counts distinct keys and cannot report
|
|
@@ -69,6 +71,24 @@ const restored = BloomFilter.fromBytes(bytes);
|
|
|
69
71
|
|
|
70
72
|
Also: `union(other)` (merge equal-parameter filters), `bitsPerKey`, and a low-level `new BloomFilter({ m, k, seed })`.
|
|
71
73
|
|
|
74
|
+
### Scalable Bloom (`distillate/scalable`)
|
|
75
|
+
|
|
76
|
+
For when you cannot say how many keys are coming. It grows by opening larger, tighter Bloom stages as keys arrive, and keeps its false-positive rate under the target however many open. Full API and trade-offs: [Scalable Bloom guide](https://distillate.akxp.net/guides/scalable/).
|
|
77
|
+
|
|
78
|
+
```ts
|
|
79
|
+
import { ScalableBloomFilter } from "distillate/scalable";
|
|
80
|
+
|
|
81
|
+
const seen = ScalableBloomFilter.create(10, 0.01); // first stage, target FPR
|
|
82
|
+
for (let i = 0; i < 30; i++) seen.add(`key-${String(i)}`);
|
|
83
|
+
|
|
84
|
+
seen.stages; // 2
|
|
85
|
+
seen.has("key-29"); // true
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A key already held is not counted again, so duplicates never use up capacity. It costs more bits per key than a Classic Bloom sized for a known `n`; if you know `n`, use that.
|
|
89
|
+
|
|
90
|
+
Also: `from(keys, epsilon)`, `union`, `equals`, `rate`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, and `growth` / `tightening` / `seed` options.
|
|
91
|
+
|
|
72
92
|
### Blocked Bloom (`distillate/blocked`)
|
|
73
93
|
|
|
74
94
|
```ts
|
|
@@ -101,6 +121,25 @@ const precise = BinaryFuse16.from(["alice", "bob", "carol"]);
|
|
|
101
121
|
|
|
102
122
|
Also: `toBytes` / `fromBytes`. No `add` / `delete`; rebuild `from` the new set to change membership.
|
|
103
123
|
|
|
124
|
+
### Cuckoo (`distillate/cuckoo`)
|
|
125
|
+
|
|
126
|
+
For a set that shrinks as well as grows: the one filter here with `delete`. Full API and trade-offs: [Cuckoo guide](https://distillate.akxp.net/guides/cuckoo/).
|
|
127
|
+
|
|
128
|
+
```ts
|
|
129
|
+
import { CuckooFilter } from "distillate/cuckoo";
|
|
130
|
+
|
|
131
|
+
const sessions = CuckooFilter.create(100_000, 0.01);
|
|
132
|
+
sessions.add("alice");
|
|
133
|
+
sessions.has("alice"); // true
|
|
134
|
+
|
|
135
|
+
sessions.delete("alice"); // true
|
|
136
|
+
sessions.has("alice"); // false
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Only delete keys you added: a key never added can share a fingerprint with one that was, and deleting it removes that one instead. An add into a full filter throws `CuckooFullError` and leaves the filter unchanged. It is larger than a Classic Bloom at common targets and smaller only at about 0.2% and below, so pick it for delete, not for space.
|
|
140
|
+
|
|
141
|
+
Also: `from(keys, epsilon)`, `equals`, `rate`, `count`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `cuckooSizing(n, epsilon)`, and a `seed` option.
|
|
142
|
+
|
|
104
143
|
### HyperLogLog (`distillate/hll`)
|
|
105
144
|
|
|
106
145
|
A **sketch**, not a filter: it counts distinct keys in space fixed by precision rather than by the answer, and cannot report whether it saw any particular key. 12 KiB counts a thousand distinct keys or a billion, at about 0.8% relative error. Full API and sizing: [HyperLogLog guide](https://distillate.akxp.net/guides/hll/).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
//#region src/core/bitset.ts
|
|
2
|
+
var BitSetRangeError = class extends RangeError {
|
|
3
|
+
name = "BitSetRangeError";
|
|
4
|
+
};
|
|
5
|
+
const MAX_BITS = 2 ** 32;
|
|
6
|
+
var BitSet = class {
|
|
7
|
+
#bits;
|
|
8
|
+
constructor(nbits) {
|
|
9
|
+
if (nbits > MAX_BITS) throw new BitSetRangeError(`BitSet capacity ${String(nbits)} exceeds the 2^32-bit limit`);
|
|
10
|
+
this.#bits = new Uint8Array(Math.ceil(nbits / 8));
|
|
11
|
+
}
|
|
12
|
+
set(i) {
|
|
13
|
+
const idx = i >>> 3;
|
|
14
|
+
this.#bits[idx] = (this.#bits[idx] ?? 0) | 1 << (i & 7);
|
|
15
|
+
}
|
|
16
|
+
get(i) {
|
|
17
|
+
return ((this.#bits[i >>> 3] ?? 0) & 1 << (i & 7)) !== 0;
|
|
18
|
+
}
|
|
19
|
+
get bytes() {
|
|
20
|
+
return this.#bits;
|
|
21
|
+
}
|
|
22
|
+
count() {
|
|
23
|
+
let total = 0;
|
|
24
|
+
for (let byte of this.#bits) while (byte) {
|
|
25
|
+
byte &= byte - 1;
|
|
26
|
+
total++;
|
|
27
|
+
}
|
|
28
|
+
return total;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
//#endregion
|
|
32
|
+
export { BitSet as t };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
//#region src/core/bitset.ts
|
|
2
|
+
var BitSetRangeError = class extends RangeError {
|
|
3
|
+
name = "BitSetRangeError";
|
|
4
|
+
};
|
|
5
|
+
const MAX_BITS = 2 ** 32;
|
|
6
|
+
var BitSet = class {
|
|
7
|
+
#bits;
|
|
8
|
+
constructor(nbits) {
|
|
9
|
+
if (nbits > MAX_BITS) throw new BitSetRangeError(`BitSet capacity ${String(nbits)} exceeds the 2^32-bit limit`);
|
|
10
|
+
this.#bits = new Uint8Array(Math.ceil(nbits / 8));
|
|
11
|
+
}
|
|
12
|
+
set(i) {
|
|
13
|
+
const idx = i >>> 3;
|
|
14
|
+
this.#bits[idx] = (this.#bits[idx] ?? 0) | 1 << (i & 7);
|
|
15
|
+
}
|
|
16
|
+
get(i) {
|
|
17
|
+
return ((this.#bits[i >>> 3] ?? 0) & 1 << (i & 7)) !== 0;
|
|
18
|
+
}
|
|
19
|
+
get bytes() {
|
|
20
|
+
return this.#bits;
|
|
21
|
+
}
|
|
22
|
+
count() {
|
|
23
|
+
let total = 0;
|
|
24
|
+
for (let byte of this.#bits) while (byte) {
|
|
25
|
+
byte &= byte - 1;
|
|
26
|
+
total++;
|
|
27
|
+
}
|
|
28
|
+
return total;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
//#endregion
|
|
32
|
+
Object.defineProperty(exports, "BitSet", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
get: function() {
|
|
35
|
+
return BitSet;
|
|
36
|
+
}
|
|
37
|
+
});
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
5
|
//#region src/blocked/blocked.ts
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as hash32x2Into,
|
|
1
|
+
import { a as hash32x2Into, u as reduce } from "../hasher-3Cyn7Yg-.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
4
|
//#region src/blocked/blocked.ts
|
package/dist/bloom/index.cjs
CHANGED
|
@@ -1,39 +1,9 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
2
|
+
const require_bitset = require("../bitset-QYGqxnEf.cjs");
|
|
3
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
6
|
const require_sizing = require("../sizing-BLzZk2zr.cjs");
|
|
6
|
-
//#region src/core/bitset.ts
|
|
7
|
-
var BitSetRangeError = class extends RangeError {
|
|
8
|
-
name = "BitSetRangeError";
|
|
9
|
-
};
|
|
10
|
-
const MAX_BITS = 2 ** 32;
|
|
11
|
-
var BitSet = class {
|
|
12
|
-
#bits;
|
|
13
|
-
constructor(nbits) {
|
|
14
|
-
if (nbits > MAX_BITS) throw new BitSetRangeError(`BitSet capacity ${String(nbits)} exceeds the 2^32-bit limit`);
|
|
15
|
-
this.#bits = new Uint8Array(Math.ceil(nbits / 8));
|
|
16
|
-
}
|
|
17
|
-
set(i) {
|
|
18
|
-
const idx = i >>> 3;
|
|
19
|
-
this.#bits[idx] = (this.#bits[idx] ?? 0) | 1 << (i & 7);
|
|
20
|
-
}
|
|
21
|
-
get(i) {
|
|
22
|
-
return ((this.#bits[i >>> 3] ?? 0) & 1 << (i & 7)) !== 0;
|
|
23
|
-
}
|
|
24
|
-
get bytes() {
|
|
25
|
-
return this.#bits;
|
|
26
|
-
}
|
|
27
|
-
count() {
|
|
28
|
-
let total = 0;
|
|
29
|
-
for (let byte of this.#bits) while (byte) {
|
|
30
|
-
byte &= byte - 1;
|
|
31
|
-
total++;
|
|
32
|
-
}
|
|
33
|
-
return total;
|
|
34
|
-
}
|
|
35
|
-
};
|
|
36
|
-
//#endregion
|
|
37
7
|
//#region src/bloom/bloom.ts
|
|
38
8
|
const TYPE = 1;
|
|
39
9
|
const PARAMS_SIZE = 16;
|
|
@@ -94,7 +64,7 @@ var BloomFilter = class BloomFilter {
|
|
|
94
64
|
require_params.assertPositiveInt(n, "n");
|
|
95
65
|
require_params.assertUint32(n, "n");
|
|
96
66
|
}
|
|
97
|
-
this.#bits = new BitSet(m);
|
|
67
|
+
this.#bits = new require_bitset.BitSet(m);
|
|
98
68
|
this.#m = m;
|
|
99
69
|
this.#k = k;
|
|
100
70
|
this.#seed = seed;
|
package/dist/bloom/index.js
CHANGED
|
@@ -1,38 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BitSet } from "../bitset-Cf2jwRJX.js";
|
|
2
|
+
import { c as probeInto } from "../hasher-3Cyn7Yg-.js";
|
|
2
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
4
|
import { a as assertUint16, i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
5
|
import { t as bloomSizing } from "../sizing-BtUrtvF_.js";
|
|
5
|
-
//#region src/core/bitset.ts
|
|
6
|
-
var BitSetRangeError = class extends RangeError {
|
|
7
|
-
name = "BitSetRangeError";
|
|
8
|
-
};
|
|
9
|
-
const MAX_BITS = 2 ** 32;
|
|
10
|
-
var BitSet = class {
|
|
11
|
-
#bits;
|
|
12
|
-
constructor(nbits) {
|
|
13
|
-
if (nbits > MAX_BITS) throw new BitSetRangeError(`BitSet capacity ${String(nbits)} exceeds the 2^32-bit limit`);
|
|
14
|
-
this.#bits = new Uint8Array(Math.ceil(nbits / 8));
|
|
15
|
-
}
|
|
16
|
-
set(i) {
|
|
17
|
-
const idx = i >>> 3;
|
|
18
|
-
this.#bits[idx] = (this.#bits[idx] ?? 0) | 1 << (i & 7);
|
|
19
|
-
}
|
|
20
|
-
get(i) {
|
|
21
|
-
return ((this.#bits[i >>> 3] ?? 0) & 1 << (i & 7)) !== 0;
|
|
22
|
-
}
|
|
23
|
-
get bytes() {
|
|
24
|
-
return this.#bits;
|
|
25
|
-
}
|
|
26
|
-
count() {
|
|
27
|
-
let total = 0;
|
|
28
|
-
for (let byte of this.#bits) while (byte) {
|
|
29
|
-
byte &= byte - 1;
|
|
30
|
-
total++;
|
|
31
|
-
}
|
|
32
|
-
return total;
|
|
33
|
-
}
|
|
34
|
-
};
|
|
35
|
-
//#endregion
|
|
36
6
|
//#region src/bloom/bloom.ts
|
|
37
7
|
const TYPE = 1;
|
|
38
8
|
const PARAMS_SIZE = 16;
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
function cuckooSizing(n, epsilon) {
|
|
6
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
7
|
+
const f = Math.max(4, Math.ceil(Math.log2(8 / epsilon)));
|
|
8
|
+
if (f > 32) throw new require_params.ParamError(`epsilon ${String(epsilon)} needs a ${String(f)}-bit fingerprint, above the maximum ${String(32)} bits`);
|
|
9
|
+
const buckets = Math.ceil(n / 3.8) + Math.ceil(Math.sqrt(n));
|
|
10
|
+
return {
|
|
11
|
+
f,
|
|
12
|
+
buckets,
|
|
13
|
+
bitsPerKey: 4 * f * buckets / n
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/cuckoo/cuckoo.ts
|
|
18
|
+
const TYPE = 7;
|
|
19
|
+
const PARAMS_SIZE = 32;
|
|
20
|
+
const PARAMS_FIELDS_END = 28;
|
|
21
|
+
const padded8 = (length) => Math.ceil(length / 8) * 8;
|
|
22
|
+
const wordBytes = (words) => new Uint8Array(words.buffer, words.byteOffset, words.byteLength);
|
|
23
|
+
const SLOTS = 4;
|
|
24
|
+
const MAX_KICKS = 500;
|
|
25
|
+
const UNDO_SLOT = new Float64Array(MAX_KICKS);
|
|
26
|
+
const UNDO_FP = new Uint32Array(MAX_KICKS);
|
|
27
|
+
const HASH = {
|
|
28
|
+
w0: 0,
|
|
29
|
+
w1: 0,
|
|
30
|
+
w2: 0,
|
|
31
|
+
w3: 0
|
|
32
|
+
};
|
|
33
|
+
let restoring;
|
|
34
|
+
var CuckooFullError = class extends Error {
|
|
35
|
+
name = "CuckooFullError";
|
|
36
|
+
};
|
|
37
|
+
var CuckooFilter = class CuckooFilter {
|
|
38
|
+
#n;
|
|
39
|
+
#epsilon;
|
|
40
|
+
#seed;
|
|
41
|
+
#f;
|
|
42
|
+
#buckets;
|
|
43
|
+
#mask;
|
|
44
|
+
#words;
|
|
45
|
+
#count = 0;
|
|
46
|
+
#fp = 0;
|
|
47
|
+
#i1 = 0;
|
|
48
|
+
#i2 = 0;
|
|
49
|
+
static create(n, epsilon, options = {}) {
|
|
50
|
+
return new CuckooFilter({
|
|
51
|
+
...options,
|
|
52
|
+
n,
|
|
53
|
+
epsilon
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
static from(keys, epsilon, options = {}) {
|
|
57
|
+
const seen = new Set();
|
|
58
|
+
const distinct = [];
|
|
59
|
+
for (const key of keys) {
|
|
60
|
+
require_hasher.hash128KeyInto(key, options.seed ?? 0, HASH);
|
|
61
|
+
const id = `${String(HASH.w0)},${String(HASH.w1)},${String(HASH.w2)},${String(HASH.w3)}`;
|
|
62
|
+
if (seen.has(id)) continue;
|
|
63
|
+
seen.add(id);
|
|
64
|
+
distinct.push(key);
|
|
65
|
+
}
|
|
66
|
+
const f = CuckooFilter.create(Math.max(1, distinct.length), epsilon, options);
|
|
67
|
+
for (const k of distinct) f.add(k);
|
|
68
|
+
return f;
|
|
69
|
+
}
|
|
70
|
+
static fromBytes(bytes) {
|
|
71
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
72
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
73
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
74
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "cuckoo");
|
|
75
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "cuckoo");
|
|
76
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
77
|
+
const params = {
|
|
78
|
+
n: view.getUint32(0, true),
|
|
79
|
+
seed: view.getUint32(4, true),
|
|
80
|
+
epsilon: view.getFloat64(8, true)
|
|
81
|
+
};
|
|
82
|
+
let filter;
|
|
83
|
+
try {
|
|
84
|
+
const f = view.getUint32(16, true);
|
|
85
|
+
const buckets = view.getUint32(20, true);
|
|
86
|
+
if (f < 4 || f > 32) throw new require_serialize.SerializationError(`cuckoo: stored f ${String(f)} is outside 4..${String(32)}`);
|
|
87
|
+
if (buckets < 1) throw new require_serialize.SerializationError("cuckoo: stored buckets is 0");
|
|
88
|
+
const m = f * SLOTS * buckets;
|
|
89
|
+
if (m > 4294967295) throw new require_serialize.SerializationError(`cuckoo: stored geometry gives m ${String(m)} bits, more than 2^32 - 1`);
|
|
90
|
+
const words = Math.ceil(m / 32);
|
|
91
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + padded8(4 * words), "cuckoo");
|
|
92
|
+
restoring = {
|
|
93
|
+
f,
|
|
94
|
+
buckets
|
|
95
|
+
};
|
|
96
|
+
try {
|
|
97
|
+
filter = new CuckooFilter(params);
|
|
98
|
+
} finally {
|
|
99
|
+
restoring = void 0;
|
|
100
|
+
}
|
|
101
|
+
} catch (err) {
|
|
102
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`cuckoo: ${err.message}`);
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
const words = filter.#words;
|
|
106
|
+
for (let w = 0; w < words.length; w++) words[w] = view.getUint32(PARAMS_SIZE + 4 * w, true);
|
|
107
|
+
const tail = filter.m % 32;
|
|
108
|
+
if (tail !== 0 && (words[words.length - 1] ?? 0) >>> tail !== 0) throw new require_serialize.SerializationError("cuckoo: bits set past the last slot");
|
|
109
|
+
if (body.subarray(PARAMS_SIZE + 4 * words.length).some((b) => b !== 0)) throw new require_serialize.SerializationError("cuckoo: slot padding is not zero");
|
|
110
|
+
const count = view.getUint32(24, true);
|
|
111
|
+
let occupied = 0;
|
|
112
|
+
for (let j = 0; j < filter.capacity; j++) if (filter.#slot(j) !== 0) occupied++;
|
|
113
|
+
if (count !== occupied) throw new require_serialize.SerializationError(`cuckoo: count ${String(count)} does not match ${String(occupied)} occupied slots`);
|
|
114
|
+
filter.#count = count;
|
|
115
|
+
return filter;
|
|
116
|
+
}
|
|
117
|
+
static fromJSON(value) {
|
|
118
|
+
return CuckooFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
119
|
+
}
|
|
120
|
+
constructor({ n, epsilon, seed = 0 }) {
|
|
121
|
+
require_params.assertPositiveInt(n, "n");
|
|
122
|
+
require_params.assertUint32(n, "n");
|
|
123
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
124
|
+
require_params.assertUint32(seed, "seed");
|
|
125
|
+
const { f, buckets } = restoring ?? cuckooSizing(n, epsilon);
|
|
126
|
+
require_params.assertUint32(f * SLOTS * buckets, "m");
|
|
127
|
+
this.#n = n;
|
|
128
|
+
this.#epsilon = epsilon;
|
|
129
|
+
this.#seed = seed;
|
|
130
|
+
this.#f = f;
|
|
131
|
+
this.#buckets = buckets;
|
|
132
|
+
this.#mask = 2 ** f - 1;
|
|
133
|
+
this.#words = new Uint32Array(Math.ceil(f * SLOTS * buckets / 32));
|
|
134
|
+
}
|
|
135
|
+
add(key) {
|
|
136
|
+
this.#hash(key);
|
|
137
|
+
if (this.#replace(this.#i1, 0, this.#fp) || this.#replace(this.#i2, 0, this.#fp)) {
|
|
138
|
+
this.#count++;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
let i = (HASH.w2 & 1) === 0 ? this.#i1 : this.#i2;
|
|
142
|
+
let x = (HASH.w3 | 1) >>> 0;
|
|
143
|
+
let fp = this.#fp;
|
|
144
|
+
for (let kick = 0; kick < MAX_KICKS; kick++) {
|
|
145
|
+
x ^= x << 13;
|
|
146
|
+
x >>>= 0;
|
|
147
|
+
x ^= x >>> 17;
|
|
148
|
+
x ^= x << 5;
|
|
149
|
+
x >>>= 0;
|
|
150
|
+
const j = i * SLOTS + (x & 3);
|
|
151
|
+
const victim = this.#slot(j);
|
|
152
|
+
UNDO_SLOT[kick] = j;
|
|
153
|
+
UNDO_FP[kick] = victim;
|
|
154
|
+
this.#setSlot(j, fp);
|
|
155
|
+
fp = victim;
|
|
156
|
+
i = this.#alt(i, fp);
|
|
157
|
+
if (this.#replace(i, 0, fp)) {
|
|
158
|
+
this.#count++;
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
for (let kick = 499; kick >= 0; kick--) this.#setSlot(UNDO_SLOT[kick] ?? 0, UNDO_FP[kick] ?? 0);
|
|
163
|
+
throw new CuckooFullError(`cuckoo filter is full at ${String(this.#count)} of ${String(this.capacity)} slots`);
|
|
164
|
+
}
|
|
165
|
+
delete(key) {
|
|
166
|
+
this.#hash(key);
|
|
167
|
+
const removed = this.#replace(this.#i1, this.#fp, 0) || this.#i2 !== this.#i1 && this.#replace(this.#i2, this.#fp, 0);
|
|
168
|
+
if (removed) this.#count--;
|
|
169
|
+
return removed;
|
|
170
|
+
}
|
|
171
|
+
has(key) {
|
|
172
|
+
this.#hash(key);
|
|
173
|
+
return this.#find(this.#i1, this.#fp) >= 0 || this.#find(this.#i2, this.#fp) >= 0;
|
|
174
|
+
}
|
|
175
|
+
rate() {
|
|
176
|
+
const occupied = 8 * this.#count / this.capacity;
|
|
177
|
+
return 1 - (1 - 1 / this.#mask) ** occupied;
|
|
178
|
+
}
|
|
179
|
+
toBytes() {
|
|
180
|
+
const words = this.#words;
|
|
181
|
+
return require_serialize.writeFrame({
|
|
182
|
+
version: 5,
|
|
183
|
+
type: TYPE,
|
|
184
|
+
flags: 0
|
|
185
|
+
}, PARAMS_SIZE, padded8(4 * words.length), (_, view) => {
|
|
186
|
+
view.setUint32(0, this.#n, true);
|
|
187
|
+
view.setUint32(4, this.#seed, true);
|
|
188
|
+
view.setFloat64(8, this.#epsilon, true);
|
|
189
|
+
view.setUint32(16, this.#f, true);
|
|
190
|
+
view.setUint32(20, this.#buckets, true);
|
|
191
|
+
view.setUint32(24, this.#count, true);
|
|
192
|
+
words.forEach((word, w) => {
|
|
193
|
+
view.setUint32(PARAMS_SIZE + 4 * w, word, true);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
equals(other) {
|
|
198
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && require_serialize.bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
199
|
+
}
|
|
200
|
+
toJSON() {
|
|
201
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
202
|
+
}
|
|
203
|
+
#hash(key) {
|
|
204
|
+
require_hasher.hash128KeyInto(key, this.#seed, HASH);
|
|
205
|
+
this.#fp = HASH.w1 >>> 32 - this.#f || 1;
|
|
206
|
+
this.#i1 = require_hasher.reduce(HASH.w0, this.#buckets);
|
|
207
|
+
this.#i2 = this.#alt(this.#i1, this.#fp);
|
|
208
|
+
}
|
|
209
|
+
#alt(i, fp) {
|
|
210
|
+
const b = this.#buckets;
|
|
211
|
+
return ((Math.imul(fp, 1540483477) >>> 0) % b + b - i) % b;
|
|
212
|
+
}
|
|
213
|
+
#find(bucket, value) {
|
|
214
|
+
for (let j = bucket * SLOTS; j < bucket * SLOTS + SLOTS; j++) if (this.#slot(j) === value) return j;
|
|
215
|
+
return -1;
|
|
216
|
+
}
|
|
217
|
+
#replace(bucket, from, to) {
|
|
218
|
+
const j = this.#find(bucket, from);
|
|
219
|
+
if (j < 0) return false;
|
|
220
|
+
this.#setSlot(j, to);
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
#slot(j) {
|
|
224
|
+
const bit = j * this.#f;
|
|
225
|
+
const w = bit >>> 5;
|
|
226
|
+
const off = bit & 31;
|
|
227
|
+
let v = (this.#words[w] ?? 0) >>> off;
|
|
228
|
+
if (off + this.#f > 32) v |= (this.#words[w + 1] ?? 0) << 32 - off;
|
|
229
|
+
return (v & this.#mask) >>> 0;
|
|
230
|
+
}
|
|
231
|
+
#setSlot(j, fp) {
|
|
232
|
+
const bit = j * this.#f;
|
|
233
|
+
const w = bit >>> 5;
|
|
234
|
+
const off = bit & 31;
|
|
235
|
+
const words = this.#words;
|
|
236
|
+
words[w] = (words[w] ?? 0) & ~(this.#mask << off) | fp << off;
|
|
237
|
+
if (off + this.#f > 32) {
|
|
238
|
+
const shift = 32 - off;
|
|
239
|
+
words[w + 1] = (words[w + 1] ?? 0) & ~(this.#mask >>> shift) | fp >>> shift;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
get count() {
|
|
243
|
+
return this.#count;
|
|
244
|
+
}
|
|
245
|
+
get capacity() {
|
|
246
|
+
return SLOTS * this.#buckets;
|
|
247
|
+
}
|
|
248
|
+
get buckets() {
|
|
249
|
+
return this.#buckets;
|
|
250
|
+
}
|
|
251
|
+
get fingerprintBits() {
|
|
252
|
+
return this.#f;
|
|
253
|
+
}
|
|
254
|
+
get m() {
|
|
255
|
+
return this.#f * this.capacity;
|
|
256
|
+
}
|
|
257
|
+
get bitsPerKey() {
|
|
258
|
+
return this.m / this.#n;
|
|
259
|
+
}
|
|
260
|
+
get seed() {
|
|
261
|
+
return this.#seed;
|
|
262
|
+
}
|
|
263
|
+
get epsilon() {
|
|
264
|
+
return this.#epsilon;
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
//#endregion
|
|
268
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
269
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
270
|
+
exports.CuckooFilter = CuckooFilter;
|
|
271
|
+
exports.CuckooFullError = CuckooFullError;
|
|
272
|
+
exports.ParamError = require_params.ParamError;
|
|
273
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
274
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
275
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
276
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
277
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
278
|
+
exports.cuckooSizing = cuckooSizing;
|