distillate 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -1
- package/dist/cuckoo/index.cjs +278 -0
- package/dist/cuckoo/index.d.cts +164 -0
- package/dist/cuckoo/index.d.ts +164 -0
- package/dist/cuckoo/index.js +267 -0
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +21 -14
- package/dist/scalable/index.js +21 -14
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
Probabilistic data structures for JavaScript: space-efficient, approximate answers with tunable error and zero false negatives. TypeScript-first, zero dependencies, and the right structure per workload. It opens with a family of membership filters and is built to grow into other sketches.
|
|
8
8
|
|
|
9
|
-
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0
|
|
9
|
+
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0`. Pin a version if you depend on it.
|
|
10
10
|
|
|
11
11
|
## Why
|
|
12
12
|
|
|
@@ -48,6 +48,7 @@ Each structure ships as its own subpath, so you only bundle what you import.
|
|
|
48
48
|
| `distillate/scalable` | Scalable Bloom | seen this key? | Key count unknown or growing; keeps its FPR bound |
|
|
49
49
|
| `distillate/blocked` | Blocked Bloom | seen this key? | Faster lookups and a lower FPR for a small space premium |
|
|
50
50
|
| `distillate/fuse` | Binary Fuse | seen this key? | Static set built once and queried a lot; least space |
|
|
51
|
+
| `distillate/cuckoo` | Cuckoo | seen this key? | Keys come and go; the one filter with delete |
|
|
51
52
|
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
52
53
|
|
|
53
54
|
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
@@ -120,6 +121,25 @@ const precise = BinaryFuse16.from(["alice", "bob", "carol"]);
|
|
|
120
121
|
|
|
121
122
|
Also: `toBytes` / `fromBytes`. No `add` / `delete`; rebuild `from` the new set to change membership.
|
|
122
123
|
|
|
124
|
+
### Cuckoo (`distillate/cuckoo`)
|
|
125
|
+
|
|
126
|
+
For a set that shrinks as well as grows: the one filter here with `delete`. Full API and trade-offs: [Cuckoo guide](https://distillate.akxp.net/guides/cuckoo/).
|
|
127
|
+
|
|
128
|
+
```ts
|
|
129
|
+
import { CuckooFilter } from "distillate/cuckoo";
|
|
130
|
+
|
|
131
|
+
const sessions = CuckooFilter.create(100_000, 0.01);
|
|
132
|
+
sessions.add("alice");
|
|
133
|
+
sessions.has("alice"); // true
|
|
134
|
+
|
|
135
|
+
sessions.delete("alice"); // true
|
|
136
|
+
sessions.has("alice"); // false
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Only delete keys you added: a key never added can share a fingerprint with one that was, and deleting it removes that one instead. An add into a full filter throws `CuckooFullError` and leaves the filter unchanged. It is larger than a Classic Bloom at common targets and smaller only at about 0.2% and below, so pick it for delete, not for space.
|
|
140
|
+
|
|
141
|
+
Also: `from(keys, epsilon)`, `equals`, `rate`, `count`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `cuckooSizing(n, epsilon)`, and a `seed` option.
|
|
142
|
+
|
|
123
143
|
### HyperLogLog (`distillate/hll`)
|
|
124
144
|
|
|
125
145
|
A **sketch**, not a filter: it counts distinct keys in space fixed by precision rather than by the answer, and cannot report whether it saw any particular key. 12 KiB counts a thousand distinct keys or a billion, at about 0.8% relative error. Full API and sizing: [HyperLogLog guide](https://distillate.akxp.net/guides/hll/).
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
function cuckooSizing(n, epsilon) {
|
|
6
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
7
|
+
const f = Math.max(4, Math.ceil(Math.log2(8 / epsilon)));
|
|
8
|
+
if (f > 32) throw new require_params.ParamError(`epsilon ${String(epsilon)} needs a ${String(f)}-bit fingerprint, above the maximum ${String(32)} bits`);
|
|
9
|
+
const buckets = Math.ceil(n / 3.8) + Math.ceil(Math.sqrt(n));
|
|
10
|
+
return {
|
|
11
|
+
f,
|
|
12
|
+
buckets,
|
|
13
|
+
bitsPerKey: 4 * f * buckets / n
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/cuckoo/cuckoo.ts
|
|
18
|
+
const TYPE = 7;
|
|
19
|
+
const PARAMS_SIZE = 32;
|
|
20
|
+
const PARAMS_FIELDS_END = 28;
|
|
21
|
+
const padded8 = (length) => Math.ceil(length / 8) * 8;
|
|
22
|
+
const wordBytes = (words) => new Uint8Array(words.buffer, words.byteOffset, words.byteLength);
|
|
23
|
+
const SLOTS = 4;
|
|
24
|
+
const MAX_KICKS = 500;
|
|
25
|
+
const UNDO_SLOT = new Float64Array(MAX_KICKS);
|
|
26
|
+
const UNDO_FP = new Uint32Array(MAX_KICKS);
|
|
27
|
+
const HASH = {
|
|
28
|
+
w0: 0,
|
|
29
|
+
w1: 0,
|
|
30
|
+
w2: 0,
|
|
31
|
+
w3: 0
|
|
32
|
+
};
|
|
33
|
+
let restoring;
|
|
34
|
+
var CuckooFullError = class extends Error {
|
|
35
|
+
name = "CuckooFullError";
|
|
36
|
+
};
|
|
37
|
+
var CuckooFilter = class CuckooFilter {
|
|
38
|
+
#n;
|
|
39
|
+
#epsilon;
|
|
40
|
+
#seed;
|
|
41
|
+
#f;
|
|
42
|
+
#buckets;
|
|
43
|
+
#mask;
|
|
44
|
+
#words;
|
|
45
|
+
#count = 0;
|
|
46
|
+
#fp = 0;
|
|
47
|
+
#i1 = 0;
|
|
48
|
+
#i2 = 0;
|
|
49
|
+
static create(n, epsilon, options = {}) {
|
|
50
|
+
return new CuckooFilter({
|
|
51
|
+
...options,
|
|
52
|
+
n,
|
|
53
|
+
epsilon
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
static from(keys, epsilon, options = {}) {
|
|
57
|
+
const seen = new Set();
|
|
58
|
+
const distinct = [];
|
|
59
|
+
for (const key of keys) {
|
|
60
|
+
require_hasher.hash128KeyInto(key, options.seed ?? 0, HASH);
|
|
61
|
+
const id = `${String(HASH.w0)},${String(HASH.w1)},${String(HASH.w2)},${String(HASH.w3)}`;
|
|
62
|
+
if (seen.has(id)) continue;
|
|
63
|
+
seen.add(id);
|
|
64
|
+
distinct.push(key);
|
|
65
|
+
}
|
|
66
|
+
const f = CuckooFilter.create(Math.max(1, distinct.length), epsilon, options);
|
|
67
|
+
for (const k of distinct) f.add(k);
|
|
68
|
+
return f;
|
|
69
|
+
}
|
|
70
|
+
static fromBytes(bytes) {
|
|
71
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
72
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
73
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
74
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "cuckoo");
|
|
75
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "cuckoo");
|
|
76
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
77
|
+
const params = {
|
|
78
|
+
n: view.getUint32(0, true),
|
|
79
|
+
seed: view.getUint32(4, true),
|
|
80
|
+
epsilon: view.getFloat64(8, true)
|
|
81
|
+
};
|
|
82
|
+
let filter;
|
|
83
|
+
try {
|
|
84
|
+
const f = view.getUint32(16, true);
|
|
85
|
+
const buckets = view.getUint32(20, true);
|
|
86
|
+
if (f < 4 || f > 32) throw new require_serialize.SerializationError(`cuckoo: stored f ${String(f)} is outside 4..${String(32)}`);
|
|
87
|
+
if (buckets < 1) throw new require_serialize.SerializationError("cuckoo: stored buckets is 0");
|
|
88
|
+
const m = f * SLOTS * buckets;
|
|
89
|
+
if (m > 4294967295) throw new require_serialize.SerializationError(`cuckoo: stored geometry gives m ${String(m)} bits, more than 2^32 - 1`);
|
|
90
|
+
const words = Math.ceil(m / 32);
|
|
91
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + padded8(4 * words), "cuckoo");
|
|
92
|
+
restoring = {
|
|
93
|
+
f,
|
|
94
|
+
buckets
|
|
95
|
+
};
|
|
96
|
+
try {
|
|
97
|
+
filter = new CuckooFilter(params);
|
|
98
|
+
} finally {
|
|
99
|
+
restoring = void 0;
|
|
100
|
+
}
|
|
101
|
+
} catch (err) {
|
|
102
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`cuckoo: ${err.message}`);
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
const words = filter.#words;
|
|
106
|
+
for (let w = 0; w < words.length; w++) words[w] = view.getUint32(PARAMS_SIZE + 4 * w, true);
|
|
107
|
+
const tail = filter.m % 32;
|
|
108
|
+
if (tail !== 0 && (words[words.length - 1] ?? 0) >>> tail !== 0) throw new require_serialize.SerializationError("cuckoo: bits set past the last slot");
|
|
109
|
+
if (body.subarray(PARAMS_SIZE + 4 * words.length).some((b) => b !== 0)) throw new require_serialize.SerializationError("cuckoo: slot padding is not zero");
|
|
110
|
+
const count = view.getUint32(24, true);
|
|
111
|
+
let occupied = 0;
|
|
112
|
+
for (let j = 0; j < filter.capacity; j++) if (filter.#slot(j) !== 0) occupied++;
|
|
113
|
+
if (count !== occupied) throw new require_serialize.SerializationError(`cuckoo: count ${String(count)} does not match ${String(occupied)} occupied slots`);
|
|
114
|
+
filter.#count = count;
|
|
115
|
+
return filter;
|
|
116
|
+
}
|
|
117
|
+
static fromJSON(value) {
|
|
118
|
+
return CuckooFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
119
|
+
}
|
|
120
|
+
constructor({ n, epsilon, seed = 0 }) {
|
|
121
|
+
require_params.assertPositiveInt(n, "n");
|
|
122
|
+
require_params.assertUint32(n, "n");
|
|
123
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
124
|
+
require_params.assertUint32(seed, "seed");
|
|
125
|
+
const { f, buckets } = restoring ?? cuckooSizing(n, epsilon);
|
|
126
|
+
require_params.assertUint32(f * SLOTS * buckets, "m");
|
|
127
|
+
this.#n = n;
|
|
128
|
+
this.#epsilon = epsilon;
|
|
129
|
+
this.#seed = seed;
|
|
130
|
+
this.#f = f;
|
|
131
|
+
this.#buckets = buckets;
|
|
132
|
+
this.#mask = 2 ** f - 1;
|
|
133
|
+
this.#words = new Uint32Array(Math.ceil(f * SLOTS * buckets / 32));
|
|
134
|
+
}
|
|
135
|
+
add(key) {
|
|
136
|
+
this.#hash(key);
|
|
137
|
+
if (this.#replace(this.#i1, 0, this.#fp) || this.#replace(this.#i2, 0, this.#fp)) {
|
|
138
|
+
this.#count++;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
let i = (HASH.w2 & 1) === 0 ? this.#i1 : this.#i2;
|
|
142
|
+
let x = (HASH.w3 | 1) >>> 0;
|
|
143
|
+
let fp = this.#fp;
|
|
144
|
+
for (let kick = 0; kick < MAX_KICKS; kick++) {
|
|
145
|
+
x ^= x << 13;
|
|
146
|
+
x >>>= 0;
|
|
147
|
+
x ^= x >>> 17;
|
|
148
|
+
x ^= x << 5;
|
|
149
|
+
x >>>= 0;
|
|
150
|
+
const j = i * SLOTS + (x & 3);
|
|
151
|
+
const victim = this.#slot(j);
|
|
152
|
+
UNDO_SLOT[kick] = j;
|
|
153
|
+
UNDO_FP[kick] = victim;
|
|
154
|
+
this.#setSlot(j, fp);
|
|
155
|
+
fp = victim;
|
|
156
|
+
i = this.#alt(i, fp);
|
|
157
|
+
if (this.#replace(i, 0, fp)) {
|
|
158
|
+
this.#count++;
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
for (let kick = 499; kick >= 0; kick--) this.#setSlot(UNDO_SLOT[kick] ?? 0, UNDO_FP[kick] ?? 0);
|
|
163
|
+
throw new CuckooFullError(`cuckoo filter is full at ${String(this.#count)} of ${String(this.capacity)} slots`);
|
|
164
|
+
}
|
|
165
|
+
delete(key) {
|
|
166
|
+
this.#hash(key);
|
|
167
|
+
const removed = this.#replace(this.#i1, this.#fp, 0) || this.#i2 !== this.#i1 && this.#replace(this.#i2, this.#fp, 0);
|
|
168
|
+
if (removed) this.#count--;
|
|
169
|
+
return removed;
|
|
170
|
+
}
|
|
171
|
+
has(key) {
|
|
172
|
+
this.#hash(key);
|
|
173
|
+
return this.#find(this.#i1, this.#fp) >= 0 || this.#find(this.#i2, this.#fp) >= 0;
|
|
174
|
+
}
|
|
175
|
+
rate() {
|
|
176
|
+
const occupied = 8 * this.#count / this.capacity;
|
|
177
|
+
return 1 - (1 - 1 / this.#mask) ** occupied;
|
|
178
|
+
}
|
|
179
|
+
toBytes() {
|
|
180
|
+
const words = this.#words;
|
|
181
|
+
return require_serialize.writeFrame({
|
|
182
|
+
version: 5,
|
|
183
|
+
type: TYPE,
|
|
184
|
+
flags: 0
|
|
185
|
+
}, PARAMS_SIZE, padded8(4 * words.length), (_, view) => {
|
|
186
|
+
view.setUint32(0, this.#n, true);
|
|
187
|
+
view.setUint32(4, this.#seed, true);
|
|
188
|
+
view.setFloat64(8, this.#epsilon, true);
|
|
189
|
+
view.setUint32(16, this.#f, true);
|
|
190
|
+
view.setUint32(20, this.#buckets, true);
|
|
191
|
+
view.setUint32(24, this.#count, true);
|
|
192
|
+
words.forEach((word, w) => {
|
|
193
|
+
view.setUint32(PARAMS_SIZE + 4 * w, word, true);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
equals(other) {
|
|
198
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && require_serialize.bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
199
|
+
}
|
|
200
|
+
toJSON() {
|
|
201
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
202
|
+
}
|
|
203
|
+
#hash(key) {
|
|
204
|
+
require_hasher.hash128KeyInto(key, this.#seed, HASH);
|
|
205
|
+
this.#fp = HASH.w1 >>> 32 - this.#f || 1;
|
|
206
|
+
this.#i1 = require_hasher.reduce(HASH.w0, this.#buckets);
|
|
207
|
+
this.#i2 = this.#alt(this.#i1, this.#fp);
|
|
208
|
+
}
|
|
209
|
+
#alt(i, fp) {
|
|
210
|
+
const b = this.#buckets;
|
|
211
|
+
return ((Math.imul(fp, 1540483477) >>> 0) % b + b - i) % b;
|
|
212
|
+
}
|
|
213
|
+
#find(bucket, value) {
|
|
214
|
+
for (let j = bucket * SLOTS; j < bucket * SLOTS + SLOTS; j++) if (this.#slot(j) === value) return j;
|
|
215
|
+
return -1;
|
|
216
|
+
}
|
|
217
|
+
#replace(bucket, from, to) {
|
|
218
|
+
const j = this.#find(bucket, from);
|
|
219
|
+
if (j < 0) return false;
|
|
220
|
+
this.#setSlot(j, to);
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
#slot(j) {
|
|
224
|
+
const bit = j * this.#f;
|
|
225
|
+
const w = bit >>> 5;
|
|
226
|
+
const off = bit & 31;
|
|
227
|
+
let v = (this.#words[w] ?? 0) >>> off;
|
|
228
|
+
if (off + this.#f > 32) v |= (this.#words[w + 1] ?? 0) << 32 - off;
|
|
229
|
+
return (v & this.#mask) >>> 0;
|
|
230
|
+
}
|
|
231
|
+
#setSlot(j, fp) {
|
|
232
|
+
const bit = j * this.#f;
|
|
233
|
+
const w = bit >>> 5;
|
|
234
|
+
const off = bit & 31;
|
|
235
|
+
const words = this.#words;
|
|
236
|
+
words[w] = (words[w] ?? 0) & ~(this.#mask << off) | fp << off;
|
|
237
|
+
if (off + this.#f > 32) {
|
|
238
|
+
const shift = 32 - off;
|
|
239
|
+
words[w + 1] = (words[w + 1] ?? 0) & ~(this.#mask >>> shift) | fp >>> shift;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
get count() {
|
|
243
|
+
return this.#count;
|
|
244
|
+
}
|
|
245
|
+
get capacity() {
|
|
246
|
+
return SLOTS * this.#buckets;
|
|
247
|
+
}
|
|
248
|
+
get buckets() {
|
|
249
|
+
return this.#buckets;
|
|
250
|
+
}
|
|
251
|
+
get fingerprintBits() {
|
|
252
|
+
return this.#f;
|
|
253
|
+
}
|
|
254
|
+
get m() {
|
|
255
|
+
return this.#f * this.capacity;
|
|
256
|
+
}
|
|
257
|
+
get bitsPerKey() {
|
|
258
|
+
return this.m / this.#n;
|
|
259
|
+
}
|
|
260
|
+
get seed() {
|
|
261
|
+
return this.#seed;
|
|
262
|
+
}
|
|
263
|
+
get epsilon() {
|
|
264
|
+
return this.#epsilon;
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
//#endregion
|
|
268
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
269
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
270
|
+
exports.CuckooFilter = CuckooFilter;
|
|
271
|
+
exports.CuckooFullError = CuckooFullError;
|
|
272
|
+
exports.ParamError = require_params.ParamError;
|
|
273
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
274
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
275
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
276
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
277
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
278
|
+
exports.cuckooSizing = cuckooSizing;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
+
//#region src/cuckoo/cuckoo.d.ts
|
|
5
|
+
/** Thrown when an add cannot find room; the filter is left exactly as it was. */
|
|
6
|
+
export declare class CuckooFullError extends Error {
|
|
7
|
+
/** Discriminates this error from other `Error`s. */
|
|
8
|
+
override readonly name = "CuckooFullError";
|
|
9
|
+
}
|
|
10
|
+
/** Settings for a {@link CuckooFilter}. */
|
|
11
|
+
interface CuckooParams {
|
|
12
|
+
/** Expected number of keys. */
|
|
13
|
+
n: number;
|
|
14
|
+
/** Target false-positive rate at `n` keys, e.g. `0.01` for 1%. */
|
|
15
|
+
epsilon: number;
|
|
16
|
+
/** Hash seed; defaults to `0`. */
|
|
17
|
+
seed?: number;
|
|
18
|
+
}
|
|
19
|
+
/** The optional settings {@link CuckooFilter.create} takes after `n` and `epsilon`. */
|
|
20
|
+
type CuckooOptions = Omit<CuckooParams, "n" | "epsilon">;
|
|
21
|
+
/**
|
|
22
|
+
* A cuckoo filter: a set with a tunable false-positive rate, zero false
|
|
23
|
+
* negatives, and delete. Keys are stored as short fingerprints in one of two
|
|
24
|
+
* candidate buckets (Fan et al., "Cuckoo Filter: Practically Better Than
|
|
25
|
+
* Bloom", 2014).
|
|
26
|
+
*/
|
|
27
|
+
export declare class CuckooFilter {
|
|
28
|
+
#private;
|
|
29
|
+
/**
|
|
30
|
+
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
31
|
+
*
|
|
32
|
+
* @param n - Expected number of keys.
|
|
33
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
34
|
+
* @param options - Optional seed.
|
|
35
|
+
* @returns A new, empty filter.
|
|
36
|
+
*/
|
|
37
|
+
static create(n: number, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
38
|
+
/**
|
|
39
|
+
* Builds a filter from the given keys; duplicates are ignored. It is sized
|
|
40
|
+
* for the distinct keys at the target false-positive rate, and holds each
|
|
41
|
+
* once, so a key listed twice needs one delete.
|
|
42
|
+
*
|
|
43
|
+
* @param keys - The keys to insert.
|
|
44
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
45
|
+
* @param options - Optional seed.
|
|
46
|
+
* @returns A new filter containing every key.
|
|
47
|
+
*/
|
|
48
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
49
|
+
/**
|
|
50
|
+
* Restores a filter from its {@link CuckooFilter.toBytes} serialization.
|
|
51
|
+
*
|
|
52
|
+
* @param bytes - The serialized filter.
|
|
53
|
+
* @returns The reconstructed filter.
|
|
54
|
+
*/
|
|
55
|
+
static fromBytes(bytes: Uint8Array): CuckooFilter;
|
|
56
|
+
/**
|
|
57
|
+
* Restores a filter from its {@link CuckooFilter.toJSON} envelope.
|
|
58
|
+
*
|
|
59
|
+
* @param value - The parsed JSON envelope.
|
|
60
|
+
* @returns The reconstructed filter.
|
|
61
|
+
*/
|
|
62
|
+
static fromJSON(value: unknown): CuckooFilter;
|
|
63
|
+
/**
|
|
64
|
+
* Constructs a filter from {@link CuckooParams}.
|
|
65
|
+
*
|
|
66
|
+
* @throws {@link ParamError} if a setting is invalid.
|
|
67
|
+
*/
|
|
68
|
+
constructor({ n, epsilon, seed }: CuckooParams);
|
|
69
|
+
/**
|
|
70
|
+
* Adds `key`. Every call stores a fingerprint, including for a key already
|
|
71
|
+
* present, so each add needs its own delete.
|
|
72
|
+
*
|
|
73
|
+
* @throws {@link CuckooFullError} if the filter has no room for it.
|
|
74
|
+
*/
|
|
75
|
+
add(key: BytesLike): void;
|
|
76
|
+
/**
|
|
77
|
+
* Removes one copy of `key`: the first slot holding its fingerprint in
|
|
78
|
+
* either candidate bucket.
|
|
79
|
+
*
|
|
80
|
+
* Only delete keys you added. A key never added can share a fingerprint and
|
|
81
|
+
* bucket with one that was, and deleting it removes that key's fingerprint
|
|
82
|
+
* instead, a false negative the filter cannot detect: it stores fingerprints,
|
|
83
|
+
* not keys, so it cannot tell the two apart.
|
|
84
|
+
*
|
|
85
|
+
* @returns `true` if a matching fingerprint was removed; `false`, with the
|
|
86
|
+
* filter unchanged, if there was none.
|
|
87
|
+
*/
|
|
88
|
+
delete(key: BytesLike): boolean;
|
|
89
|
+
/**
|
|
90
|
+
* Tests whether `key` may be in the filter.
|
|
91
|
+
*
|
|
92
|
+
* @returns `false` if `key` is definitely absent; `true` if it was added
|
|
93
|
+
* (and not deleted) or on a false positive.
|
|
94
|
+
*/
|
|
95
|
+
has(key: BytesLike): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Estimates the current false-positive rate from the load: a query compares
|
|
98
|
+
* its fingerprint against about `8 * count / capacity` occupied slots across
|
|
99
|
+
* its two buckets, each matching with probability `1 / (2^f - 1)`.
|
|
100
|
+
*
|
|
101
|
+
* @returns The estimated false-positive rate, `0` for an empty filter.
|
|
102
|
+
*/
|
|
103
|
+
rate(): number;
|
|
104
|
+
/**
|
|
105
|
+
* Serializes the filter to a DSTL type 7 frame, readable by
|
|
106
|
+
* {@link CuckooFilter.fromBytes} and by any reader of the documented format.
|
|
107
|
+
*
|
|
108
|
+
* @returns The frame bytes.
|
|
109
|
+
*/
|
|
110
|
+
toBytes(): Uint8Array;
|
|
111
|
+
/**
|
|
112
|
+
* Whether `other` holds the same settings and slots, so the two serialize to
|
|
113
|
+
* identical bytes. Which slot a key lands in depends on insertion order, so
|
|
114
|
+
* filters holding the same keys can still differ.
|
|
115
|
+
*
|
|
116
|
+
* @param other - The filter to compare against.
|
|
117
|
+
* @returns `true` if the two are indistinguishable.
|
|
118
|
+
*/
|
|
119
|
+
equals(other: CuckooFilter): boolean;
|
|
120
|
+
/**
|
|
121
|
+
* Serializes the filter as a JSON envelope around its {@link CuckooFilter.toBytes} frame.
|
|
122
|
+
*
|
|
123
|
+
* @returns The JSON-safe envelope.
|
|
124
|
+
*/
|
|
125
|
+
toJSON(): FilterJSON;
|
|
126
|
+
/** Fingerprints stored, one per add not undone by a delete. */
|
|
127
|
+
get count(): number;
|
|
128
|
+
/** Slots in the table, `4 * buckets`. */
|
|
129
|
+
get capacity(): number;
|
|
130
|
+
/** Number of 4-slot buckets. */
|
|
131
|
+
get buckets(): number;
|
|
132
|
+
/** Width of each stored fingerprint, in bits. */
|
|
133
|
+
get fingerprintBits(): number;
|
|
134
|
+
/** Total slot bits, `fingerprintBits * capacity`. */
|
|
135
|
+
get m(): number;
|
|
136
|
+
/** Slot bits per expected key, `m / n`. */
|
|
137
|
+
get bitsPerKey(): number;
|
|
138
|
+
/** Hash seed. */
|
|
139
|
+
get seed(): number;
|
|
140
|
+
/** Target false-positive rate at `n` keys. */
|
|
141
|
+
get epsilon(): number;
|
|
142
|
+
}
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/cuckoo/sizing.d.ts
|
|
145
|
+
/** Cuckoo filter geometry: what a sizing solve determines. */
|
|
146
|
+
interface CuckooSizing {
|
|
147
|
+
/** Fingerprint width in bits. */
|
|
148
|
+
f: number;
|
|
149
|
+
/** Number of 4-slot buckets. */
|
|
150
|
+
buckets: number;
|
|
151
|
+
/** Slot bits per expected key, `4 * f * buckets / n`. */
|
|
152
|
+
bitsPerKey: number;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Cuckoo geometry for `n` keys at target FPR `epsilon`: `f` bits from the
|
|
156
|
+
* `2b / epsilon` bound with 4-slot buckets, and enough buckets for 95% load
|
|
157
|
+
* plus `sqrt(n)` slack. At exactly 95%, small tables fail to take `n` keys at
|
|
158
|
+
* about 0.6% of sizes (a few buckets collect more keys than they hold, which
|
|
159
|
+
* more kicks do not fix); the slack removed every failure in about 170k
|
|
160
|
+
* prototype runs and still leaves a million-key filter at 94.6% load.
|
|
161
|
+
*/
|
|
162
|
+
export declare function cuckooSizing(n: number, epsilon: number): CuckooSizing;
|
|
163
|
+
//#endregion
|
|
164
|
+
export { BadMagicError, ChecksumError, type CuckooOptions, type CuckooParams, type CuckooSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
+
//#region src/cuckoo/cuckoo.d.ts
|
|
5
|
+
/** Thrown when an add cannot find room; the filter is left exactly as it was. */
|
|
6
|
+
export declare class CuckooFullError extends Error {
|
|
7
|
+
/** Discriminates this error from other `Error`s. */
|
|
8
|
+
override readonly name = "CuckooFullError";
|
|
9
|
+
}
|
|
10
|
+
/** Settings for a {@link CuckooFilter}. */
|
|
11
|
+
interface CuckooParams {
|
|
12
|
+
/** Expected number of keys. */
|
|
13
|
+
n: number;
|
|
14
|
+
/** Target false-positive rate at `n` keys, e.g. `0.01` for 1%. */
|
|
15
|
+
epsilon: number;
|
|
16
|
+
/** Hash seed; defaults to `0`. */
|
|
17
|
+
seed?: number;
|
|
18
|
+
}
|
|
19
|
+
/** The optional settings {@link CuckooFilter.create} takes after `n` and `epsilon`. */
|
|
20
|
+
type CuckooOptions = Omit<CuckooParams, "n" | "epsilon">;
|
|
21
|
+
/**
|
|
22
|
+
* A cuckoo filter: a set with a tunable false-positive rate, zero false
|
|
23
|
+
* negatives, and delete. Keys are stored as short fingerprints in one of two
|
|
24
|
+
* candidate buckets (Fan et al., "Cuckoo Filter: Practically Better Than
|
|
25
|
+
* Bloom", 2014).
|
|
26
|
+
*/
|
|
27
|
+
export declare class CuckooFilter {
|
|
28
|
+
#private;
|
|
29
|
+
/**
|
|
30
|
+
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
31
|
+
*
|
|
32
|
+
* @param n - Expected number of keys.
|
|
33
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
34
|
+
* @param options - Optional seed.
|
|
35
|
+
* @returns A new, empty filter.
|
|
36
|
+
*/
|
|
37
|
+
static create(n: number, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
38
|
+
/**
|
|
39
|
+
* Builds a filter from the given keys; duplicates are ignored. It is sized
|
|
40
|
+
* for the distinct keys at the target false-positive rate, and holds each
|
|
41
|
+
* once, so a key listed twice needs one delete.
|
|
42
|
+
*
|
|
43
|
+
* @param keys - The keys to insert.
|
|
44
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
45
|
+
* @param options - Optional seed.
|
|
46
|
+
* @returns A new filter containing every key.
|
|
47
|
+
*/
|
|
48
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
49
|
+
/**
|
|
50
|
+
* Restores a filter from its {@link CuckooFilter.toBytes} serialization.
|
|
51
|
+
*
|
|
52
|
+
* @param bytes - The serialized filter.
|
|
53
|
+
* @returns The reconstructed filter.
|
|
54
|
+
*/
|
|
55
|
+
static fromBytes(bytes: Uint8Array): CuckooFilter;
|
|
56
|
+
/**
|
|
57
|
+
* Restores a filter from its {@link CuckooFilter.toJSON} envelope.
|
|
58
|
+
*
|
|
59
|
+
* @param value - The parsed JSON envelope.
|
|
60
|
+
* @returns The reconstructed filter.
|
|
61
|
+
*/
|
|
62
|
+
static fromJSON(value: unknown): CuckooFilter;
|
|
63
|
+
/**
|
|
64
|
+
* Constructs a filter from {@link CuckooParams}.
|
|
65
|
+
*
|
|
66
|
+
* @throws {@link ParamError} if a setting is invalid.
|
|
67
|
+
*/
|
|
68
|
+
constructor({ n, epsilon, seed }: CuckooParams);
|
|
69
|
+
/**
|
|
70
|
+
* Adds `key`. Every call stores a fingerprint, including for a key already
|
|
71
|
+
* present, so each add needs its own delete.
|
|
72
|
+
*
|
|
73
|
+
* @throws {@link CuckooFullError} if the filter has no room for it.
|
|
74
|
+
*/
|
|
75
|
+
add(key: BytesLike): void;
|
|
76
|
+
/**
|
|
77
|
+
* Removes one copy of `key`: the first slot holding its fingerprint in
|
|
78
|
+
* either candidate bucket.
|
|
79
|
+
*
|
|
80
|
+
* Only delete keys you added. A key never added can share a fingerprint and
|
|
81
|
+
* bucket with one that was, and deleting it removes that key's fingerprint
|
|
82
|
+
* instead, a false negative the filter cannot detect: it stores fingerprints,
|
|
83
|
+
* not keys, so it cannot tell the two apart.
|
|
84
|
+
*
|
|
85
|
+
* @returns `true` if a matching fingerprint was removed; `false`, with the
|
|
86
|
+
* filter unchanged, if there was none.
|
|
87
|
+
*/
|
|
88
|
+
delete(key: BytesLike): boolean;
|
|
89
|
+
/**
|
|
90
|
+
* Tests whether `key` may be in the filter.
|
|
91
|
+
*
|
|
92
|
+
* @returns `false` if `key` is definitely absent; `true` if it was added
|
|
93
|
+
* (and not deleted) or on a false positive.
|
|
94
|
+
*/
|
|
95
|
+
has(key: BytesLike): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Estimates the current false-positive rate from the load: a query compares
|
|
98
|
+
* its fingerprint against about `8 * count / capacity` occupied slots across
|
|
99
|
+
* its two buckets, each matching with probability `1 / (2^f - 1)`.
|
|
100
|
+
*
|
|
101
|
+
* @returns The estimated false-positive rate, `0` for an empty filter.
|
|
102
|
+
*/
|
|
103
|
+
rate(): number;
|
|
104
|
+
/**
|
|
105
|
+
* Serializes the filter to a DSTL type 7 frame, readable by
|
|
106
|
+
* {@link CuckooFilter.fromBytes} and by any reader of the documented format.
|
|
107
|
+
*
|
|
108
|
+
* @returns The frame bytes.
|
|
109
|
+
*/
|
|
110
|
+
toBytes(): Uint8Array;
|
|
111
|
+
/**
|
|
112
|
+
* Whether `other` holds the same settings and slots, so the two serialize to
|
|
113
|
+
* identical bytes. Which slot a key lands in depends on insertion order, so
|
|
114
|
+
* filters holding the same keys can still differ.
|
|
115
|
+
*
|
|
116
|
+
* @param other - The filter to compare against.
|
|
117
|
+
* @returns `true` if the two are indistinguishable.
|
|
118
|
+
*/
|
|
119
|
+
equals(other: CuckooFilter): boolean;
|
|
120
|
+
/**
|
|
121
|
+
* Serializes the filter as a JSON envelope around its {@link CuckooFilter.toBytes} frame.
|
|
122
|
+
*
|
|
123
|
+
* @returns The JSON-safe envelope.
|
|
124
|
+
*/
|
|
125
|
+
toJSON(): FilterJSON;
|
|
126
|
+
/** Fingerprints stored, one per add not undone by a delete. */
|
|
127
|
+
get count(): number;
|
|
128
|
+
/** Slots in the table, `4 * buckets`. */
|
|
129
|
+
get capacity(): number;
|
|
130
|
+
/** Number of 4-slot buckets. */
|
|
131
|
+
get buckets(): number;
|
|
132
|
+
/** Width of each stored fingerprint, in bits. */
|
|
133
|
+
get fingerprintBits(): number;
|
|
134
|
+
/** Total slot bits, `fingerprintBits * capacity`. */
|
|
135
|
+
get m(): number;
|
|
136
|
+
/** Slot bits per expected key, `m / n`. */
|
|
137
|
+
get bitsPerKey(): number;
|
|
138
|
+
/** Hash seed. */
|
|
139
|
+
get seed(): number;
|
|
140
|
+
/** Target false-positive rate at `n` keys. */
|
|
141
|
+
get epsilon(): number;
|
|
142
|
+
}
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/cuckoo/sizing.d.ts
|
|
145
|
+
/** Cuckoo filter geometry: what a sizing solve determines. */
|
|
146
|
+
interface CuckooSizing {
|
|
147
|
+
/** Fingerprint width in bits. */
|
|
148
|
+
f: number;
|
|
149
|
+
/** Number of 4-slot buckets. */
|
|
150
|
+
buckets: number;
|
|
151
|
+
/** Slot bits per expected key, `4 * f * buckets / n`. */
|
|
152
|
+
bitsPerKey: number;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Cuckoo geometry for `n` keys at target FPR `epsilon`: `f` bits from the
|
|
156
|
+
* `2b / epsilon` bound with 4-slot buckets, and enough buckets for 95% load
|
|
157
|
+
* plus `sqrt(n)` slack. At exactly 95%, small tables fail to take `n` keys at
|
|
158
|
+
* about 0.6% of sizes (a few buckets collect more keys than they hold, which
|
|
159
|
+
* more kicks do not fix); the slack removed every failure in about 170k
|
|
160
|
+
* prototype runs and still leaves a million-key filter at 94.6% load.
|
|
161
|
+
*/
|
|
162
|
+
export declare function cuckooSizing(n: number, epsilon: number): CuckooSizing;
|
|
163
|
+
//#endregion
|
|
164
|
+
export { BadMagicError, ChecksumError, type CuckooOptions, type CuckooParams, type CuckooSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { i as hash128KeyInto, u as reduce } from "../hasher-3Cyn7Yg-.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
|
+
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
+
function cuckooSizing(n, epsilon) {
|
|
5
|
+
assertProbability(epsilon, "epsilon");
|
|
6
|
+
const f = Math.max(4, Math.ceil(Math.log2(8 / epsilon)));
|
|
7
|
+
if (f > 32) throw new ParamError(`epsilon ${String(epsilon)} needs a ${String(f)}-bit fingerprint, above the maximum ${String(32)} bits`);
|
|
8
|
+
const buckets = Math.ceil(n / 3.8) + Math.ceil(Math.sqrt(n));
|
|
9
|
+
return {
|
|
10
|
+
f,
|
|
11
|
+
buckets,
|
|
12
|
+
bitsPerKey: 4 * f * buckets / n
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
//#endregion
|
|
16
|
+
//#region src/cuckoo/cuckoo.ts
|
|
17
|
+
const TYPE = 7;
|
|
18
|
+
const PARAMS_SIZE = 32;
|
|
19
|
+
const PARAMS_FIELDS_END = 28;
|
|
20
|
+
const padded8 = (length) => Math.ceil(length / 8) * 8;
|
|
21
|
+
const wordBytes = (words) => new Uint8Array(words.buffer, words.byteOffset, words.byteLength);
|
|
22
|
+
const SLOTS = 4;
|
|
23
|
+
const MAX_KICKS = 500;
|
|
24
|
+
const UNDO_SLOT = new Float64Array(MAX_KICKS);
|
|
25
|
+
const UNDO_FP = new Uint32Array(MAX_KICKS);
|
|
26
|
+
const HASH = {
|
|
27
|
+
w0: 0,
|
|
28
|
+
w1: 0,
|
|
29
|
+
w2: 0,
|
|
30
|
+
w3: 0
|
|
31
|
+
};
|
|
32
|
+
let restoring;
|
|
33
|
+
var CuckooFullError = class extends Error {
|
|
34
|
+
name = "CuckooFullError";
|
|
35
|
+
};
|
|
36
|
+
var CuckooFilter = class CuckooFilter {
|
|
37
|
+
#n;
|
|
38
|
+
#epsilon;
|
|
39
|
+
#seed;
|
|
40
|
+
#f;
|
|
41
|
+
#buckets;
|
|
42
|
+
#mask;
|
|
43
|
+
#words;
|
|
44
|
+
#count = 0;
|
|
45
|
+
#fp = 0;
|
|
46
|
+
#i1 = 0;
|
|
47
|
+
#i2 = 0;
|
|
48
|
+
static create(n, epsilon, options = {}) {
|
|
49
|
+
return new CuckooFilter({
|
|
50
|
+
...options,
|
|
51
|
+
n,
|
|
52
|
+
epsilon
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
static from(keys, epsilon, options = {}) {
|
|
56
|
+
const seen = new Set();
|
|
57
|
+
const distinct = [];
|
|
58
|
+
for (const key of keys) {
|
|
59
|
+
hash128KeyInto(key, options.seed ?? 0, HASH);
|
|
60
|
+
const id = `${String(HASH.w0)},${String(HASH.w1)},${String(HASH.w2)},${String(HASH.w3)}`;
|
|
61
|
+
if (seen.has(id)) continue;
|
|
62
|
+
seen.add(id);
|
|
63
|
+
distinct.push(key);
|
|
64
|
+
}
|
|
65
|
+
const f = CuckooFilter.create(Math.max(1, distinct.length), epsilon, options);
|
|
66
|
+
for (const k of distinct) f.add(k);
|
|
67
|
+
return f;
|
|
68
|
+
}
|
|
69
|
+
static fromBytes(bytes) {
|
|
70
|
+
const { type, flags, body } = readHeader(bytes);
|
|
71
|
+
if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
72
|
+
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
73
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "cuckoo");
|
|
74
|
+
assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "cuckoo");
|
|
75
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
76
|
+
const params = {
|
|
77
|
+
n: view.getUint32(0, true),
|
|
78
|
+
seed: view.getUint32(4, true),
|
|
79
|
+
epsilon: view.getFloat64(8, true)
|
|
80
|
+
};
|
|
81
|
+
let filter;
|
|
82
|
+
try {
|
|
83
|
+
const f = view.getUint32(16, true);
|
|
84
|
+
const buckets = view.getUint32(20, true);
|
|
85
|
+
if (f < 4 || f > 32) throw new SerializationError(`cuckoo: stored f ${String(f)} is outside 4..${String(32)}`);
|
|
86
|
+
if (buckets < 1) throw new SerializationError("cuckoo: stored buckets is 0");
|
|
87
|
+
const m = f * SLOTS * buckets;
|
|
88
|
+
if (m > 4294967295) throw new SerializationError(`cuckoo: stored geometry gives m ${String(m)} bits, more than 2^32 - 1`);
|
|
89
|
+
const words = Math.ceil(m / 32);
|
|
90
|
+
assertBodyLength(body.length, PARAMS_SIZE + padded8(4 * words), "cuckoo");
|
|
91
|
+
restoring = {
|
|
92
|
+
f,
|
|
93
|
+
buckets
|
|
94
|
+
};
|
|
95
|
+
try {
|
|
96
|
+
filter = new CuckooFilter(params);
|
|
97
|
+
} finally {
|
|
98
|
+
restoring = void 0;
|
|
99
|
+
}
|
|
100
|
+
} catch (err) {
|
|
101
|
+
if (err instanceof ParamError) throw new SerializationError(`cuckoo: ${err.message}`);
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
const words = filter.#words;
|
|
105
|
+
for (let w = 0; w < words.length; w++) words[w] = view.getUint32(PARAMS_SIZE + 4 * w, true);
|
|
106
|
+
const tail = filter.m % 32;
|
|
107
|
+
if (tail !== 0 && (words[words.length - 1] ?? 0) >>> tail !== 0) throw new SerializationError("cuckoo: bits set past the last slot");
|
|
108
|
+
if (body.subarray(PARAMS_SIZE + 4 * words.length).some((b) => b !== 0)) throw new SerializationError("cuckoo: slot padding is not zero");
|
|
109
|
+
const count = view.getUint32(24, true);
|
|
110
|
+
let occupied = 0;
|
|
111
|
+
for (let j = 0; j < filter.capacity; j++) if (filter.#slot(j) !== 0) occupied++;
|
|
112
|
+
if (count !== occupied) throw new SerializationError(`cuckoo: count ${String(count)} does not match ${String(occupied)} occupied slots`);
|
|
113
|
+
filter.#count = count;
|
|
114
|
+
return filter;
|
|
115
|
+
}
|
|
116
|
+
static fromJSON(value) {
|
|
117
|
+
return CuckooFilter.fromBytes(fromJSONEnvelope(value));
|
|
118
|
+
}
|
|
119
|
+
constructor({ n, epsilon, seed = 0 }) {
|
|
120
|
+
assertPositiveInt(n, "n");
|
|
121
|
+
assertUint32(n, "n");
|
|
122
|
+
assertProbability(epsilon, "epsilon");
|
|
123
|
+
assertUint32(seed, "seed");
|
|
124
|
+
const { f, buckets } = restoring ?? cuckooSizing(n, epsilon);
|
|
125
|
+
assertUint32(f * SLOTS * buckets, "m");
|
|
126
|
+
this.#n = n;
|
|
127
|
+
this.#epsilon = epsilon;
|
|
128
|
+
this.#seed = seed;
|
|
129
|
+
this.#f = f;
|
|
130
|
+
this.#buckets = buckets;
|
|
131
|
+
this.#mask = 2 ** f - 1;
|
|
132
|
+
this.#words = new Uint32Array(Math.ceil(f * SLOTS * buckets / 32));
|
|
133
|
+
}
|
|
134
|
+
add(key) {
|
|
135
|
+
this.#hash(key);
|
|
136
|
+
if (this.#replace(this.#i1, 0, this.#fp) || this.#replace(this.#i2, 0, this.#fp)) {
|
|
137
|
+
this.#count++;
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
let i = (HASH.w2 & 1) === 0 ? this.#i1 : this.#i2;
|
|
141
|
+
let x = (HASH.w3 | 1) >>> 0;
|
|
142
|
+
let fp = this.#fp;
|
|
143
|
+
for (let kick = 0; kick < MAX_KICKS; kick++) {
|
|
144
|
+
x ^= x << 13;
|
|
145
|
+
x >>>= 0;
|
|
146
|
+
x ^= x >>> 17;
|
|
147
|
+
x ^= x << 5;
|
|
148
|
+
x >>>= 0;
|
|
149
|
+
const j = i * SLOTS + (x & 3);
|
|
150
|
+
const victim = this.#slot(j);
|
|
151
|
+
UNDO_SLOT[kick] = j;
|
|
152
|
+
UNDO_FP[kick] = victim;
|
|
153
|
+
this.#setSlot(j, fp);
|
|
154
|
+
fp = victim;
|
|
155
|
+
i = this.#alt(i, fp);
|
|
156
|
+
if (this.#replace(i, 0, fp)) {
|
|
157
|
+
this.#count++;
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
for (let kick = 499; kick >= 0; kick--) this.#setSlot(UNDO_SLOT[kick] ?? 0, UNDO_FP[kick] ?? 0);
|
|
162
|
+
throw new CuckooFullError(`cuckoo filter is full at ${String(this.#count)} of ${String(this.capacity)} slots`);
|
|
163
|
+
}
|
|
164
|
+
delete(key) {
|
|
165
|
+
this.#hash(key);
|
|
166
|
+
const removed = this.#replace(this.#i1, this.#fp, 0) || this.#i2 !== this.#i1 && this.#replace(this.#i2, this.#fp, 0);
|
|
167
|
+
if (removed) this.#count--;
|
|
168
|
+
return removed;
|
|
169
|
+
}
|
|
170
|
+
has(key) {
|
|
171
|
+
this.#hash(key);
|
|
172
|
+
return this.#find(this.#i1, this.#fp) >= 0 || this.#find(this.#i2, this.#fp) >= 0;
|
|
173
|
+
}
|
|
174
|
+
rate() {
|
|
175
|
+
const occupied = 8 * this.#count / this.capacity;
|
|
176
|
+
return 1 - (1 - 1 / this.#mask) ** occupied;
|
|
177
|
+
}
|
|
178
|
+
toBytes() {
|
|
179
|
+
const words = this.#words;
|
|
180
|
+
return writeFrame({
|
|
181
|
+
version: 5,
|
|
182
|
+
type: TYPE,
|
|
183
|
+
flags: 0
|
|
184
|
+
}, PARAMS_SIZE, padded8(4 * words.length), (_, view) => {
|
|
185
|
+
view.setUint32(0, this.#n, true);
|
|
186
|
+
view.setUint32(4, this.#seed, true);
|
|
187
|
+
view.setFloat64(8, this.#epsilon, true);
|
|
188
|
+
view.setUint32(16, this.#f, true);
|
|
189
|
+
view.setUint32(20, this.#buckets, true);
|
|
190
|
+
view.setUint32(24, this.#count, true);
|
|
191
|
+
words.forEach((word, w) => {
|
|
192
|
+
view.setUint32(PARAMS_SIZE + 4 * w, word, true);
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
equals(other) {
|
|
197
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
198
|
+
}
|
|
199
|
+
toJSON() {
|
|
200
|
+
return toJSONEnvelope(this.toBytes());
|
|
201
|
+
}
|
|
202
|
+
#hash(key) {
|
|
203
|
+
hash128KeyInto(key, this.#seed, HASH);
|
|
204
|
+
this.#fp = HASH.w1 >>> 32 - this.#f || 1;
|
|
205
|
+
this.#i1 = reduce(HASH.w0, this.#buckets);
|
|
206
|
+
this.#i2 = this.#alt(this.#i1, this.#fp);
|
|
207
|
+
}
|
|
208
|
+
#alt(i, fp) {
|
|
209
|
+
const b = this.#buckets;
|
|
210
|
+
return ((Math.imul(fp, 1540483477) >>> 0) % b + b - i) % b;
|
|
211
|
+
}
|
|
212
|
+
#find(bucket, value) {
|
|
213
|
+
for (let j = bucket * SLOTS; j < bucket * SLOTS + SLOTS; j++) if (this.#slot(j) === value) return j;
|
|
214
|
+
return -1;
|
|
215
|
+
}
|
|
216
|
+
#replace(bucket, from, to) {
|
|
217
|
+
const j = this.#find(bucket, from);
|
|
218
|
+
if (j < 0) return false;
|
|
219
|
+
this.#setSlot(j, to);
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
#slot(j) {
|
|
223
|
+
const bit = j * this.#f;
|
|
224
|
+
const w = bit >>> 5;
|
|
225
|
+
const off = bit & 31;
|
|
226
|
+
let v = (this.#words[w] ?? 0) >>> off;
|
|
227
|
+
if (off + this.#f > 32) v |= (this.#words[w + 1] ?? 0) << 32 - off;
|
|
228
|
+
return (v & this.#mask) >>> 0;
|
|
229
|
+
}
|
|
230
|
+
#setSlot(j, fp) {
|
|
231
|
+
const bit = j * this.#f;
|
|
232
|
+
const w = bit >>> 5;
|
|
233
|
+
const off = bit & 31;
|
|
234
|
+
const words = this.#words;
|
|
235
|
+
words[w] = (words[w] ?? 0) & ~(this.#mask << off) | fp << off;
|
|
236
|
+
if (off + this.#f > 32) {
|
|
237
|
+
const shift = 32 - off;
|
|
238
|
+
words[w + 1] = (words[w + 1] ?? 0) & ~(this.#mask >>> shift) | fp >>> shift;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
get count() {
|
|
242
|
+
return this.#count;
|
|
243
|
+
}
|
|
244
|
+
get capacity() {
|
|
245
|
+
return SLOTS * this.#buckets;
|
|
246
|
+
}
|
|
247
|
+
get buckets() {
|
|
248
|
+
return this.#buckets;
|
|
249
|
+
}
|
|
250
|
+
get fingerprintBits() {
|
|
251
|
+
return this.#f;
|
|
252
|
+
}
|
|
253
|
+
get m() {
|
|
254
|
+
return this.#f * this.capacity;
|
|
255
|
+
}
|
|
256
|
+
get bitsPerKey() {
|
|
257
|
+
return this.m / this.#n;
|
|
258
|
+
}
|
|
259
|
+
get seed() {
|
|
260
|
+
return this.#seed;
|
|
261
|
+
}
|
|
262
|
+
get epsilon() {
|
|
263
|
+
return this.#epsilon;
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
//#endregion
|
|
267
|
+
export { BadMagicError, ChecksumError, CuckooFilter, CuckooFullError, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, cuckooSizing };
|
package/dist/index.cjs
CHANGED
package/dist/index.js
CHANGED
package/dist/scalable/index.cjs
CHANGED
|
@@ -156,8 +156,11 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
156
156
|
}
|
|
157
157
|
#adopt(stages) {
|
|
158
158
|
this.#stages.length = 0;
|
|
159
|
-
|
|
160
|
-
const
|
|
159
|
+
let k = 0;
|
|
160
|
+
for (const s of stages) {
|
|
161
|
+
this.#stages.push(s);
|
|
162
|
+
if (s.k > k) k = s.k;
|
|
163
|
+
}
|
|
161
164
|
if (k > this.#probes.length) this.#probes = new Uint32Array(k);
|
|
162
165
|
return stages.reduce((_, s) => s);
|
|
163
166
|
}
|
|
@@ -201,13 +204,6 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
201
204
|
}
|
|
202
205
|
union(other) {
|
|
203
206
|
this.#assertMergeable(other);
|
|
204
|
-
const result = new ScalableBloomFilter({
|
|
205
|
-
n: this.#n,
|
|
206
|
-
epsilon: this.#epsilon,
|
|
207
|
-
growth: this.#growth,
|
|
208
|
-
tightening: this.#tightening,
|
|
209
|
-
seed: this.#seed
|
|
210
|
-
});
|
|
211
207
|
const [longer, shorter] = this.#stages.length >= other.#stages.length ? [this.#stages, other.#stages] : [other.#stages, this.#stages];
|
|
212
208
|
const stages = longer.map((stage, i) => {
|
|
213
209
|
const bits = new require_bitset.BitSet(stage.m);
|
|
@@ -226,8 +222,13 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
226
222
|
count: Math.min(stage.capacity, stage.count + (twin?.count ?? 0))
|
|
227
223
|
};
|
|
228
224
|
});
|
|
229
|
-
|
|
230
|
-
|
|
225
|
+
return ScalableBloomFilter.#withStages({
|
|
226
|
+
n: this.#n,
|
|
227
|
+
epsilon: this.#epsilon,
|
|
228
|
+
growth: this.#growth,
|
|
229
|
+
tightening: this.#tightening,
|
|
230
|
+
seed: this.#seed
|
|
231
|
+
}, stages);
|
|
231
232
|
}
|
|
232
233
|
equals(other) {
|
|
233
234
|
if (this.#settingPairs(other).some(([, a, b]) => a !== b)) return false;
|
|
@@ -252,18 +253,24 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
252
253
|
const entries = readTable(view, stageCount);
|
|
253
254
|
const expected = entries.reduce((sum, e) => sum + padded8(Math.ceil(e.m / 8)), tableEnd);
|
|
254
255
|
require_serialize.assertBodyLength(body.length, expected, "scalable");
|
|
255
|
-
|
|
256
|
+
const stages = readStages(body, entries, tableEnd);
|
|
256
257
|
try {
|
|
257
|
-
return
|
|
258
|
+
return ScalableBloomFilter.#withStages({
|
|
258
259
|
n: view.getUint32(0, true),
|
|
259
260
|
seed: view.getUint32(4, true),
|
|
260
261
|
epsilon: view.getFloat64(8, true),
|
|
261
262
|
growth: view.getFloat64(16, true),
|
|
262
263
|
tightening: view.getFloat64(24, true)
|
|
263
|
-
});
|
|
264
|
+
}, stages);
|
|
264
265
|
} catch (err) {
|
|
265
266
|
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`scalable: ${err.message}`);
|
|
266
267
|
throw err;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
static #withStages(params, stages) {
|
|
271
|
+
restoring = stages;
|
|
272
|
+
try {
|
|
273
|
+
return new ScalableBloomFilter(params);
|
|
267
274
|
} finally {
|
|
268
275
|
restoring = void 0;
|
|
269
276
|
}
|
package/dist/scalable/index.js
CHANGED
|
@@ -155,8 +155,11 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
155
155
|
}
|
|
156
156
|
#adopt(stages) {
|
|
157
157
|
this.#stages.length = 0;
|
|
158
|
-
|
|
159
|
-
const
|
|
158
|
+
let k = 0;
|
|
159
|
+
for (const s of stages) {
|
|
160
|
+
this.#stages.push(s);
|
|
161
|
+
if (s.k > k) k = s.k;
|
|
162
|
+
}
|
|
160
163
|
if (k > this.#probes.length) this.#probes = new Uint32Array(k);
|
|
161
164
|
return stages.reduce((_, s) => s);
|
|
162
165
|
}
|
|
@@ -200,13 +203,6 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
200
203
|
}
|
|
201
204
|
union(other) {
|
|
202
205
|
this.#assertMergeable(other);
|
|
203
|
-
const result = new ScalableBloomFilter({
|
|
204
|
-
n: this.#n,
|
|
205
|
-
epsilon: this.#epsilon,
|
|
206
|
-
growth: this.#growth,
|
|
207
|
-
tightening: this.#tightening,
|
|
208
|
-
seed: this.#seed
|
|
209
|
-
});
|
|
210
206
|
const [longer, shorter] = this.#stages.length >= other.#stages.length ? [this.#stages, other.#stages] : [other.#stages, this.#stages];
|
|
211
207
|
const stages = longer.map((stage, i) => {
|
|
212
208
|
const bits = new BitSet(stage.m);
|
|
@@ -225,8 +221,13 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
225
221
|
count: Math.min(stage.capacity, stage.count + (twin?.count ?? 0))
|
|
226
222
|
};
|
|
227
223
|
});
|
|
228
|
-
|
|
229
|
-
|
|
224
|
+
return ScalableBloomFilter.#withStages({
|
|
225
|
+
n: this.#n,
|
|
226
|
+
epsilon: this.#epsilon,
|
|
227
|
+
growth: this.#growth,
|
|
228
|
+
tightening: this.#tightening,
|
|
229
|
+
seed: this.#seed
|
|
230
|
+
}, stages);
|
|
230
231
|
}
|
|
231
232
|
equals(other) {
|
|
232
233
|
if (this.#settingPairs(other).some(([, a, b]) => a !== b)) return false;
|
|
@@ -251,18 +252,24 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
251
252
|
const entries = readTable(view, stageCount);
|
|
252
253
|
const expected = entries.reduce((sum, e) => sum + padded8(Math.ceil(e.m / 8)), tableEnd);
|
|
253
254
|
assertBodyLength(body.length, expected, "scalable");
|
|
254
|
-
|
|
255
|
+
const stages = readStages(body, entries, tableEnd);
|
|
255
256
|
try {
|
|
256
|
-
return
|
|
257
|
+
return ScalableBloomFilter.#withStages({
|
|
257
258
|
n: view.getUint32(0, true),
|
|
258
259
|
seed: view.getUint32(4, true),
|
|
259
260
|
epsilon: view.getFloat64(8, true),
|
|
260
261
|
growth: view.getFloat64(16, true),
|
|
261
262
|
tightening: view.getFloat64(24, true)
|
|
262
|
-
});
|
|
263
|
+
}, stages);
|
|
263
264
|
} catch (err) {
|
|
264
265
|
if (err instanceof ParamError) throw new SerializationError(`scalable: ${err.message}`);
|
|
265
266
|
throw err;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
static #withStages(params, stages) {
|
|
270
|
+
restoring = stages;
|
|
271
|
+
try {
|
|
272
|
+
return new ScalableBloomFilter(params);
|
|
266
273
|
} finally {
|
|
267
274
|
restoring = void 0;
|
|
268
275
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "distillate",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"description": "Probabilistic data structures for JavaScript. Space-efficient, approximate-membership filters (Bloom, Blocked Bloom, Binary Fuse) with tunable error and a portable binary format; zero dependencies, universal.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bloom-filter",
|
|
@@ -79,6 +79,14 @@
|
|
|
79
79
|
"import": "./dist/scalable/index.js",
|
|
80
80
|
"require": "./dist/scalable/index.cjs"
|
|
81
81
|
},
|
|
82
|
+
"./cuckoo": {
|
|
83
|
+
"types": {
|
|
84
|
+
"import": "./dist/cuckoo/index.d.ts",
|
|
85
|
+
"require": "./dist/cuckoo/index.d.cts"
|
|
86
|
+
},
|
|
87
|
+
"import": "./dist/cuckoo/index.js",
|
|
88
|
+
"require": "./dist/cuckoo/index.cjs"
|
|
89
|
+
},
|
|
82
90
|
"./package.json": "./package.json"
|
|
83
91
|
},
|
|
84
92
|
"main": "./dist/index.cjs",
|