distillate 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -3
- package/dist/bloom/index.cjs +1 -1
- package/dist/bloom/index.d.cts +1 -1
- package/dist/bloom/index.d.ts +1 -1
- package/dist/bloom/index.js +1 -1
- package/dist/countmin/index.cjs +194 -0
- package/dist/countmin/index.d.cts +178 -0
- package/dist/countmin/index.d.ts +178 -0
- package/dist/countmin/index.js +182 -0
- package/dist/cuckoo/index.cjs +278 -0
- package/dist/cuckoo/index.d.cts +164 -0
- package/dist/cuckoo/index.d.ts +164 -0
- package/dist/cuckoo/index.js +267 -0
- package/dist/hll/index.cjs +1 -1
- package/dist/hll/index.d.cts +1 -1
- package/dist/hll/index.d.ts +1 -1
- package/dist/hll/index.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +22 -15
- package/dist/scalable/index.js +22 -15
- package/dist/{sizing-wYOW27eY.d.cts → sizing-BmKfHf_V.d.cts} +13 -1
- package/dist/{sizing-wYOW27eY.d.ts → sizing-BmKfHf_V.d.ts} +13 -1
- package/dist/{sizing-BtUrtvF_.js → sizing-D3-_7pdP.js} +9 -1
- package/dist/{sizing-BLzZk2zr.cjs → sizing-R1pwbzjC.cjs} +14 -0
- package/package.json +17 -1
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { a as hash32x2Into, s as probeAt } from "../hasher-3Cyn7Yg-.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
|
+
import { o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
+
import { n as countMinSizing } from "../sizing-D3-_7pdP.js";
|
|
5
|
+
//#region src/countmin/countmin.ts
|
|
6
|
+
const TYPE = 8;
|
|
7
|
+
const PARAMS_SIZE = 16;
|
|
8
|
+
const PARAMS_FIELDS_END = 12;
|
|
9
|
+
const counterBytes = (counters) => new Uint8Array(counters.buffer, counters.byteOffset, counters.byteLength);
|
|
10
|
+
function rowSum(counters, width, depth) {
|
|
11
|
+
let expected = 0;
|
|
12
|
+
for (let r = 0; r < depth; r++) {
|
|
13
|
+
let sum = 0;
|
|
14
|
+
const from = r * width;
|
|
15
|
+
for (let c = from; c < from + width; c++) sum += counters[c] ?? 0;
|
|
16
|
+
if (sum > Number.MAX_SAFE_INTEGER) throw new SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, past the safe integer range`);
|
|
17
|
+
if (r === 0) expected = sum;
|
|
18
|
+
else if (sum !== expected) throw new SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, but row 0 sums to ${String(expected)}`);
|
|
19
|
+
}
|
|
20
|
+
return expected;
|
|
21
|
+
}
|
|
22
|
+
var CountMinParamMismatchError = class extends Error {
|
|
23
|
+
name = "CountMinParamMismatchError";
|
|
24
|
+
};
|
|
25
|
+
var CountMinOverflowError = class extends RangeError {
|
|
26
|
+
name = "CountMinOverflowError";
|
|
27
|
+
};
|
|
28
|
+
var CountMinSketch = class CountMinSketch {
|
|
29
|
+
#counters;
|
|
30
|
+
#width;
|
|
31
|
+
#depth;
|
|
32
|
+
#seed;
|
|
33
|
+
#words = new Uint32Array(2);
|
|
34
|
+
#positions;
|
|
35
|
+
#total = 0;
|
|
36
|
+
static create(epsilon, delta, options = {}) {
|
|
37
|
+
return new CountMinSketch({
|
|
38
|
+
...options,
|
|
39
|
+
...countMinSizing(epsilon, delta)
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
static from(keys, epsilon, delta, options = {}) {
|
|
43
|
+
const sketch = CountMinSketch.create(epsilon, delta, options);
|
|
44
|
+
for (const key of keys) sketch.add(key);
|
|
45
|
+
return sketch;
|
|
46
|
+
}
|
|
47
|
+
static fromBytes(bytes) {
|
|
48
|
+
const { type, flags, body } = readHeader(bytes);
|
|
49
|
+
if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
50
|
+
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
51
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "countmin");
|
|
52
|
+
assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "countmin");
|
|
53
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
54
|
+
const width = view.getUint32(0, true);
|
|
55
|
+
const depth = view.getUint32(4, true);
|
|
56
|
+
const seed = view.getUint32(8, true);
|
|
57
|
+
assertBodyLength(body.length, PARAMS_SIZE + 4 * width * depth, "countmin");
|
|
58
|
+
let sketch;
|
|
59
|
+
try {
|
|
60
|
+
sketch = new CountMinSketch({
|
|
61
|
+
width,
|
|
62
|
+
depth,
|
|
63
|
+
seed
|
|
64
|
+
});
|
|
65
|
+
} catch (err) {
|
|
66
|
+
if (err instanceof ParamError) throw new SerializationError(`countmin: ${err.message}`);
|
|
67
|
+
throw err;
|
|
68
|
+
}
|
|
69
|
+
const counters = sketch.#counters;
|
|
70
|
+
for (let i = 0; i < counters.length; i++) counters[i] = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
71
|
+
sketch.#total = rowSum(counters, width, depth);
|
|
72
|
+
return sketch;
|
|
73
|
+
}
|
|
74
|
+
constructor({ width, depth, seed = 0 }) {
|
|
75
|
+
assertPositiveInt(width, "width");
|
|
76
|
+
assertUint32(width, "width");
|
|
77
|
+
assertPositiveInt(depth, "depth");
|
|
78
|
+
assertUint32(depth, "depth");
|
|
79
|
+
assertUint32(seed, "seed");
|
|
80
|
+
if (width * depth > 4294967295) throw new ParamError(`width ${String(width)} by depth ${String(depth)} needs ${String(width * depth)} counters, above the maximum ${String(4294967295)}`);
|
|
81
|
+
this.#counters = new Uint32Array(width * depth);
|
|
82
|
+
this.#positions = new Uint32Array(depth);
|
|
83
|
+
this.#width = width;
|
|
84
|
+
this.#depth = depth;
|
|
85
|
+
this.#seed = seed;
|
|
86
|
+
}
|
|
87
|
+
get width() {
|
|
88
|
+
return this.#width;
|
|
89
|
+
}
|
|
90
|
+
get depth() {
|
|
91
|
+
return this.#depth;
|
|
92
|
+
}
|
|
93
|
+
get seed() {
|
|
94
|
+
return this.#seed;
|
|
95
|
+
}
|
|
96
|
+
get total() {
|
|
97
|
+
return this.#total;
|
|
98
|
+
}
|
|
99
|
+
get epsilon() {
|
|
100
|
+
return Math.E / this.#width;
|
|
101
|
+
}
|
|
102
|
+
get delta() {
|
|
103
|
+
return Math.exp(-this.#depth);
|
|
104
|
+
}
|
|
105
|
+
error() {
|
|
106
|
+
return this.epsilon * this.#total;
|
|
107
|
+
}
|
|
108
|
+
#positionAt(row) {
|
|
109
|
+
return row * this.#width + probeAt(this.#words[0] ?? 0, this.#words[1] ?? 0, row, this.#width);
|
|
110
|
+
}
|
|
111
|
+
add(key, count = 1) {
|
|
112
|
+
assertPositiveInt(count, "count");
|
|
113
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
114
|
+
hash32x2Into(key, this.#seed, this.#words);
|
|
115
|
+
const headroom = 4294967295 - count;
|
|
116
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
117
|
+
const at = this.#positionAt(r);
|
|
118
|
+
if ((this.#counters[at] ?? 0) > headroom) throw new CountMinOverflowError(`adding ${String(count)} would carry a counter past ${String(4294967295)}`);
|
|
119
|
+
this.#positions[r] = at;
|
|
120
|
+
}
|
|
121
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
122
|
+
const at = this.#positions[r] ?? 0;
|
|
123
|
+
this.#counters[at] = (this.#counters[at] ?? 0) + count;
|
|
124
|
+
}
|
|
125
|
+
this.#total += count;
|
|
126
|
+
}
|
|
127
|
+
union(other) {
|
|
128
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) throw new CountMinParamMismatchError("cannot union Count-Min sketches whose parameters do not match");
|
|
129
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
130
|
+
const mine = this.#counters;
|
|
131
|
+
const theirs = other.#counters;
|
|
132
|
+
const merged = new CountMinSketch({
|
|
133
|
+
width: this.#width,
|
|
134
|
+
depth: this.#depth,
|
|
135
|
+
seed: this.#seed
|
|
136
|
+
});
|
|
137
|
+
const into = merged.#counters;
|
|
138
|
+
for (let i = 0; i < into.length; i++) {
|
|
139
|
+
const sum = (mine[i] ?? 0) + (theirs[i] ?? 0);
|
|
140
|
+
if (sum > 4294967295) throw new CountMinOverflowError(`union would carry a counter to ${String(sum)}, past ${String(4294967295)}`);
|
|
141
|
+
into[i] = sum;
|
|
142
|
+
}
|
|
143
|
+
merged.#total = this.#total + other.#total;
|
|
144
|
+
return merged;
|
|
145
|
+
}
|
|
146
|
+
toBytes() {
|
|
147
|
+
const counters = this.#counters;
|
|
148
|
+
return writeFrame({
|
|
149
|
+
version: 5,
|
|
150
|
+
type: TYPE,
|
|
151
|
+
flags: 0
|
|
152
|
+
}, PARAMS_SIZE, 4 * counters.length, (_, view) => {
|
|
153
|
+
view.setUint32(0, this.#width, true);
|
|
154
|
+
view.setUint32(4, this.#depth, true);
|
|
155
|
+
view.setUint32(8, this.#seed, true);
|
|
156
|
+
counters.forEach((c, i) => {
|
|
157
|
+
view.setUint32(PARAMS_SIZE + 4 * i, c, true);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
equals(other) {
|
|
162
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) return false;
|
|
163
|
+
return bytesEqual(counterBytes(this.#counters), counterBytes(other.#counters));
|
|
164
|
+
}
|
|
165
|
+
toJSON() {
|
|
166
|
+
return toJSONEnvelope(this.toBytes());
|
|
167
|
+
}
|
|
168
|
+
static fromJSON(value) {
|
|
169
|
+
return CountMinSketch.fromBytes(fromJSONEnvelope(value));
|
|
170
|
+
}
|
|
171
|
+
count(key) {
|
|
172
|
+
hash32x2Into(key, this.#seed, this.#words);
|
|
173
|
+
let min = Infinity;
|
|
174
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
175
|
+
const at = this.#counters[this.#positionAt(r)] ?? 0;
|
|
176
|
+
if (at < min) min = at;
|
|
177
|
+
}
|
|
178
|
+
return min;
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
//#endregion
|
|
182
|
+
export { BadMagicError, ChecksumError, CountMinOverflowError, CountMinParamMismatchError, CountMinSketch, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, countMinSizing };
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
function cuckooSizing(n, epsilon) {
|
|
6
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
7
|
+
const f = Math.max(4, Math.ceil(Math.log2(8 / epsilon)));
|
|
8
|
+
if (f > 32) throw new require_params.ParamError(`epsilon ${String(epsilon)} needs a ${String(f)}-bit fingerprint, above the maximum ${String(32)} bits`);
|
|
9
|
+
const buckets = Math.ceil(n / 3.8) + Math.ceil(Math.sqrt(n));
|
|
10
|
+
return {
|
|
11
|
+
f,
|
|
12
|
+
buckets,
|
|
13
|
+
bitsPerKey: 4 * f * buckets / n
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/cuckoo/cuckoo.ts
|
|
18
|
+
const TYPE = 7;
|
|
19
|
+
const PARAMS_SIZE = 32;
|
|
20
|
+
const PARAMS_FIELDS_END = 28;
|
|
21
|
+
const padded8 = (length) => Math.ceil(length / 8) * 8;
|
|
22
|
+
const wordBytes = (words) => new Uint8Array(words.buffer, words.byteOffset, words.byteLength);
|
|
23
|
+
const SLOTS = 4;
|
|
24
|
+
const MAX_KICKS = 500;
|
|
25
|
+
const UNDO_SLOT = new Float64Array(MAX_KICKS);
|
|
26
|
+
const UNDO_FP = new Uint32Array(MAX_KICKS);
|
|
27
|
+
const HASH = {
|
|
28
|
+
w0: 0,
|
|
29
|
+
w1: 0,
|
|
30
|
+
w2: 0,
|
|
31
|
+
w3: 0
|
|
32
|
+
};
|
|
33
|
+
let restoring;
|
|
34
|
+
var CuckooFullError = class extends Error {
|
|
35
|
+
name = "CuckooFullError";
|
|
36
|
+
};
|
|
37
|
+
var CuckooFilter = class CuckooFilter {
|
|
38
|
+
#n;
|
|
39
|
+
#epsilon;
|
|
40
|
+
#seed;
|
|
41
|
+
#f;
|
|
42
|
+
#buckets;
|
|
43
|
+
#mask;
|
|
44
|
+
#words;
|
|
45
|
+
#count = 0;
|
|
46
|
+
#fp = 0;
|
|
47
|
+
#i1 = 0;
|
|
48
|
+
#i2 = 0;
|
|
49
|
+
static create(n, epsilon, options = {}) {
|
|
50
|
+
return new CuckooFilter({
|
|
51
|
+
...options,
|
|
52
|
+
n,
|
|
53
|
+
epsilon
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
static from(keys, epsilon, options = {}) {
|
|
57
|
+
const seen = new Set();
|
|
58
|
+
const distinct = [];
|
|
59
|
+
for (const key of keys) {
|
|
60
|
+
require_hasher.hash128KeyInto(key, options.seed ?? 0, HASH);
|
|
61
|
+
const id = `${String(HASH.w0)},${String(HASH.w1)},${String(HASH.w2)},${String(HASH.w3)}`;
|
|
62
|
+
if (seen.has(id)) continue;
|
|
63
|
+
seen.add(id);
|
|
64
|
+
distinct.push(key);
|
|
65
|
+
}
|
|
66
|
+
const f = CuckooFilter.create(Math.max(1, distinct.length), epsilon, options);
|
|
67
|
+
for (const k of distinct) f.add(k);
|
|
68
|
+
return f;
|
|
69
|
+
}
|
|
70
|
+
static fromBytes(bytes) {
|
|
71
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
72
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
73
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
74
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "cuckoo");
|
|
75
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "cuckoo");
|
|
76
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
77
|
+
const params = {
|
|
78
|
+
n: view.getUint32(0, true),
|
|
79
|
+
seed: view.getUint32(4, true),
|
|
80
|
+
epsilon: view.getFloat64(8, true)
|
|
81
|
+
};
|
|
82
|
+
let filter;
|
|
83
|
+
try {
|
|
84
|
+
const f = view.getUint32(16, true);
|
|
85
|
+
const buckets = view.getUint32(20, true);
|
|
86
|
+
if (f < 4 || f > 32) throw new require_serialize.SerializationError(`cuckoo: stored f ${String(f)} is outside 4..${String(32)}`);
|
|
87
|
+
if (buckets < 1) throw new require_serialize.SerializationError("cuckoo: stored buckets is 0");
|
|
88
|
+
const m = f * SLOTS * buckets;
|
|
89
|
+
if (m > 4294967295) throw new require_serialize.SerializationError(`cuckoo: stored geometry gives m ${String(m)} bits, more than 2^32 - 1`);
|
|
90
|
+
const words = Math.ceil(m / 32);
|
|
91
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + padded8(4 * words), "cuckoo");
|
|
92
|
+
restoring = {
|
|
93
|
+
f,
|
|
94
|
+
buckets
|
|
95
|
+
};
|
|
96
|
+
try {
|
|
97
|
+
filter = new CuckooFilter(params);
|
|
98
|
+
} finally {
|
|
99
|
+
restoring = void 0;
|
|
100
|
+
}
|
|
101
|
+
} catch (err) {
|
|
102
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`cuckoo: ${err.message}`);
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
const words = filter.#words;
|
|
106
|
+
for (let w = 0; w < words.length; w++) words[w] = view.getUint32(PARAMS_SIZE + 4 * w, true);
|
|
107
|
+
const tail = filter.m % 32;
|
|
108
|
+
if (tail !== 0 && (words[words.length - 1] ?? 0) >>> tail !== 0) throw new require_serialize.SerializationError("cuckoo: bits set past the last slot");
|
|
109
|
+
if (body.subarray(PARAMS_SIZE + 4 * words.length).some((b) => b !== 0)) throw new require_serialize.SerializationError("cuckoo: slot padding is not zero");
|
|
110
|
+
const count = view.getUint32(24, true);
|
|
111
|
+
let occupied = 0;
|
|
112
|
+
for (let j = 0; j < filter.capacity; j++) if (filter.#slot(j) !== 0) occupied++;
|
|
113
|
+
if (count !== occupied) throw new require_serialize.SerializationError(`cuckoo: count ${String(count)} does not match ${String(occupied)} occupied slots`);
|
|
114
|
+
filter.#count = count;
|
|
115
|
+
return filter;
|
|
116
|
+
}
|
|
117
|
+
static fromJSON(value) {
|
|
118
|
+
return CuckooFilter.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
119
|
+
}
|
|
120
|
+
constructor({ n, epsilon, seed = 0 }) {
|
|
121
|
+
require_params.assertPositiveInt(n, "n");
|
|
122
|
+
require_params.assertUint32(n, "n");
|
|
123
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
124
|
+
require_params.assertUint32(seed, "seed");
|
|
125
|
+
const { f, buckets } = restoring ?? cuckooSizing(n, epsilon);
|
|
126
|
+
require_params.assertUint32(f * SLOTS * buckets, "m");
|
|
127
|
+
this.#n = n;
|
|
128
|
+
this.#epsilon = epsilon;
|
|
129
|
+
this.#seed = seed;
|
|
130
|
+
this.#f = f;
|
|
131
|
+
this.#buckets = buckets;
|
|
132
|
+
this.#mask = 2 ** f - 1;
|
|
133
|
+
this.#words = new Uint32Array(Math.ceil(f * SLOTS * buckets / 32));
|
|
134
|
+
}
|
|
135
|
+
add(key) {
|
|
136
|
+
this.#hash(key);
|
|
137
|
+
if (this.#replace(this.#i1, 0, this.#fp) || this.#replace(this.#i2, 0, this.#fp)) {
|
|
138
|
+
this.#count++;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
let i = (HASH.w2 & 1) === 0 ? this.#i1 : this.#i2;
|
|
142
|
+
let x = (HASH.w3 | 1) >>> 0;
|
|
143
|
+
let fp = this.#fp;
|
|
144
|
+
for (let kick = 0; kick < MAX_KICKS; kick++) {
|
|
145
|
+
x ^= x << 13;
|
|
146
|
+
x >>>= 0;
|
|
147
|
+
x ^= x >>> 17;
|
|
148
|
+
x ^= x << 5;
|
|
149
|
+
x >>>= 0;
|
|
150
|
+
const j = i * SLOTS + (x & 3);
|
|
151
|
+
const victim = this.#slot(j);
|
|
152
|
+
UNDO_SLOT[kick] = j;
|
|
153
|
+
UNDO_FP[kick] = victim;
|
|
154
|
+
this.#setSlot(j, fp);
|
|
155
|
+
fp = victim;
|
|
156
|
+
i = this.#alt(i, fp);
|
|
157
|
+
if (this.#replace(i, 0, fp)) {
|
|
158
|
+
this.#count++;
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
for (let kick = 499; kick >= 0; kick--) this.#setSlot(UNDO_SLOT[kick] ?? 0, UNDO_FP[kick] ?? 0);
|
|
163
|
+
throw new CuckooFullError(`cuckoo filter is full at ${String(this.#count)} of ${String(this.capacity)} slots`);
|
|
164
|
+
}
|
|
165
|
+
delete(key) {
|
|
166
|
+
this.#hash(key);
|
|
167
|
+
const removed = this.#replace(this.#i1, this.#fp, 0) || this.#i2 !== this.#i1 && this.#replace(this.#i2, this.#fp, 0);
|
|
168
|
+
if (removed) this.#count--;
|
|
169
|
+
return removed;
|
|
170
|
+
}
|
|
171
|
+
has(key) {
|
|
172
|
+
this.#hash(key);
|
|
173
|
+
return this.#find(this.#i1, this.#fp) >= 0 || this.#find(this.#i2, this.#fp) >= 0;
|
|
174
|
+
}
|
|
175
|
+
rate() {
|
|
176
|
+
const occupied = 8 * this.#count / this.capacity;
|
|
177
|
+
return 1 - (1 - 1 / this.#mask) ** occupied;
|
|
178
|
+
}
|
|
179
|
+
toBytes() {
|
|
180
|
+
const words = this.#words;
|
|
181
|
+
return require_serialize.writeFrame({
|
|
182
|
+
version: 5,
|
|
183
|
+
type: TYPE,
|
|
184
|
+
flags: 0
|
|
185
|
+
}, PARAMS_SIZE, padded8(4 * words.length), (_, view) => {
|
|
186
|
+
view.setUint32(0, this.#n, true);
|
|
187
|
+
view.setUint32(4, this.#seed, true);
|
|
188
|
+
view.setFloat64(8, this.#epsilon, true);
|
|
189
|
+
view.setUint32(16, this.#f, true);
|
|
190
|
+
view.setUint32(20, this.#buckets, true);
|
|
191
|
+
view.setUint32(24, this.#count, true);
|
|
192
|
+
words.forEach((word, w) => {
|
|
193
|
+
view.setUint32(PARAMS_SIZE + 4 * w, word, true);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
equals(other) {
|
|
198
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && this.#f === other.#f && this.#buckets === other.#buckets && require_serialize.bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
199
|
+
}
|
|
200
|
+
toJSON() {
|
|
201
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
202
|
+
}
|
|
203
|
+
#hash(key) {
|
|
204
|
+
require_hasher.hash128KeyInto(key, this.#seed, HASH);
|
|
205
|
+
this.#fp = HASH.w1 >>> 32 - this.#f || 1;
|
|
206
|
+
this.#i1 = require_hasher.reduce(HASH.w0, this.#buckets);
|
|
207
|
+
this.#i2 = this.#alt(this.#i1, this.#fp);
|
|
208
|
+
}
|
|
209
|
+
#alt(i, fp) {
|
|
210
|
+
const b = this.#buckets;
|
|
211
|
+
return ((Math.imul(fp, 1540483477) >>> 0) % b + b - i) % b;
|
|
212
|
+
}
|
|
213
|
+
#find(bucket, value) {
|
|
214
|
+
for (let j = bucket * SLOTS; j < bucket * SLOTS + SLOTS; j++) if (this.#slot(j) === value) return j;
|
|
215
|
+
return -1;
|
|
216
|
+
}
|
|
217
|
+
#replace(bucket, from, to) {
|
|
218
|
+
const j = this.#find(bucket, from);
|
|
219
|
+
if (j < 0) return false;
|
|
220
|
+
this.#setSlot(j, to);
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
#slot(j) {
|
|
224
|
+
const bit = j * this.#f;
|
|
225
|
+
const w = bit >>> 5;
|
|
226
|
+
const off = bit & 31;
|
|
227
|
+
let v = (this.#words[w] ?? 0) >>> off;
|
|
228
|
+
if (off + this.#f > 32) v |= (this.#words[w + 1] ?? 0) << 32 - off;
|
|
229
|
+
return (v & this.#mask) >>> 0;
|
|
230
|
+
}
|
|
231
|
+
#setSlot(j, fp) {
|
|
232
|
+
const bit = j * this.#f;
|
|
233
|
+
const w = bit >>> 5;
|
|
234
|
+
const off = bit & 31;
|
|
235
|
+
const words = this.#words;
|
|
236
|
+
words[w] = (words[w] ?? 0) & ~(this.#mask << off) | fp << off;
|
|
237
|
+
if (off + this.#f > 32) {
|
|
238
|
+
const shift = 32 - off;
|
|
239
|
+
words[w + 1] = (words[w + 1] ?? 0) & ~(this.#mask >>> shift) | fp >>> shift;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
get count() {
|
|
243
|
+
return this.#count;
|
|
244
|
+
}
|
|
245
|
+
get capacity() {
|
|
246
|
+
return SLOTS * this.#buckets;
|
|
247
|
+
}
|
|
248
|
+
get buckets() {
|
|
249
|
+
return this.#buckets;
|
|
250
|
+
}
|
|
251
|
+
get fingerprintBits() {
|
|
252
|
+
return this.#f;
|
|
253
|
+
}
|
|
254
|
+
get m() {
|
|
255
|
+
return this.#f * this.capacity;
|
|
256
|
+
}
|
|
257
|
+
get bitsPerKey() {
|
|
258
|
+
return this.m / this.#n;
|
|
259
|
+
}
|
|
260
|
+
get seed() {
|
|
261
|
+
return this.#seed;
|
|
262
|
+
}
|
|
263
|
+
get epsilon() {
|
|
264
|
+
return this.#epsilon;
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
//#endregion
|
|
268
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
269
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
270
|
+
exports.CuckooFilter = CuckooFilter;
|
|
271
|
+
exports.CuckooFullError = CuckooFullError;
|
|
272
|
+
exports.ParamError = require_params.ParamError;
|
|
273
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
274
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
275
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
276
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
277
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
278
|
+
exports.cuckooSizing = cuckooSizing;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
+
//#region src/cuckoo/cuckoo.d.ts
|
|
5
|
+
/** Thrown when an add cannot find room; the filter is left exactly as it was. */
|
|
6
|
+
export declare class CuckooFullError extends Error {
|
|
7
|
+
/** Discriminates this error from other `Error`s. */
|
|
8
|
+
override readonly name = "CuckooFullError";
|
|
9
|
+
}
|
|
10
|
+
/** Settings for a {@link CuckooFilter}. */
|
|
11
|
+
interface CuckooParams {
|
|
12
|
+
/** Expected number of keys. */
|
|
13
|
+
n: number;
|
|
14
|
+
/** Target false-positive rate at `n` keys, e.g. `0.01` for 1%. */
|
|
15
|
+
epsilon: number;
|
|
16
|
+
/** Hash seed; defaults to `0`. */
|
|
17
|
+
seed?: number;
|
|
18
|
+
}
|
|
19
|
+
/** The optional settings {@link CuckooFilter.create} takes after `n` and `epsilon`. */
|
|
20
|
+
type CuckooOptions = Omit<CuckooParams, "n" | "epsilon">;
|
|
21
|
+
/**
|
|
22
|
+
* A cuckoo filter: a set with a tunable false-positive rate, zero false
|
|
23
|
+
* negatives, and delete. Keys are stored as short fingerprints in one of two
|
|
24
|
+
* candidate buckets (Fan et al., "Cuckoo Filter: Practically Better Than
|
|
25
|
+
* Bloom", 2014).
|
|
26
|
+
*/
|
|
27
|
+
export declare class CuckooFilter {
|
|
28
|
+
#private;
|
|
29
|
+
/**
|
|
30
|
+
* Creates a filter sized for `n` expected keys at a target false-positive rate.
|
|
31
|
+
*
|
|
32
|
+
* @param n - Expected number of keys.
|
|
33
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
34
|
+
* @param options - Optional seed.
|
|
35
|
+
* @returns A new, empty filter.
|
|
36
|
+
*/
|
|
37
|
+
static create(n: number, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
38
|
+
/**
|
|
39
|
+
* Builds a filter from the given keys; duplicates are ignored. It is sized
|
|
40
|
+
* for the distinct keys at the target false-positive rate, and holds each
|
|
41
|
+
* once, so a key listed twice needs one delete.
|
|
42
|
+
*
|
|
43
|
+
* @param keys - The keys to insert.
|
|
44
|
+
* @param epsilon - Target false-positive rate, e.g. `0.01` for 1%.
|
|
45
|
+
* @param options - Optional seed.
|
|
46
|
+
* @returns A new filter containing every key.
|
|
47
|
+
*/
|
|
48
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, options?: CuckooOptions): CuckooFilter;
|
|
49
|
+
/**
|
|
50
|
+
* Restores a filter from its {@link CuckooFilter.toBytes} serialization.
|
|
51
|
+
*
|
|
52
|
+
* @param bytes - The serialized filter.
|
|
53
|
+
* @returns The reconstructed filter.
|
|
54
|
+
*/
|
|
55
|
+
static fromBytes(bytes: Uint8Array): CuckooFilter;
|
|
56
|
+
/**
|
|
57
|
+
* Restores a filter from its {@link CuckooFilter.toJSON} envelope.
|
|
58
|
+
*
|
|
59
|
+
* @param value - The parsed JSON envelope.
|
|
60
|
+
* @returns The reconstructed filter.
|
|
61
|
+
*/
|
|
62
|
+
static fromJSON(value: unknown): CuckooFilter;
|
|
63
|
+
/**
|
|
64
|
+
* Constructs a filter from {@link CuckooParams}.
|
|
65
|
+
*
|
|
66
|
+
* @throws {@link ParamError} if a setting is invalid.
|
|
67
|
+
*/
|
|
68
|
+
constructor({ n, epsilon, seed }: CuckooParams);
|
|
69
|
+
/**
|
|
70
|
+
* Adds `key`. Every call stores a fingerprint, including for a key already
|
|
71
|
+
* present, so each add needs its own delete.
|
|
72
|
+
*
|
|
73
|
+
* @throws {@link CuckooFullError} if the filter has no room for it.
|
|
74
|
+
*/
|
|
75
|
+
add(key: BytesLike): void;
|
|
76
|
+
/**
|
|
77
|
+
* Removes one copy of `key`: the first slot holding its fingerprint in
|
|
78
|
+
* either candidate bucket.
|
|
79
|
+
*
|
|
80
|
+
* Only delete keys you added. A key never added can share a fingerprint and
|
|
81
|
+
* bucket with one that was, and deleting it removes that key's fingerprint
|
|
82
|
+
* instead, a false negative the filter cannot detect: it stores fingerprints,
|
|
83
|
+
* not keys, so it cannot tell the two apart.
|
|
84
|
+
*
|
|
85
|
+
* @returns `true` if a matching fingerprint was removed; `false`, with the
|
|
86
|
+
* filter unchanged, if there was none.
|
|
87
|
+
*/
|
|
88
|
+
delete(key: BytesLike): boolean;
|
|
89
|
+
/**
|
|
90
|
+
* Tests whether `key` may be in the filter.
|
|
91
|
+
*
|
|
92
|
+
* @returns `false` if `key` is definitely absent; `true` if it was added
|
|
93
|
+
* (and not deleted) or on a false positive.
|
|
94
|
+
*/
|
|
95
|
+
has(key: BytesLike): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Estimates the current false-positive rate from the load: a query compares
|
|
98
|
+
* its fingerprint against about `8 * count / capacity` occupied slots across
|
|
99
|
+
* its two buckets, each matching with probability `1 / (2^f - 1)`.
|
|
100
|
+
*
|
|
101
|
+
* @returns The estimated false-positive rate, `0` for an empty filter.
|
|
102
|
+
*/
|
|
103
|
+
rate(): number;
|
|
104
|
+
/**
|
|
105
|
+
* Serializes the filter to a DSTL type 7 frame, readable by
|
|
106
|
+
* {@link CuckooFilter.fromBytes} and by any reader of the documented format.
|
|
107
|
+
*
|
|
108
|
+
* @returns The frame bytes.
|
|
109
|
+
*/
|
|
110
|
+
toBytes(): Uint8Array;
|
|
111
|
+
/**
|
|
112
|
+
* Whether `other` holds the same settings and slots, so the two serialize to
|
|
113
|
+
* identical bytes. Which slot a key lands in depends on insertion order, so
|
|
114
|
+
* filters holding the same keys can still differ.
|
|
115
|
+
*
|
|
116
|
+
* @param other - The filter to compare against.
|
|
117
|
+
* @returns `true` if the two are indistinguishable.
|
|
118
|
+
*/
|
|
119
|
+
equals(other: CuckooFilter): boolean;
|
|
120
|
+
/**
|
|
121
|
+
* Serializes the filter as a JSON envelope around its {@link CuckooFilter.toBytes} frame.
|
|
122
|
+
*
|
|
123
|
+
* @returns The JSON-safe envelope.
|
|
124
|
+
*/
|
|
125
|
+
toJSON(): FilterJSON;
|
|
126
|
+
/** Fingerprints stored, one per add not undone by a delete. */
|
|
127
|
+
get count(): number;
|
|
128
|
+
/** Slots in the table, `4 * buckets`. */
|
|
129
|
+
get capacity(): number;
|
|
130
|
+
/** Number of 4-slot buckets. */
|
|
131
|
+
get buckets(): number;
|
|
132
|
+
/** Width of each stored fingerprint, in bits. */
|
|
133
|
+
get fingerprintBits(): number;
|
|
134
|
+
/** Total slot bits, `fingerprintBits * capacity`. */
|
|
135
|
+
get m(): number;
|
|
136
|
+
/** Slot bits per expected key, `m / n`. */
|
|
137
|
+
get bitsPerKey(): number;
|
|
138
|
+
/** Hash seed. */
|
|
139
|
+
get seed(): number;
|
|
140
|
+
/** Target false-positive rate at `n` keys. */
|
|
141
|
+
get epsilon(): number;
|
|
142
|
+
}
|
|
143
|
+
//#endregion
|
|
144
|
+
//#region src/cuckoo/sizing.d.ts
|
|
145
|
+
/** Cuckoo filter geometry: what a sizing solve determines. */
|
|
146
|
+
interface CuckooSizing {
|
|
147
|
+
/** Fingerprint width in bits. */
|
|
148
|
+
f: number;
|
|
149
|
+
/** Number of 4-slot buckets. */
|
|
150
|
+
buckets: number;
|
|
151
|
+
/** Slot bits per expected key, `4 * f * buckets / n`. */
|
|
152
|
+
bitsPerKey: number;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Cuckoo geometry for `n` keys at target FPR `epsilon`: `f` bits from the
|
|
156
|
+
* `2b / epsilon` bound with 4-slot buckets, and enough buckets for 95% load
|
|
157
|
+
* plus `sqrt(n)` slack. At exactly 95%, small tables fail to take `n` keys at
|
|
158
|
+
* about 0.6% of sizes (a few buckets collect more keys than they hold, which
|
|
159
|
+
* more kicks do not fix); the slack removed every failure in about 170k
|
|
160
|
+
* prototype runs and still leaves a million-key filter at 94.6% load.
|
|
161
|
+
*/
|
|
162
|
+
export declare function cuckooSizing(n: number, epsilon: number): CuckooSizing;
|
|
163
|
+
//#endregion
|
|
164
|
+
export { BadMagicError, ChecksumError, type CuckooOptions, type CuckooParams, type CuckooSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|