distillate 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -12
- package/dist/blocked/index.cjs +1 -1
- package/dist/blocked/index.js +1 -1
- package/dist/bloom/index.cjs +3 -4
- package/dist/bloom/index.d.cts +12 -2
- package/dist/bloom/index.d.ts +12 -2
- package/dist/bloom/index.js +2 -3
- package/dist/bytes-Be7tq1F7.js +9 -0
- package/dist/bytes-D2duG1jq.cjs +14 -0
- package/dist/countmin/index.cjs +13 -4
- package/dist/countmin/index.d.cts +15 -2
- package/dist/countmin/index.d.ts +15 -2
- package/dist/countmin/index.js +12 -3
- package/dist/cuckoo/index.cjs +20 -9
- package/dist/cuckoo/index.js +20 -9
- package/dist/distinct-BhxDB8mW.js +61 -0
- package/dist/distinct-Ck5PKuCS.cjs +66 -0
- package/dist/fuse/index.cjs +6 -11
- package/dist/fuse/index.js +6 -11
- package/dist/{hasher-3Cyn7Yg-.js → hasher-DOdmZ-LI.js} +1 -8
- package/dist/{hasher-MhY7aQ8F.cjs → hasher-DPXmP7SH.cjs} +2 -9
- package/dist/hll/index.cjs +10 -4
- package/dist/hll/index.d.cts +13 -2
- package/dist/hll/index.d.ts +13 -2
- package/dist/hll/index.js +9 -3
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +5 -6
- package/dist/scalable/index.js +2 -3
- package/dist/{bitset-QYGqxnEf.cjs → sizing-Ch0SDdza.cjs} +15 -0
- package/dist/{bitset-Cf2jwRJX.js → sizing-zxCBlyhO.js} +10 -1
- package/dist/topk/index.cjs +423 -0
- package/dist/topk/index.d.cts +207 -0
- package/dist/topk/index.d.ts +207 -0
- package/dist/topk/index.js +411 -0
- package/package.json +15 -7
- package/dist/sizing-BmKfHf_V.d.cts +0 -34
- package/dist/sizing-BmKfHf_V.d.ts +0 -34
- package/dist/sizing-D3-_7pdP.js +0 -25
- package/dist/sizing-R1pwbzjC.cjs +0 -42
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
import { t as normalize } from "../bytes-Be7tq1F7.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
|
+
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
+
//#region src/topk/halfsip.ts
|
|
5
|
+
function halfSipHash13(bytes, k0, k1) {
|
|
6
|
+
let v0 = k0 | 0;
|
|
7
|
+
let v1 = k1 | 0;
|
|
8
|
+
let v2 = 1819895653 ^ k0 | 0;
|
|
9
|
+
let v3 = 1952801890 ^ k1 | 0;
|
|
10
|
+
const len = bytes.length;
|
|
11
|
+
const end = len & -4;
|
|
12
|
+
for (let i = 0; i < end; i += 4) {
|
|
13
|
+
const m = (bytes[i] ?? 0) | (bytes[i + 1] ?? 0) << 8 | (bytes[i + 2] ?? 0) << 16 | (bytes[i + 3] ?? 0) << 24;
|
|
14
|
+
v3 ^= m;
|
|
15
|
+
v0 = v0 + v1 | 0;
|
|
16
|
+
v1 = v1 << 5 | v1 >>> 27;
|
|
17
|
+
v1 ^= v0;
|
|
18
|
+
v0 = v0 << 16 | v0 >>> 16;
|
|
19
|
+
v2 = v2 + v3 | 0;
|
|
20
|
+
v3 = v3 << 8 | v3 >>> 24;
|
|
21
|
+
v3 ^= v2;
|
|
22
|
+
v0 = v0 + v3 | 0;
|
|
23
|
+
v3 = v3 << 7 | v3 >>> 25;
|
|
24
|
+
v3 ^= v0;
|
|
25
|
+
v2 = v2 + v1 | 0;
|
|
26
|
+
v1 = v1 << 13 | v1 >>> 19;
|
|
27
|
+
v1 ^= v2;
|
|
28
|
+
v2 = v2 << 16 | v2 >>> 16;
|
|
29
|
+
v0 ^= m;
|
|
30
|
+
}
|
|
31
|
+
let b = len << 24;
|
|
32
|
+
for (let i = len - 1; i >= end; i--) b |= (bytes[i] ?? 0) << 8 * (i - end);
|
|
33
|
+
v3 ^= b;
|
|
34
|
+
for (let r = 0; r < 4; r++) {
|
|
35
|
+
if (r === 1) {
|
|
36
|
+
v0 ^= b;
|
|
37
|
+
v2 ^= 255;
|
|
38
|
+
}
|
|
39
|
+
v0 = v0 + v1 | 0;
|
|
40
|
+
v1 = v1 << 5 | v1 >>> 27;
|
|
41
|
+
v1 ^= v0;
|
|
42
|
+
v0 = v0 << 16 | v0 >>> 16;
|
|
43
|
+
v2 = v2 + v3 | 0;
|
|
44
|
+
v3 = v3 << 8 | v3 >>> 24;
|
|
45
|
+
v3 ^= v2;
|
|
46
|
+
v0 = v0 + v3 | 0;
|
|
47
|
+
v3 = v3 << 7 | v3 >>> 25;
|
|
48
|
+
v3 ^= v0;
|
|
49
|
+
v2 = v2 + v1 | 0;
|
|
50
|
+
v1 = v1 << 13 | v1 >>> 19;
|
|
51
|
+
v1 ^= v2;
|
|
52
|
+
v2 = v2 << 16 | v2 >>> 16;
|
|
53
|
+
}
|
|
54
|
+
return (v1 ^ v3) >>> 0;
|
|
55
|
+
}
|
|
56
|
+
const TOPK_MAX_CAPACITY = 2 ** 24;
|
|
57
|
+
function topKLoadLimit(capacity) {
|
|
58
|
+
return Math.floor(.75 * capacity);
|
|
59
|
+
}
|
|
60
|
+
function topKPurgeWidth(capacity) {
|
|
61
|
+
return topKLoadLimit(capacity) + 2 >> 1;
|
|
62
|
+
}
|
|
63
|
+
function topKSizing(epsilon) {
|
|
64
|
+
assertProbability(epsilon, "epsilon");
|
|
65
|
+
for (let capacity = 4; capacity <= TOPK_MAX_CAPACITY; capacity *= 2) if (1 / topKPurgeWidth(capacity) <= epsilon) return { capacity };
|
|
66
|
+
throw new ParamError(`epsilon ${String(epsilon)} needs more than the maximum capacity ${String(TOPK_MAX_CAPACITY)}`);
|
|
67
|
+
}
|
|
68
|
+
//#endregion
|
|
69
|
+
//#region src/topk/topk.ts
|
|
70
|
+
const TYPE = 9;
|
|
71
|
+
const TABLE_START = 8;
|
|
72
|
+
const PARAMS_SIZE = 32;
|
|
73
|
+
const PARAMS_FIELDS_END = 24;
|
|
74
|
+
function assertCount(value, label) {
|
|
75
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new SerializationError(`topk: ${label} ${String(value)} is not an integer in [0, ${String(Number.MAX_SAFE_INTEGER)}]`);
|
|
76
|
+
}
|
|
77
|
+
function compareBytes(a, b) {
|
|
78
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
79
|
+
const diff = (a[i] ?? 0) - (b[i] ?? 0);
|
|
80
|
+
if (diff !== 0) return diff;
|
|
81
|
+
}
|
|
82
|
+
return a.length - b.length;
|
|
83
|
+
}
|
|
84
|
+
var TopKOverflowError = class extends RangeError {
|
|
85
|
+
name = "TopKOverflowError";
|
|
86
|
+
};
|
|
87
|
+
var TopKParamMismatchError = class extends Error {
|
|
88
|
+
name = "TopKParamMismatchError";
|
|
89
|
+
};
|
|
90
|
+
var TopK = class TopK {
|
|
91
|
+
#capacity;
|
|
92
|
+
#loadLimit;
|
|
93
|
+
#k0 = 0;
|
|
94
|
+
#k1 = 0;
|
|
95
|
+
#keyed = false;
|
|
96
|
+
#size = 0;
|
|
97
|
+
#counts = new Uint32Array(0);
|
|
98
|
+
#keyOffsets = new Uint32Array(0);
|
|
99
|
+
#keyLengths = new Uint32Array(0);
|
|
100
|
+
#arena;
|
|
101
|
+
#arenaLen = 0;
|
|
102
|
+
#entries = 0;
|
|
103
|
+
#offset = 0;
|
|
104
|
+
#total = 0;
|
|
105
|
+
static create(epsilon) {
|
|
106
|
+
return new TopK(topKSizing(epsilon));
|
|
107
|
+
}
|
|
108
|
+
static from(keys, epsilon) {
|
|
109
|
+
const sketch = TopK.create(epsilon);
|
|
110
|
+
for (const key of keys) sketch.add(key);
|
|
111
|
+
return sketch;
|
|
112
|
+
}
|
|
113
|
+
static fromBytes(bytes) {
|
|
114
|
+
const { type, flags, body } = readHeader(bytes);
|
|
115
|
+
if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
116
|
+
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
117
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "topk");
|
|
118
|
+
assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "topk");
|
|
119
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
120
|
+
const capacity = view.getUint32(0, true);
|
|
121
|
+
const entries = view.getUint32(4, true);
|
|
122
|
+
if (entries > topKLoadLimit(capacity)) throw new SerializationError(`topk: frame holds ${String(entries)} entries, above the load limit ${String(topKLoadLimit(capacity))} for capacity ${String(capacity)}`);
|
|
123
|
+
const tableEnd = PARAMS_SIZE + 8 * entries;
|
|
124
|
+
assertMinBodyLength(body.length, tableEnd, "topk");
|
|
125
|
+
let keyBytes = 0;
|
|
126
|
+
for (let i = 0; i < entries; i++) keyBytes += view.getUint32(PARAMS_SIZE + 4 * (entries + i), true);
|
|
127
|
+
assertBodyLength(body.length, tableEnd + keyBytes, "topk");
|
|
128
|
+
const offset = view.getFloat64(8, true);
|
|
129
|
+
const total = view.getFloat64(16, true);
|
|
130
|
+
assertCount(offset, "offset");
|
|
131
|
+
assertCount(total, "total");
|
|
132
|
+
let sketch;
|
|
133
|
+
try {
|
|
134
|
+
sketch = new TopK({ capacity });
|
|
135
|
+
sketch.#allocate(sketch.#fit(entries));
|
|
136
|
+
} catch (err) {
|
|
137
|
+
if (err instanceof ParamError) throw new SerializationError(`topk: ${err.message}`);
|
|
138
|
+
throw err;
|
|
139
|
+
}
|
|
140
|
+
let keyAt = tableEnd;
|
|
141
|
+
let stored = 0;
|
|
142
|
+
let prevCount = Infinity;
|
|
143
|
+
let prevKey = new Uint8Array(0);
|
|
144
|
+
for (let i = 0; i < entries; i++) {
|
|
145
|
+
const count = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
146
|
+
const len = view.getUint32(PARAMS_SIZE + 4 * (entries + i), true);
|
|
147
|
+
const key = body.subarray(keyAt, keyAt + len);
|
|
148
|
+
keyAt += len;
|
|
149
|
+
if (count === 0) throw new SerializationError(`topk: entry ${String(i)} has count 0`);
|
|
150
|
+
if (count > prevCount || count === prevCount && compareBytes(prevKey, key) >= 0) throw new SerializationError(`topk: entry ${String(i)} is out of canonical order`);
|
|
151
|
+
prevCount = count;
|
|
152
|
+
prevKey = key;
|
|
153
|
+
const slot = sketch.#slotFor(key);
|
|
154
|
+
if ((sketch.#counts[slot] ?? 0) !== 0) throw new SerializationError(`topk: entry ${String(i)} repeats a key already held`);
|
|
155
|
+
sketch.#storeKey(slot, key);
|
|
156
|
+
sketch.#counts[slot] = count;
|
|
157
|
+
stored += count;
|
|
158
|
+
}
|
|
159
|
+
const width = topKPurgeWidth(capacity);
|
|
160
|
+
if (stored + width * offset > total) throw new SerializationError(`topk: stored counts ${String(stored)} plus ${String(width)} x offset ${String(offset)} exceed total ${String(total)}`);
|
|
161
|
+
sketch.#entries = entries;
|
|
162
|
+
sketch.#offset = offset;
|
|
163
|
+
sketch.#total = total;
|
|
164
|
+
return sketch;
|
|
165
|
+
}
|
|
166
|
+
constructor({ capacity }) {
|
|
167
|
+
assertUint32(capacity, "capacity");
|
|
168
|
+
if (!Number.isInteger(Math.log2(capacity))) throw new ParamError(`capacity must be a power of two, got ${String(capacity)}`);
|
|
169
|
+
if (capacity < 4 || capacity > TOPK_MAX_CAPACITY) throw new ParamError(`capacity must be in [${String(4)}, ${String(TOPK_MAX_CAPACITY)}], got ${String(capacity)}`);
|
|
170
|
+
this.#capacity = capacity;
|
|
171
|
+
this.#loadLimit = topKLoadLimit(capacity);
|
|
172
|
+
this.#allocate(Math.min(TABLE_START, capacity));
|
|
173
|
+
this.#arena = new Uint8Array(256);
|
|
174
|
+
}
|
|
175
|
+
get capacity() {
|
|
176
|
+
return this.#capacity;
|
|
177
|
+
}
|
|
178
|
+
get epsilon() {
|
|
179
|
+
return 1 / topKPurgeWidth(this.#capacity);
|
|
180
|
+
}
|
|
181
|
+
get total() {
|
|
182
|
+
return this.#total;
|
|
183
|
+
}
|
|
184
|
+
error() {
|
|
185
|
+
return this.#offset;
|
|
186
|
+
}
|
|
187
|
+
#purge() {
|
|
188
|
+
const keys = [];
|
|
189
|
+
const counts = [];
|
|
190
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
191
|
+
const stored = this.#counts[slot] ?? 0;
|
|
192
|
+
if (stored === 0) continue;
|
|
193
|
+
keys.push(this.#keyAt(slot).slice());
|
|
194
|
+
counts.push(stored);
|
|
195
|
+
}
|
|
196
|
+
this.#reduceTo(keys, counts);
|
|
197
|
+
}
|
|
198
|
+
#reduceTo(keys, counts) {
|
|
199
|
+
while (keys.length > this.#loadLimit) {
|
|
200
|
+
const median = [...counts].sort((a, b) => a - b)[counts.length >> 1] ?? 0;
|
|
201
|
+
let kept = 0;
|
|
202
|
+
for (let i = 0; i < keys.length; i++) {
|
|
203
|
+
const survived = (counts[i] ?? 0) - median;
|
|
204
|
+
if (survived <= 0) continue;
|
|
205
|
+
keys[kept] = keys[i] ?? new Uint8Array(0);
|
|
206
|
+
counts[kept] = survived;
|
|
207
|
+
kept++;
|
|
208
|
+
}
|
|
209
|
+
keys.length = kept;
|
|
210
|
+
counts.length = kept;
|
|
211
|
+
this.#offset += median;
|
|
212
|
+
}
|
|
213
|
+
const size = this.#fit(keys.length);
|
|
214
|
+
if (size > this.#size) this.#allocate(size);
|
|
215
|
+
else this.#counts.fill(0);
|
|
216
|
+
this.#arenaLen = 0;
|
|
217
|
+
this.#entries = keys.length;
|
|
218
|
+
for (let i = 0; i < keys.length; i++) {
|
|
219
|
+
const bytes = keys[i] ?? new Uint8Array(0);
|
|
220
|
+
const slot = this.#slotFor(bytes);
|
|
221
|
+
this.#storeKey(slot, bytes);
|
|
222
|
+
this.#counts[slot] = counts[i] ?? 0;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
#allocate(size) {
|
|
226
|
+
this.#size = size;
|
|
227
|
+
this.#counts = new Uint32Array(size);
|
|
228
|
+
this.#keyOffsets = new Uint32Array(size);
|
|
229
|
+
this.#keyLengths = new Uint32Array(size);
|
|
230
|
+
}
|
|
231
|
+
#resize(size) {
|
|
232
|
+
const counts = this.#counts;
|
|
233
|
+
const offsets = this.#keyOffsets;
|
|
234
|
+
const lengths = this.#keyLengths;
|
|
235
|
+
this.#allocate(size);
|
|
236
|
+
for (let old = 0; old < counts.length; old++) {
|
|
237
|
+
const stored = counts[old] ?? 0;
|
|
238
|
+
if (stored === 0) continue;
|
|
239
|
+
const at = offsets[old] ?? 0;
|
|
240
|
+
const len = lengths[old] ?? 0;
|
|
241
|
+
const slot = this.#slotFor(this.#arena.subarray(at, at + len));
|
|
242
|
+
this.#counts[slot] = stored;
|
|
243
|
+
this.#keyOffsets[slot] = at;
|
|
244
|
+
this.#keyLengths[slot] = len;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
#fit(entries) {
|
|
248
|
+
let size = Math.min(TABLE_START, this.#capacity);
|
|
249
|
+
while (size < this.#capacity && topKLoadLimit(size) < entries) size *= 2;
|
|
250
|
+
return size;
|
|
251
|
+
}
|
|
252
|
+
#slotHolds(slot, bytes) {
|
|
253
|
+
const len = this.#keyLengths[slot] ?? 0;
|
|
254
|
+
if (len !== bytes.length) return false;
|
|
255
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
256
|
+
for (let i = 0; i < len; i++) if (this.#arena[at + i] !== bytes[i]) return false;
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
#storeKey(slot, bytes) {
|
|
260
|
+
let arena = this.#arena;
|
|
261
|
+
if (this.#arenaLen + bytes.length > arena.length) {
|
|
262
|
+
let size = arena.length;
|
|
263
|
+
while (size < this.#arenaLen + bytes.length) size *= 2;
|
|
264
|
+
const grown = new Uint8Array(size);
|
|
265
|
+
grown.set(arena.subarray(0, this.#arenaLen));
|
|
266
|
+
arena = grown;
|
|
267
|
+
this.#arena = grown;
|
|
268
|
+
}
|
|
269
|
+
arena.set(bytes, this.#arenaLen);
|
|
270
|
+
this.#keyOffsets[slot] = this.#arenaLen;
|
|
271
|
+
this.#keyLengths[slot] = bytes.length;
|
|
272
|
+
this.#arenaLen += bytes.length;
|
|
273
|
+
}
|
|
274
|
+
#slotFor(bytes) {
|
|
275
|
+
if (!this.#keyed) {
|
|
276
|
+
const key = crypto.getRandomValues(new Uint32Array(2));
|
|
277
|
+
this.#k0 = key[0] ?? 0;
|
|
278
|
+
this.#k1 = key[1] ?? 0;
|
|
279
|
+
this.#keyed = true;
|
|
280
|
+
}
|
|
281
|
+
const mask = this.#size - 1;
|
|
282
|
+
let slot = halfSipHash13(bytes, this.#k0, this.#k1) & mask;
|
|
283
|
+
for (;;) {
|
|
284
|
+
if ((this.#counts[slot] ?? 0) === 0) return slot;
|
|
285
|
+
if (this.#slotHolds(slot, bytes)) return slot;
|
|
286
|
+
slot = slot + 1 & mask;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
add(key, count = 1) {
|
|
290
|
+
assertPositiveInt(count, "count");
|
|
291
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new TopKOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
292
|
+
const bytes = normalize(key);
|
|
293
|
+
const slot = this.#slotFor(bytes);
|
|
294
|
+
const stored = this.#counts[slot] ?? 0;
|
|
295
|
+
if (stored + count > 4294967295) throw new TopKOverflowError(`adding ${String(count)} would carry a stored count past ${String(4294967295)}`);
|
|
296
|
+
if (stored === 0) {
|
|
297
|
+
this.#storeKey(slot, bytes);
|
|
298
|
+
this.#entries++;
|
|
299
|
+
}
|
|
300
|
+
this.#counts[slot] = stored + count;
|
|
301
|
+
this.#total += count;
|
|
302
|
+
if (this.#entries > topKLoadLimit(this.#size)) {
|
|
303
|
+
if (this.#size < this.#capacity) this.#resize(this.#size * 2);
|
|
304
|
+
else this.#purge();
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
count(key) {
|
|
308
|
+
const bytes = normalize(key);
|
|
309
|
+
const stored = this.#counts[this.#slotFor(bytes)] ?? 0;
|
|
310
|
+
return stored === 0 ? 0 : stored + this.#offset;
|
|
311
|
+
}
|
|
312
|
+
top(k) {
|
|
313
|
+
assertPositiveInt(k, "k");
|
|
314
|
+
return this.#canonicalSlots().slice(0, k).map((slot) => {
|
|
315
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
316
|
+
return {
|
|
317
|
+
key: this.#arena.slice(at, at + (this.#keyLengths[slot] ?? 0)),
|
|
318
|
+
count: (this.#counts[slot] ?? 0) + this.#offset,
|
|
319
|
+
lowerBound: this.#counts[slot] ?? 0
|
|
320
|
+
};
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
union(other) {
|
|
324
|
+
if (this.#capacity !== other.#capacity) throw new TopKParamMismatchError(`cannot union Top-K sketches with capacity ${String(this.#capacity)} and ${String(other.#capacity)}`);
|
|
325
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new TopKOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
326
|
+
const keys = [];
|
|
327
|
+
const counts = [];
|
|
328
|
+
const indexOf = new Int32Array(this.#size).fill(-1);
|
|
329
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
330
|
+
const stored = this.#counts[slot] ?? 0;
|
|
331
|
+
if (stored === 0) continue;
|
|
332
|
+
indexOf[slot] = keys.length;
|
|
333
|
+
keys.push(this.#keyAt(slot).slice());
|
|
334
|
+
counts.push(stored);
|
|
335
|
+
}
|
|
336
|
+
for (let slot = 0; slot < other.#size; slot++) {
|
|
337
|
+
const stored = other.#counts[slot] ?? 0;
|
|
338
|
+
if (stored === 0) continue;
|
|
339
|
+
const key = other.#keyAt(slot);
|
|
340
|
+
const at = indexOf[this.#slotFor(key)] ?? -1;
|
|
341
|
+
if (at >= 0) {
|
|
342
|
+
const sum = (counts[at] ?? 0) + stored;
|
|
343
|
+
if (sum > 4294967295) throw new TopKOverflowError(`union would carry a stored count to ${String(sum)}, past ${String(4294967295)}`);
|
|
344
|
+
counts[at] = sum;
|
|
345
|
+
} else {
|
|
346
|
+
keys.push(key.slice());
|
|
347
|
+
counts.push(stored);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
const merged = new TopK({ capacity: this.#capacity });
|
|
351
|
+
merged.#offset = this.#offset + other.#offset;
|
|
352
|
+
merged.#total = this.#total + other.#total;
|
|
353
|
+
merged.#reduceTo(keys, counts);
|
|
354
|
+
return merged;
|
|
355
|
+
}
|
|
356
|
+
toBytes() {
|
|
357
|
+
const slots = this.#canonicalSlots();
|
|
358
|
+
const tableEnd = PARAMS_SIZE + 8 * slots.length;
|
|
359
|
+
return writeFrame({
|
|
360
|
+
version: 5,
|
|
361
|
+
type: TYPE,
|
|
362
|
+
flags: 0
|
|
363
|
+
}, PARAMS_SIZE, 8 * slots.length + this.#arenaLen, (body, view) => {
|
|
364
|
+
view.setUint32(0, this.#capacity, true);
|
|
365
|
+
view.setUint32(4, slots.length, true);
|
|
366
|
+
view.setFloat64(8, this.#offset, true);
|
|
367
|
+
view.setFloat64(16, this.#total, true);
|
|
368
|
+
let keyAt = tableEnd;
|
|
369
|
+
slots.forEach((slot, i) => {
|
|
370
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
371
|
+
const len = this.#keyLengths[slot] ?? 0;
|
|
372
|
+
view.setUint32(PARAMS_SIZE + 4 * i, this.#counts[slot] ?? 0, true);
|
|
373
|
+
view.setUint32(PARAMS_SIZE + 4 * (slots.length + i), len, true);
|
|
374
|
+
body.set(this.#arena.subarray(at, at + len), keyAt);
|
|
375
|
+
keyAt += len;
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
equals(other) {
|
|
380
|
+
if (this.#capacity !== other.#capacity || this.#offset !== other.#offset || this.#total !== other.#total || this.#entries !== other.#entries) return false;
|
|
381
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
382
|
+
const stored = this.#counts[slot] ?? 0;
|
|
383
|
+
if (stored === 0) continue;
|
|
384
|
+
if (other.#counts[other.#slotFor(this.#keyAt(slot))] !== stored) return false;
|
|
385
|
+
}
|
|
386
|
+
return true;
|
|
387
|
+
}
|
|
388
|
+
toJSON() {
|
|
389
|
+
return toJSONEnvelope(this.toBytes());
|
|
390
|
+
}
|
|
391
|
+
static fromJSON(value) {
|
|
392
|
+
return TopK.fromBytes(fromJSONEnvelope(value));
|
|
393
|
+
}
|
|
394
|
+
#canonicalSlots() {
|
|
395
|
+
const slots = [];
|
|
396
|
+
for (let slot = 0; slot < this.#size; slot++) if ((this.#counts[slot] ?? 0) !== 0) slots.push(slot);
|
|
397
|
+
return slots.sort((a, b) => {
|
|
398
|
+
const byCount = (this.#counts[b] ?? 0) - (this.#counts[a] ?? 0);
|
|
399
|
+
return byCount !== 0 ? byCount : this.#compareKeys(a, b);
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
#compareKeys(a, b) {
|
|
403
|
+
return compareBytes(this.#keyAt(a), this.#keyAt(b));
|
|
404
|
+
}
|
|
405
|
+
#keyAt(slot) {
|
|
406
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
407
|
+
return this.#arena.subarray(at, at + (this.#keyLengths[slot] ?? 0));
|
|
408
|
+
}
|
|
409
|
+
};
|
|
410
|
+
//#endregion
|
|
411
|
+
export { BadMagicError, ChecksumError, ParamError, ReservedBitsError, SerializationError, TopK, TopKOverflowError, TopKParamMismatchError, TruncatedError, UnknownHashVariantError, UnknownVersionError, topKSizing };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "distillate",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.1",
|
|
4
4
|
"description": "Probabilistic data structures for JavaScript. Space-efficient, approximate-membership filters (Bloom, Blocked Bloom, Binary Fuse) with tunable error and a portable binary format; zero dependencies, universal.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bloom-filter",
|
|
@@ -95,6 +95,14 @@
|
|
|
95
95
|
"import": "./dist/countmin/index.js",
|
|
96
96
|
"require": "./dist/countmin/index.cjs"
|
|
97
97
|
},
|
|
98
|
+
"./topk": {
|
|
99
|
+
"types": {
|
|
100
|
+
"import": "./dist/topk/index.d.ts",
|
|
101
|
+
"require": "./dist/topk/index.d.cts"
|
|
102
|
+
},
|
|
103
|
+
"import": "./dist/topk/index.js",
|
|
104
|
+
"require": "./dist/topk/index.cjs"
|
|
105
|
+
},
|
|
98
106
|
"./package.json": "./package.json"
|
|
99
107
|
},
|
|
100
108
|
"main": "./dist/index.cjs",
|
|
@@ -105,17 +113,17 @@
|
|
|
105
113
|
],
|
|
106
114
|
"devDependencies": {
|
|
107
115
|
"@arethetypeswrong/cli": "^0.18.5",
|
|
108
|
-
"@microsoft/api-extractor": "^7.59.
|
|
109
|
-
"@types/node": "^22.20.
|
|
110
|
-
"@vitest/coverage-v8": "^5.0.
|
|
111
|
-
"fast-check": "^4.10.
|
|
116
|
+
"@microsoft/api-extractor": "^7.59.2",
|
|
117
|
+
"@types/node": "^22.20.4",
|
|
118
|
+
"@vitest/coverage-v8": "^5.0.2",
|
|
119
|
+
"fast-check": "^4.10.2",
|
|
112
120
|
"mitata": "^1.0.34",
|
|
113
121
|
"publint": "^0.3.24",
|
|
114
122
|
"tsdown": "^0.23.0",
|
|
115
|
-
"tsx": "^4.23.
|
|
123
|
+
"tsx": "^4.23.15",
|
|
116
124
|
"typedoc": "^0.28.20",
|
|
117
125
|
"typescript": "^5.9.3",
|
|
118
|
-
"vitest": "^5.0.
|
|
126
|
+
"vitest": "^5.0.2"
|
|
119
127
|
},
|
|
120
128
|
"engines": {
|
|
121
129
|
"node": ">=22"
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
//#region src/core/sizing.d.ts
|
|
2
|
-
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
3
|
-
interface BloomSizing {
|
|
4
|
-
/** Number of bits in the filter. */
|
|
5
|
-
m: number;
|
|
6
|
-
/** Number of hash probes per key. */
|
|
7
|
-
k: number;
|
|
8
|
-
}
|
|
9
|
-
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
10
|
-
declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
11
|
-
/** HyperLogLog geometry: the `HllParams` fields a sizing solve determines. */
|
|
12
|
-
interface HllSizing {
|
|
13
|
-
/** Precision: the sketch holds `2 ** p` registers. */
|
|
14
|
-
p: number;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Smallest precision whose standard error `1.04 / sqrt(2 ** p)` meets
|
|
18
|
-
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
19
|
-
*/
|
|
20
|
-
declare function hllSizing(relativeError: number): HllSizing;
|
|
21
|
-
/** Count-Min geometry: the `CountMinParams` fields a sizing solve determines. */
|
|
22
|
-
interface CountMinSizing {
|
|
23
|
-
/** Counters per row. */
|
|
24
|
-
width: number;
|
|
25
|
-
/** Number of rows, one probe each. */
|
|
26
|
-
depth: number;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Count-Min sizing: `width` columns and `depth` rows for an estimate at most
|
|
30
|
-
* `epsilon * total` above the true count, with probability `1 - delta`.
|
|
31
|
-
*/
|
|
32
|
-
declare function countMinSizing(epsilon: number, delta: number): CountMinSizing;
|
|
33
|
-
//#endregion
|
|
34
|
-
export { countMinSizing as a, bloomSizing as i, CountMinSizing as n, hllSizing as o, HllSizing as r, BloomSizing as t };
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
//#region src/core/sizing.d.ts
|
|
2
|
-
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
3
|
-
interface BloomSizing {
|
|
4
|
-
/** Number of bits in the filter. */
|
|
5
|
-
m: number;
|
|
6
|
-
/** Number of hash probes per key. */
|
|
7
|
-
k: number;
|
|
8
|
-
}
|
|
9
|
-
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
10
|
-
declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
11
|
-
/** HyperLogLog geometry: the `HllParams` fields a sizing solve determines. */
|
|
12
|
-
interface HllSizing {
|
|
13
|
-
/** Precision: the sketch holds `2 ** p` registers. */
|
|
14
|
-
p: number;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Smallest precision whose standard error `1.04 / sqrt(2 ** p)` meets
|
|
18
|
-
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
19
|
-
*/
|
|
20
|
-
declare function hllSizing(relativeError: number): HllSizing;
|
|
21
|
-
/** Count-Min geometry: the `CountMinParams` fields a sizing solve determines. */
|
|
22
|
-
interface CountMinSizing {
|
|
23
|
-
/** Counters per row. */
|
|
24
|
-
width: number;
|
|
25
|
-
/** Number of rows, one probe each. */
|
|
26
|
-
depth: number;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Count-Min sizing: `width` columns and `depth` rows for an estimate at most
|
|
30
|
-
* `epsilon * total` above the true count, with probability `1 - delta`.
|
|
31
|
-
*/
|
|
32
|
-
declare function countMinSizing(epsilon: number, delta: number): CountMinSizing;
|
|
33
|
-
//#endregion
|
|
34
|
-
export { countMinSizing as a, bloomSizing as i, CountMinSizing as n, hllSizing as o, HllSizing as r, BloomSizing as t };
|
package/dist/sizing-D3-_7pdP.js
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import { i as assertProbability, t as ParamError } from "./params-CeajSrwv.js";
|
|
2
|
-
//#region src/core/sizing.ts
|
|
3
|
-
function bloomSizing(n, epsilon) {
|
|
4
|
-
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
5
|
-
return {
|
|
6
|
-
m,
|
|
7
|
-
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
8
|
-
};
|
|
9
|
-
}
|
|
10
|
-
function hllSizing(relativeError) {
|
|
11
|
-
assertProbability(relativeError, "relativeError");
|
|
12
|
-
const p = Math.max(4, Math.ceil(2 * Math.log2(1.04 / relativeError)));
|
|
13
|
-
if (p > 18) throw new ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
14
|
-
return { p };
|
|
15
|
-
}
|
|
16
|
-
function countMinSizing(epsilon, delta) {
|
|
17
|
-
assertProbability(epsilon, "epsilon");
|
|
18
|
-
assertProbability(delta, "delta");
|
|
19
|
-
return {
|
|
20
|
-
width: Math.ceil(Math.E / epsilon),
|
|
21
|
-
depth: Math.ceil(Math.log(1 / delta))
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
//#endregion
|
|
25
|
-
export { countMinSizing as n, hllSizing as r, bloomSizing as t };
|
package/dist/sizing-R1pwbzjC.cjs
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
const require_params = require("./params-UTZbJ22c.cjs");
|
|
2
|
-
//#region src/core/sizing.ts
|
|
3
|
-
function bloomSizing(n, epsilon) {
|
|
4
|
-
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
5
|
-
return {
|
|
6
|
-
m,
|
|
7
|
-
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
8
|
-
};
|
|
9
|
-
}
|
|
10
|
-
function hllSizing(relativeError) {
|
|
11
|
-
require_params.assertProbability(relativeError, "relativeError");
|
|
12
|
-
const p = Math.max(4, Math.ceil(2 * Math.log2(1.04 / relativeError)));
|
|
13
|
-
if (p > 18) throw new require_params.ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
14
|
-
return { p };
|
|
15
|
-
}
|
|
16
|
-
function countMinSizing(epsilon, delta) {
|
|
17
|
-
require_params.assertProbability(epsilon, "epsilon");
|
|
18
|
-
require_params.assertProbability(delta, "delta");
|
|
19
|
-
return {
|
|
20
|
-
width: Math.ceil(Math.E / epsilon),
|
|
21
|
-
depth: Math.ceil(Math.log(1 / delta))
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
//#endregion
|
|
25
|
-
Object.defineProperty(exports, "bloomSizing", {
|
|
26
|
-
enumerable: true,
|
|
27
|
-
get: function() {
|
|
28
|
-
return bloomSizing;
|
|
29
|
-
}
|
|
30
|
-
});
|
|
31
|
-
Object.defineProperty(exports, "countMinSizing", {
|
|
32
|
-
enumerable: true,
|
|
33
|
-
get: function() {
|
|
34
|
-
return countMinSizing;
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
Object.defineProperty(exports, "hllSizing", {
|
|
38
|
-
enumerable: true,
|
|
39
|
-
get: function() {
|
|
40
|
-
return hllSizing;
|
|
41
|
-
}
|
|
42
|
-
});
|