distillate 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -12
- package/dist/blocked/index.cjs +1 -1
- package/dist/blocked/index.js +1 -1
- package/dist/bloom/index.cjs +3 -4
- package/dist/bloom/index.d.cts +12 -2
- package/dist/bloom/index.d.ts +12 -2
- package/dist/bloom/index.js +2 -3
- package/dist/bytes-Be7tq1F7.js +9 -0
- package/dist/bytes-D2duG1jq.cjs +14 -0
- package/dist/countmin/index.cjs +13 -4
- package/dist/countmin/index.d.cts +15 -2
- package/dist/countmin/index.d.ts +15 -2
- package/dist/countmin/index.js +12 -3
- package/dist/cuckoo/index.cjs +20 -9
- package/dist/cuckoo/index.js +20 -9
- package/dist/distinct-BhxDB8mW.js +61 -0
- package/dist/distinct-Ck5PKuCS.cjs +66 -0
- package/dist/fuse/index.cjs +6 -11
- package/dist/fuse/index.js +6 -11
- package/dist/{hasher-3Cyn7Yg-.js → hasher-DOdmZ-LI.js} +1 -8
- package/dist/{hasher-MhY7aQ8F.cjs → hasher-DPXmP7SH.cjs} +2 -9
- package/dist/hll/index.cjs +10 -4
- package/dist/hll/index.d.cts +13 -2
- package/dist/hll/index.d.ts +13 -2
- package/dist/hll/index.js +9 -3
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +5 -6
- package/dist/scalable/index.js +2 -3
- package/dist/{bitset-QYGqxnEf.cjs → sizing-Ch0SDdza.cjs} +15 -0
- package/dist/{bitset-Cf2jwRJX.js → sizing-zxCBlyhO.js} +10 -1
- package/dist/topk/index.cjs +423 -0
- package/dist/topk/index.d.cts +207 -0
- package/dist/topk/index.d.ts +207 -0
- package/dist/topk/index.js +411 -0
- package/package.json +15 -7
- package/dist/sizing-BmKfHf_V.d.cts +0 -34
- package/dist/sizing-BmKfHf_V.d.ts +0 -34
- package/dist/sizing-D3-_7pdP.js +0 -25
- package/dist/sizing-R1pwbzjC.cjs +0 -42
|
@@ -1,11 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const encoder = new TextEncoder();
|
|
3
|
-
function normalize(input) {
|
|
4
|
-
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
-
if (input instanceof Uint8Array) return input;
|
|
6
|
-
return new Uint8Array(input);
|
|
7
|
-
}
|
|
8
|
-
//#endregion
|
|
1
|
+
const require_bytes = require("./bytes-D2duG1jq.cjs");
|
|
9
2
|
//#region src/core/hasher.ts
|
|
10
3
|
let RLO = 0;
|
|
11
4
|
let RHI = 0;
|
|
@@ -185,7 +178,7 @@ function encodeKey(key) {
|
|
|
185
178
|
encBytes = keyBuf;
|
|
186
179
|
return;
|
|
187
180
|
}
|
|
188
|
-
encBytes = normalize(key);
|
|
181
|
+
encBytes = require_bytes.normalize(key);
|
|
189
182
|
encLen = encBytes.length;
|
|
190
183
|
}
|
|
191
184
|
function keyToLanes(key, seed) {
|
package/dist/hll/index.cjs
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-DPXmP7SH.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
-
|
|
5
|
+
function hllSizing(relativeError) {
|
|
6
|
+
require_params.assertProbability(relativeError, "relativeError");
|
|
7
|
+
const p = Math.max(4, Math.ceil(2 * Math.log2(1.04 / relativeError)));
|
|
8
|
+
if (p > 18) throw new require_params.ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
9
|
+
return { p };
|
|
10
|
+
}
|
|
11
|
+
//#endregion
|
|
6
12
|
//#region src/hll/estimate.ts
|
|
7
13
|
const ALPHA_INF = .5 / Math.LN2;
|
|
8
14
|
function sigma(x) {
|
|
@@ -171,7 +177,7 @@ var HyperLogLog = class HyperLogLog {
|
|
|
171
177
|
this.#sparse = new Int32Array(sparseCapacity(this.#registers));
|
|
172
178
|
}
|
|
173
179
|
static create(relativeError) {
|
|
174
|
-
return new HyperLogLog(
|
|
180
|
+
return new HyperLogLog(hllSizing(relativeError));
|
|
175
181
|
}
|
|
176
182
|
static from(keys, relativeError) {
|
|
177
183
|
const sketch = HyperLogLog.create(relativeError);
|
|
@@ -363,4 +369,4 @@ exports.SerializationError = require_serialize.SerializationError;
|
|
|
363
369
|
exports.TruncatedError = require_serialize.TruncatedError;
|
|
364
370
|
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
365
371
|
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
366
|
-
exports.hllSizing =
|
|
372
|
+
exports.hllSizing = hllSizing;
|
package/dist/hll/index.d.cts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
-
import { o as hllSizing, r as HllSizing } from "../sizing-BmKfHf_V.cjs";
|
|
5
4
|
//#region src/hll/hll.d.ts
|
|
6
5
|
/** Construction parameters for a {@link HyperLogLog} sketch. */
|
|
7
6
|
interface HllParams {
|
|
@@ -117,4 +116,16 @@ export declare class HyperLogLog {
|
|
|
117
116
|
count(): number;
|
|
118
117
|
}
|
|
119
118
|
//#endregion
|
|
120
|
-
|
|
119
|
+
//#region src/hll/sizing.d.ts
|
|
120
|
+
/** HyperLogLog geometry: the `HllParams` fields a sizing solve determines. */
|
|
121
|
+
interface HllSizing {
|
|
122
|
+
/** Precision: the sketch holds `2 ** p` registers. */
|
|
123
|
+
p: number;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Smallest precision whose standard error `1.04 / sqrt(2 ** p)` meets
|
|
127
|
+
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
128
|
+
*/
|
|
129
|
+
export declare function hllSizing(relativeError: number): HllSizing;
|
|
130
|
+
//#endregion
|
|
131
|
+
export { BadMagicError, ChecksumError, type FilterJSON, type HllParams, type HllSizing, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/hll/index.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
-
import { o as hllSizing, r as HllSizing } from "../sizing-BmKfHf_V.js";
|
|
5
4
|
//#region src/hll/hll.d.ts
|
|
6
5
|
/** Construction parameters for a {@link HyperLogLog} sketch. */
|
|
7
6
|
interface HllParams {
|
|
@@ -117,4 +116,16 @@ export declare class HyperLogLog {
|
|
|
117
116
|
count(): number;
|
|
118
117
|
}
|
|
119
118
|
//#endregion
|
|
120
|
-
|
|
119
|
+
//#region src/hll/sizing.d.ts
|
|
120
|
+
/** HyperLogLog geometry: the `HllParams` fields a sizing solve determines. */
|
|
121
|
+
interface HllSizing {
|
|
122
|
+
/** Precision: the sketch holds `2 ** p` registers. */
|
|
123
|
+
p: number;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Smallest precision whose standard error `1.04 / sqrt(2 ** p)` meets
|
|
127
|
+
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
128
|
+
*/
|
|
129
|
+
export declare function hllSizing(relativeError: number): HllSizing;
|
|
130
|
+
//#endregion
|
|
131
|
+
export { BadMagicError, ChecksumError, type FilterJSON, type HllParams, type HllSizing, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/hll/index.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
|
-
import { i as hash128KeyInto } from "../hasher-
|
|
1
|
+
import { i as hash128KeyInto } from "../hasher-DOdmZ-LI.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
|
-
import { o as assertUint32, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
-
|
|
3
|
+
import { i as assertProbability, o as assertUint32, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
+
function hllSizing(relativeError) {
|
|
5
|
+
assertProbability(relativeError, "relativeError");
|
|
6
|
+
const p = Math.max(4, Math.ceil(2 * Math.log2(1.04 / relativeError)));
|
|
7
|
+
if (p > 18) throw new ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
8
|
+
return { p };
|
|
9
|
+
}
|
|
10
|
+
//#endregion
|
|
5
11
|
//#region src/hll/estimate.ts
|
|
6
12
|
const ALPHA_INF = .5 / Math.LN2;
|
|
7
13
|
function sigma(x) {
|
package/dist/index.cjs
CHANGED
package/dist/index.js
CHANGED
package/dist/scalable/index.cjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
3
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_sizing = require("../sizing-Ch0SDdza.cjs");
|
|
3
|
+
const require_hasher = require("../hasher-DPXmP7SH.cjs");
|
|
4
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
5
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
6
|
-
const require_sizing = require("../sizing-R1pwbzjC.cjs");
|
|
7
6
|
//#region src/scalable/scalable.ts
|
|
8
7
|
const MAX_BITS = 4294967295;
|
|
9
8
|
const TYPE = 6;
|
|
@@ -35,7 +34,7 @@ function readStages(body, entries, start) {
|
|
|
35
34
|
let at = start;
|
|
36
35
|
return entries.map((entry, i) => {
|
|
37
36
|
const length = Math.ceil(entry.m / 8);
|
|
38
|
-
const bits = new
|
|
37
|
+
const bits = new require_sizing.BitSet(entry.m);
|
|
39
38
|
bits.bytes.set(body.subarray(at, at + length));
|
|
40
39
|
for (let j = at + length; j < at + padded8(length); j++) if (body[j] !== 0) throw new require_serialize.SerializationError(`scalable: stage ${String(i)} bit padding is not zero`);
|
|
41
40
|
at += padded8(length);
|
|
@@ -103,7 +102,7 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
103
102
|
}
|
|
104
103
|
#open({ capacity, m, k }) {
|
|
105
104
|
const stage = {
|
|
106
|
-
bits: new
|
|
105
|
+
bits: new require_sizing.BitSet(m),
|
|
107
106
|
m,
|
|
108
107
|
k,
|
|
109
108
|
capacity,
|
|
@@ -206,7 +205,7 @@ var ScalableBloomFilter = class ScalableBloomFilter {
|
|
|
206
205
|
this.#assertMergeable(other);
|
|
207
206
|
const [longer, shorter] = this.#stages.length >= other.#stages.length ? [this.#stages, other.#stages] : [other.#stages, this.#stages];
|
|
208
207
|
const stages = longer.map((stage, i) => {
|
|
209
|
-
const bits = new
|
|
208
|
+
const bits = new require_sizing.BitSet(stage.m);
|
|
210
209
|
bits.bytes.set(stage.bits.bytes);
|
|
211
210
|
const twin = shorter[i];
|
|
212
211
|
if (twin) {
|
package/dist/scalable/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { i as hash128KeyInto, l as probeLanesInto, s as probeAt } from "../hasher-
|
|
1
|
+
import { n as BitSet, t as bloomSizing } from "../sizing-zxCBlyhO.js";
|
|
2
|
+
import { i as hash128KeyInto, l as probeLanesInto, s as probeAt } from "../hasher-DOdmZ-LI.js";
|
|
3
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
4
4
|
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
5
|
-
import { t as bloomSizing } from "../sizing-D3-_7pdP.js";
|
|
6
5
|
//#region src/scalable/scalable.ts
|
|
7
6
|
const MAX_BITS = 4294967295;
|
|
8
7
|
const TYPE = 6;
|
|
@@ -29,9 +29,24 @@ var BitSet = class {
|
|
|
29
29
|
}
|
|
30
30
|
};
|
|
31
31
|
//#endregion
|
|
32
|
+
//#region src/core/sizing.ts
|
|
33
|
+
function bloomSizing(n, epsilon) {
|
|
34
|
+
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
35
|
+
return {
|
|
36
|
+
m,
|
|
37
|
+
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//#endregion
|
|
32
41
|
Object.defineProperty(exports, "BitSet", {
|
|
33
42
|
enumerable: true,
|
|
34
43
|
get: function() {
|
|
35
44
|
return BitSet;
|
|
36
45
|
}
|
|
37
46
|
});
|
|
47
|
+
Object.defineProperty(exports, "bloomSizing", {
|
|
48
|
+
enumerable: true,
|
|
49
|
+
get: function() {
|
|
50
|
+
return bloomSizing;
|
|
51
|
+
}
|
|
52
|
+
});
|
|
@@ -29,4 +29,13 @@ var BitSet = class {
|
|
|
29
29
|
}
|
|
30
30
|
};
|
|
31
31
|
//#endregion
|
|
32
|
-
|
|
32
|
+
//#region src/core/sizing.ts
|
|
33
|
+
function bloomSizing(n, epsilon) {
|
|
34
|
+
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
35
|
+
return {
|
|
36
|
+
m,
|
|
37
|
+
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//#endregion
|
|
41
|
+
export { BitSet as n, bloomSizing as t };
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_bytes = require("../bytes-D2duG1jq.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
//#region src/topk/halfsip.ts
|
|
6
|
+
function halfSipHash13(bytes, k0, k1) {
|
|
7
|
+
let v0 = k0 | 0;
|
|
8
|
+
let v1 = k1 | 0;
|
|
9
|
+
let v2 = 1819895653 ^ k0 | 0;
|
|
10
|
+
let v3 = 1952801890 ^ k1 | 0;
|
|
11
|
+
const len = bytes.length;
|
|
12
|
+
const end = len & -4;
|
|
13
|
+
for (let i = 0; i < end; i += 4) {
|
|
14
|
+
const m = (bytes[i] ?? 0) | (bytes[i + 1] ?? 0) << 8 | (bytes[i + 2] ?? 0) << 16 | (bytes[i + 3] ?? 0) << 24;
|
|
15
|
+
v3 ^= m;
|
|
16
|
+
v0 = v0 + v1 | 0;
|
|
17
|
+
v1 = v1 << 5 | v1 >>> 27;
|
|
18
|
+
v1 ^= v0;
|
|
19
|
+
v0 = v0 << 16 | v0 >>> 16;
|
|
20
|
+
v2 = v2 + v3 | 0;
|
|
21
|
+
v3 = v3 << 8 | v3 >>> 24;
|
|
22
|
+
v3 ^= v2;
|
|
23
|
+
v0 = v0 + v3 | 0;
|
|
24
|
+
v3 = v3 << 7 | v3 >>> 25;
|
|
25
|
+
v3 ^= v0;
|
|
26
|
+
v2 = v2 + v1 | 0;
|
|
27
|
+
v1 = v1 << 13 | v1 >>> 19;
|
|
28
|
+
v1 ^= v2;
|
|
29
|
+
v2 = v2 << 16 | v2 >>> 16;
|
|
30
|
+
v0 ^= m;
|
|
31
|
+
}
|
|
32
|
+
let b = len << 24;
|
|
33
|
+
for (let i = len - 1; i >= end; i--) b |= (bytes[i] ?? 0) << 8 * (i - end);
|
|
34
|
+
v3 ^= b;
|
|
35
|
+
for (let r = 0; r < 4; r++) {
|
|
36
|
+
if (r === 1) {
|
|
37
|
+
v0 ^= b;
|
|
38
|
+
v2 ^= 255;
|
|
39
|
+
}
|
|
40
|
+
v0 = v0 + v1 | 0;
|
|
41
|
+
v1 = v1 << 5 | v1 >>> 27;
|
|
42
|
+
v1 ^= v0;
|
|
43
|
+
v0 = v0 << 16 | v0 >>> 16;
|
|
44
|
+
v2 = v2 + v3 | 0;
|
|
45
|
+
v3 = v3 << 8 | v3 >>> 24;
|
|
46
|
+
v3 ^= v2;
|
|
47
|
+
v0 = v0 + v3 | 0;
|
|
48
|
+
v3 = v3 << 7 | v3 >>> 25;
|
|
49
|
+
v3 ^= v0;
|
|
50
|
+
v2 = v2 + v1 | 0;
|
|
51
|
+
v1 = v1 << 13 | v1 >>> 19;
|
|
52
|
+
v1 ^= v2;
|
|
53
|
+
v2 = v2 << 16 | v2 >>> 16;
|
|
54
|
+
}
|
|
55
|
+
return (v1 ^ v3) >>> 0;
|
|
56
|
+
}
|
|
57
|
+
const TOPK_MAX_CAPACITY = 2 ** 24;
|
|
58
|
+
function topKLoadLimit(capacity) {
|
|
59
|
+
return Math.floor(.75 * capacity);
|
|
60
|
+
}
|
|
61
|
+
function topKPurgeWidth(capacity) {
|
|
62
|
+
return topKLoadLimit(capacity) + 2 >> 1;
|
|
63
|
+
}
|
|
64
|
+
function topKSizing(epsilon) {
|
|
65
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
66
|
+
for (let capacity = 4; capacity <= TOPK_MAX_CAPACITY; capacity *= 2) if (1 / topKPurgeWidth(capacity) <= epsilon) return { capacity };
|
|
67
|
+
throw new require_params.ParamError(`epsilon ${String(epsilon)} needs more than the maximum capacity ${String(TOPK_MAX_CAPACITY)}`);
|
|
68
|
+
}
|
|
69
|
+
//#endregion
|
|
70
|
+
//#region src/topk/topk.ts
|
|
71
|
+
const TYPE = 9;
|
|
72
|
+
const TABLE_START = 8;
|
|
73
|
+
const PARAMS_SIZE = 32;
|
|
74
|
+
const PARAMS_FIELDS_END = 24;
|
|
75
|
+
function assertCount(value, label) {
|
|
76
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new require_serialize.SerializationError(`topk: ${label} ${String(value)} is not an integer in [0, ${String(Number.MAX_SAFE_INTEGER)}]`);
|
|
77
|
+
}
|
|
78
|
+
function compareBytes(a, b) {
|
|
79
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
80
|
+
const diff = (a[i] ?? 0) - (b[i] ?? 0);
|
|
81
|
+
if (diff !== 0) return diff;
|
|
82
|
+
}
|
|
83
|
+
return a.length - b.length;
|
|
84
|
+
}
|
|
85
|
+
var TopKOverflowError = class extends RangeError {
|
|
86
|
+
name = "TopKOverflowError";
|
|
87
|
+
};
|
|
88
|
+
var TopKParamMismatchError = class extends Error {
|
|
89
|
+
name = "TopKParamMismatchError";
|
|
90
|
+
};
|
|
91
|
+
var TopK = class TopK {
|
|
92
|
+
#capacity;
|
|
93
|
+
#loadLimit;
|
|
94
|
+
#k0 = 0;
|
|
95
|
+
#k1 = 0;
|
|
96
|
+
#keyed = false;
|
|
97
|
+
#size = 0;
|
|
98
|
+
#counts = new Uint32Array(0);
|
|
99
|
+
#keyOffsets = new Uint32Array(0);
|
|
100
|
+
#keyLengths = new Uint32Array(0);
|
|
101
|
+
#arena;
|
|
102
|
+
#arenaLen = 0;
|
|
103
|
+
#entries = 0;
|
|
104
|
+
#offset = 0;
|
|
105
|
+
#total = 0;
|
|
106
|
+
static create(epsilon) {
|
|
107
|
+
return new TopK(topKSizing(epsilon));
|
|
108
|
+
}
|
|
109
|
+
static from(keys, epsilon) {
|
|
110
|
+
const sketch = TopK.create(epsilon);
|
|
111
|
+
for (const key of keys) sketch.add(key);
|
|
112
|
+
return sketch;
|
|
113
|
+
}
|
|
114
|
+
static fromBytes(bytes) {
|
|
115
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
116
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
117
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
118
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "topk");
|
|
119
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "topk");
|
|
120
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
121
|
+
const capacity = view.getUint32(0, true);
|
|
122
|
+
const entries = view.getUint32(4, true);
|
|
123
|
+
if (entries > topKLoadLimit(capacity)) throw new require_serialize.SerializationError(`topk: frame holds ${String(entries)} entries, above the load limit ${String(topKLoadLimit(capacity))} for capacity ${String(capacity)}`);
|
|
124
|
+
const tableEnd = PARAMS_SIZE + 8 * entries;
|
|
125
|
+
require_serialize.assertMinBodyLength(body.length, tableEnd, "topk");
|
|
126
|
+
let keyBytes = 0;
|
|
127
|
+
for (let i = 0; i < entries; i++) keyBytes += view.getUint32(PARAMS_SIZE + 4 * (entries + i), true);
|
|
128
|
+
require_serialize.assertBodyLength(body.length, tableEnd + keyBytes, "topk");
|
|
129
|
+
const offset = view.getFloat64(8, true);
|
|
130
|
+
const total = view.getFloat64(16, true);
|
|
131
|
+
assertCount(offset, "offset");
|
|
132
|
+
assertCount(total, "total");
|
|
133
|
+
let sketch;
|
|
134
|
+
try {
|
|
135
|
+
sketch = new TopK({ capacity });
|
|
136
|
+
sketch.#allocate(sketch.#fit(entries));
|
|
137
|
+
} catch (err) {
|
|
138
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`topk: ${err.message}`);
|
|
139
|
+
throw err;
|
|
140
|
+
}
|
|
141
|
+
let keyAt = tableEnd;
|
|
142
|
+
let stored = 0;
|
|
143
|
+
let prevCount = Infinity;
|
|
144
|
+
let prevKey = new Uint8Array(0);
|
|
145
|
+
for (let i = 0; i < entries; i++) {
|
|
146
|
+
const count = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
147
|
+
const len = view.getUint32(PARAMS_SIZE + 4 * (entries + i), true);
|
|
148
|
+
const key = body.subarray(keyAt, keyAt + len);
|
|
149
|
+
keyAt += len;
|
|
150
|
+
if (count === 0) throw new require_serialize.SerializationError(`topk: entry ${String(i)} has count 0`);
|
|
151
|
+
if (count > prevCount || count === prevCount && compareBytes(prevKey, key) >= 0) throw new require_serialize.SerializationError(`topk: entry ${String(i)} is out of canonical order`);
|
|
152
|
+
prevCount = count;
|
|
153
|
+
prevKey = key;
|
|
154
|
+
const slot = sketch.#slotFor(key);
|
|
155
|
+
if ((sketch.#counts[slot] ?? 0) !== 0) throw new require_serialize.SerializationError(`topk: entry ${String(i)} repeats a key already held`);
|
|
156
|
+
sketch.#storeKey(slot, key);
|
|
157
|
+
sketch.#counts[slot] = count;
|
|
158
|
+
stored += count;
|
|
159
|
+
}
|
|
160
|
+
const width = topKPurgeWidth(capacity);
|
|
161
|
+
if (stored + width * offset > total) throw new require_serialize.SerializationError(`topk: stored counts ${String(stored)} plus ${String(width)} x offset ${String(offset)} exceed total ${String(total)}`);
|
|
162
|
+
sketch.#entries = entries;
|
|
163
|
+
sketch.#offset = offset;
|
|
164
|
+
sketch.#total = total;
|
|
165
|
+
return sketch;
|
|
166
|
+
}
|
|
167
|
+
constructor({ capacity }) {
|
|
168
|
+
require_params.assertUint32(capacity, "capacity");
|
|
169
|
+
if (!Number.isInteger(Math.log2(capacity))) throw new require_params.ParamError(`capacity must be a power of two, got ${String(capacity)}`);
|
|
170
|
+
if (capacity < 4 || capacity > TOPK_MAX_CAPACITY) throw new require_params.ParamError(`capacity must be in [${String(4)}, ${String(TOPK_MAX_CAPACITY)}], got ${String(capacity)}`);
|
|
171
|
+
this.#capacity = capacity;
|
|
172
|
+
this.#loadLimit = topKLoadLimit(capacity);
|
|
173
|
+
this.#allocate(Math.min(TABLE_START, capacity));
|
|
174
|
+
this.#arena = new Uint8Array(256);
|
|
175
|
+
}
|
|
176
|
+
get capacity() {
|
|
177
|
+
return this.#capacity;
|
|
178
|
+
}
|
|
179
|
+
get epsilon() {
|
|
180
|
+
return 1 / topKPurgeWidth(this.#capacity);
|
|
181
|
+
}
|
|
182
|
+
get total() {
|
|
183
|
+
return this.#total;
|
|
184
|
+
}
|
|
185
|
+
error() {
|
|
186
|
+
return this.#offset;
|
|
187
|
+
}
|
|
188
|
+
#purge() {
|
|
189
|
+
const keys = [];
|
|
190
|
+
const counts = [];
|
|
191
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
192
|
+
const stored = this.#counts[slot] ?? 0;
|
|
193
|
+
if (stored === 0) continue;
|
|
194
|
+
keys.push(this.#keyAt(slot).slice());
|
|
195
|
+
counts.push(stored);
|
|
196
|
+
}
|
|
197
|
+
this.#reduceTo(keys, counts);
|
|
198
|
+
}
|
|
199
|
+
#reduceTo(keys, counts) {
|
|
200
|
+
while (keys.length > this.#loadLimit) {
|
|
201
|
+
const median = [...counts].sort((a, b) => a - b)[counts.length >> 1] ?? 0;
|
|
202
|
+
let kept = 0;
|
|
203
|
+
for (let i = 0; i < keys.length; i++) {
|
|
204
|
+
const survived = (counts[i] ?? 0) - median;
|
|
205
|
+
if (survived <= 0) continue;
|
|
206
|
+
keys[kept] = keys[i] ?? new Uint8Array(0);
|
|
207
|
+
counts[kept] = survived;
|
|
208
|
+
kept++;
|
|
209
|
+
}
|
|
210
|
+
keys.length = kept;
|
|
211
|
+
counts.length = kept;
|
|
212
|
+
this.#offset += median;
|
|
213
|
+
}
|
|
214
|
+
const size = this.#fit(keys.length);
|
|
215
|
+
if (size > this.#size) this.#allocate(size);
|
|
216
|
+
else this.#counts.fill(0);
|
|
217
|
+
this.#arenaLen = 0;
|
|
218
|
+
this.#entries = keys.length;
|
|
219
|
+
for (let i = 0; i < keys.length; i++) {
|
|
220
|
+
const bytes = keys[i] ?? new Uint8Array(0);
|
|
221
|
+
const slot = this.#slotFor(bytes);
|
|
222
|
+
this.#storeKey(slot, bytes);
|
|
223
|
+
this.#counts[slot] = counts[i] ?? 0;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
#allocate(size) {
|
|
227
|
+
this.#size = size;
|
|
228
|
+
this.#counts = new Uint32Array(size);
|
|
229
|
+
this.#keyOffsets = new Uint32Array(size);
|
|
230
|
+
this.#keyLengths = new Uint32Array(size);
|
|
231
|
+
}
|
|
232
|
+
#resize(size) {
|
|
233
|
+
const counts = this.#counts;
|
|
234
|
+
const offsets = this.#keyOffsets;
|
|
235
|
+
const lengths = this.#keyLengths;
|
|
236
|
+
this.#allocate(size);
|
|
237
|
+
for (let old = 0; old < counts.length; old++) {
|
|
238
|
+
const stored = counts[old] ?? 0;
|
|
239
|
+
if (stored === 0) continue;
|
|
240
|
+
const at = offsets[old] ?? 0;
|
|
241
|
+
const len = lengths[old] ?? 0;
|
|
242
|
+
const slot = this.#slotFor(this.#arena.subarray(at, at + len));
|
|
243
|
+
this.#counts[slot] = stored;
|
|
244
|
+
this.#keyOffsets[slot] = at;
|
|
245
|
+
this.#keyLengths[slot] = len;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
#fit(entries) {
|
|
249
|
+
let size = Math.min(TABLE_START, this.#capacity);
|
|
250
|
+
while (size < this.#capacity && topKLoadLimit(size) < entries) size *= 2;
|
|
251
|
+
return size;
|
|
252
|
+
}
|
|
253
|
+
#slotHolds(slot, bytes) {
|
|
254
|
+
const len = this.#keyLengths[slot] ?? 0;
|
|
255
|
+
if (len !== bytes.length) return false;
|
|
256
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
257
|
+
for (let i = 0; i < len; i++) if (this.#arena[at + i] !== bytes[i]) return false;
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
#storeKey(slot, bytes) {
|
|
261
|
+
let arena = this.#arena;
|
|
262
|
+
if (this.#arenaLen + bytes.length > arena.length) {
|
|
263
|
+
let size = arena.length;
|
|
264
|
+
while (size < this.#arenaLen + bytes.length) size *= 2;
|
|
265
|
+
const grown = new Uint8Array(size);
|
|
266
|
+
grown.set(arena.subarray(0, this.#arenaLen));
|
|
267
|
+
arena = grown;
|
|
268
|
+
this.#arena = grown;
|
|
269
|
+
}
|
|
270
|
+
arena.set(bytes, this.#arenaLen);
|
|
271
|
+
this.#keyOffsets[slot] = this.#arenaLen;
|
|
272
|
+
this.#keyLengths[slot] = bytes.length;
|
|
273
|
+
this.#arenaLen += bytes.length;
|
|
274
|
+
}
|
|
275
|
+
#slotFor(bytes) {
|
|
276
|
+
if (!this.#keyed) {
|
|
277
|
+
const key = crypto.getRandomValues(new Uint32Array(2));
|
|
278
|
+
this.#k0 = key[0] ?? 0;
|
|
279
|
+
this.#k1 = key[1] ?? 0;
|
|
280
|
+
this.#keyed = true;
|
|
281
|
+
}
|
|
282
|
+
const mask = this.#size - 1;
|
|
283
|
+
let slot = halfSipHash13(bytes, this.#k0, this.#k1) & mask;
|
|
284
|
+
for (;;) {
|
|
285
|
+
if ((this.#counts[slot] ?? 0) === 0) return slot;
|
|
286
|
+
if (this.#slotHolds(slot, bytes)) return slot;
|
|
287
|
+
slot = slot + 1 & mask;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
add(key, count = 1) {
|
|
291
|
+
require_params.assertPositiveInt(count, "count");
|
|
292
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new TopKOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
293
|
+
const bytes = require_bytes.normalize(key);
|
|
294
|
+
const slot = this.#slotFor(bytes);
|
|
295
|
+
const stored = this.#counts[slot] ?? 0;
|
|
296
|
+
if (stored + count > 4294967295) throw new TopKOverflowError(`adding ${String(count)} would carry a stored count past ${String(4294967295)}`);
|
|
297
|
+
if (stored === 0) {
|
|
298
|
+
this.#storeKey(slot, bytes);
|
|
299
|
+
this.#entries++;
|
|
300
|
+
}
|
|
301
|
+
this.#counts[slot] = stored + count;
|
|
302
|
+
this.#total += count;
|
|
303
|
+
if (this.#entries > topKLoadLimit(this.#size)) {
|
|
304
|
+
if (this.#size < this.#capacity) this.#resize(this.#size * 2);
|
|
305
|
+
else this.#purge();
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
count(key) {
|
|
309
|
+
const bytes = require_bytes.normalize(key);
|
|
310
|
+
const stored = this.#counts[this.#slotFor(bytes)] ?? 0;
|
|
311
|
+
return stored === 0 ? 0 : stored + this.#offset;
|
|
312
|
+
}
|
|
313
|
+
top(k) {
|
|
314
|
+
require_params.assertPositiveInt(k, "k");
|
|
315
|
+
return this.#canonicalSlots().slice(0, k).map((slot) => {
|
|
316
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
317
|
+
return {
|
|
318
|
+
key: this.#arena.slice(at, at + (this.#keyLengths[slot] ?? 0)),
|
|
319
|
+
count: (this.#counts[slot] ?? 0) + this.#offset,
|
|
320
|
+
lowerBound: this.#counts[slot] ?? 0
|
|
321
|
+
};
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
union(other) {
|
|
325
|
+
if (this.#capacity !== other.#capacity) throw new TopKParamMismatchError(`cannot union Top-K sketches with capacity ${String(this.#capacity)} and ${String(other.#capacity)}`);
|
|
326
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new TopKOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
327
|
+
const keys = [];
|
|
328
|
+
const counts = [];
|
|
329
|
+
const indexOf = new Int32Array(this.#size).fill(-1);
|
|
330
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
331
|
+
const stored = this.#counts[slot] ?? 0;
|
|
332
|
+
if (stored === 0) continue;
|
|
333
|
+
indexOf[slot] = keys.length;
|
|
334
|
+
keys.push(this.#keyAt(slot).slice());
|
|
335
|
+
counts.push(stored);
|
|
336
|
+
}
|
|
337
|
+
for (let slot = 0; slot < other.#size; slot++) {
|
|
338
|
+
const stored = other.#counts[slot] ?? 0;
|
|
339
|
+
if (stored === 0) continue;
|
|
340
|
+
const key = other.#keyAt(slot);
|
|
341
|
+
const at = indexOf[this.#slotFor(key)] ?? -1;
|
|
342
|
+
if (at >= 0) {
|
|
343
|
+
const sum = (counts[at] ?? 0) + stored;
|
|
344
|
+
if (sum > 4294967295) throw new TopKOverflowError(`union would carry a stored count to ${String(sum)}, past ${String(4294967295)}`);
|
|
345
|
+
counts[at] = sum;
|
|
346
|
+
} else {
|
|
347
|
+
keys.push(key.slice());
|
|
348
|
+
counts.push(stored);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
const merged = new TopK({ capacity: this.#capacity });
|
|
352
|
+
merged.#offset = this.#offset + other.#offset;
|
|
353
|
+
merged.#total = this.#total + other.#total;
|
|
354
|
+
merged.#reduceTo(keys, counts);
|
|
355
|
+
return merged;
|
|
356
|
+
}
|
|
357
|
+
toBytes() {
|
|
358
|
+
const slots = this.#canonicalSlots();
|
|
359
|
+
const tableEnd = PARAMS_SIZE + 8 * slots.length;
|
|
360
|
+
return require_serialize.writeFrame({
|
|
361
|
+
version: 5,
|
|
362
|
+
type: TYPE,
|
|
363
|
+
flags: 0
|
|
364
|
+
}, PARAMS_SIZE, 8 * slots.length + this.#arenaLen, (body, view) => {
|
|
365
|
+
view.setUint32(0, this.#capacity, true);
|
|
366
|
+
view.setUint32(4, slots.length, true);
|
|
367
|
+
view.setFloat64(8, this.#offset, true);
|
|
368
|
+
view.setFloat64(16, this.#total, true);
|
|
369
|
+
let keyAt = tableEnd;
|
|
370
|
+
slots.forEach((slot, i) => {
|
|
371
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
372
|
+
const len = this.#keyLengths[slot] ?? 0;
|
|
373
|
+
view.setUint32(PARAMS_SIZE + 4 * i, this.#counts[slot] ?? 0, true);
|
|
374
|
+
view.setUint32(PARAMS_SIZE + 4 * (slots.length + i), len, true);
|
|
375
|
+
body.set(this.#arena.subarray(at, at + len), keyAt);
|
|
376
|
+
keyAt += len;
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
equals(other) {
|
|
381
|
+
if (this.#capacity !== other.#capacity || this.#offset !== other.#offset || this.#total !== other.#total || this.#entries !== other.#entries) return false;
|
|
382
|
+
for (let slot = 0; slot < this.#size; slot++) {
|
|
383
|
+
const stored = this.#counts[slot] ?? 0;
|
|
384
|
+
if (stored === 0) continue;
|
|
385
|
+
if (other.#counts[other.#slotFor(this.#keyAt(slot))] !== stored) return false;
|
|
386
|
+
}
|
|
387
|
+
return true;
|
|
388
|
+
}
|
|
389
|
+
toJSON() {
|
|
390
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
391
|
+
}
|
|
392
|
+
static fromJSON(value) {
|
|
393
|
+
return TopK.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
394
|
+
}
|
|
395
|
+
#canonicalSlots() {
|
|
396
|
+
const slots = [];
|
|
397
|
+
for (let slot = 0; slot < this.#size; slot++) if ((this.#counts[slot] ?? 0) !== 0) slots.push(slot);
|
|
398
|
+
return slots.sort((a, b) => {
|
|
399
|
+
const byCount = (this.#counts[b] ?? 0) - (this.#counts[a] ?? 0);
|
|
400
|
+
return byCount !== 0 ? byCount : this.#compareKeys(a, b);
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
#compareKeys(a, b) {
|
|
404
|
+
return compareBytes(this.#keyAt(a), this.#keyAt(b));
|
|
405
|
+
}
|
|
406
|
+
#keyAt(slot) {
|
|
407
|
+
const at = this.#keyOffsets[slot] ?? 0;
|
|
408
|
+
return this.#arena.subarray(at, at + (this.#keyLengths[slot] ?? 0));
|
|
409
|
+
}
|
|
410
|
+
};
|
|
411
|
+
//#endregion
|
|
412
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
413
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
414
|
+
exports.ParamError = require_params.ParamError;
|
|
415
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
416
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
417
|
+
exports.TopK = TopK;
|
|
418
|
+
exports.TopKOverflowError = TopKOverflowError;
|
|
419
|
+
exports.TopKParamMismatchError = TopKParamMismatchError;
|
|
420
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
421
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
422
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
423
|
+
exports.topKSizing = topKSizing;
|