distillate 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist/blocked/index.cjs +14 -9
- package/dist/blocked/index.d.cts +7 -6
- package/dist/blocked/index.d.ts +7 -6
- package/dist/blocked/index.js +12 -8
- package/dist/bloom/index.cjs +14 -9
- package/dist/bloom/index.d.cts +5 -4
- package/dist/bloom/index.d.ts +5 -4
- package/dist/bloom/index.js +12 -8
- package/dist/bytes-DCuYtUVS.d.cts +4 -0
- package/dist/bytes-DCuYtUVS.d.ts +4 -0
- package/dist/frame/index.cjs +9 -0
- package/dist/frame/index.d.cts +2 -0
- package/dist/frame/index.d.ts +2 -0
- package/dist/frame/index.js +2 -0
- package/dist/fuse/index.cjs +26 -22
- package/dist/fuse/index.d.cts +7 -6
- package/dist/fuse/index.d.ts +7 -6
- package/dist/fuse/index.js +14 -11
- package/dist/hasher-D8LCBTOM.cjs +276 -0
- package/dist/hasher-DVWXS_vJ.js +229 -0
- package/dist/hll/index.cjs +12 -8
- package/dist/hll/index.d.cts +4 -3
- package/dist/hll/index.d.ts +4 -3
- package/dist/hll/index.js +11 -8
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +2 -3
- package/dist/index.d.ts +2 -3
- package/dist/index.js +1 -1
- package/dist/{serialize-Bv1zWzrh.d.cts → serialize--xR6vGSW.d.cts} +44 -8
- package/dist/{serialize-Bv1zWzrh.d.ts → serialize--xR6vGSW.d.ts} +44 -8
- package/dist/serialize-CHikZ4GH.cjs +296 -0
- package/dist/serialize-CMSYFym-.js +201 -0
- package/package.json +17 -9
- package/dist/serialize-D59Ri81a.js +0 -392
- package/dist/serialize-Ul2LKFJU.cjs +0 -517
package/dist/fuse/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
2
3
|
//#region src/fuse/fuse.d.ts
|
|
3
4
|
/** Thrown when binary fuse construction fails to converge on the key set. */
|
|
4
|
-
declare class BinaryFuseBuildError extends Error {
|
|
5
|
+
export declare class BinaryFuseBuildError extends Error {
|
|
5
6
|
/** Discriminates this error from other `Error`s. */
|
|
6
7
|
override readonly name = "BinaryFuseBuildError";
|
|
7
8
|
}
|
|
@@ -20,7 +21,7 @@ interface FuseParams {
|
|
|
20
21
|
* @param width - Fingerprint width in bits: `8` for {@link BinaryFuse8}, `16` for {@link BinaryFuse16}.
|
|
21
22
|
* @returns Bits per key (`0` for an empty filter).
|
|
22
23
|
*/
|
|
23
|
-
declare function fuseBitsPerKey(n: number, width: 8 | 16): number;
|
|
24
|
+
export declare function fuseBitsPerKey(n: number, width: 8 | 16): number;
|
|
24
25
|
interface FuseState {
|
|
25
26
|
fp: Uint8Array | Uint16Array;
|
|
26
27
|
seed: number;
|
|
@@ -81,7 +82,7 @@ declare abstract class BinaryFuse {
|
|
|
81
82
|
* filter.size; // 3
|
|
82
83
|
* ```
|
|
83
84
|
*/
|
|
84
|
-
declare class BinaryFuse8 extends BinaryFuse {
|
|
85
|
+
export declare class BinaryFuse8 extends BinaryFuse {
|
|
85
86
|
/**
|
|
86
87
|
* Builds a filter from the given keys; duplicates are ignored.
|
|
87
88
|
*
|
|
@@ -115,7 +116,7 @@ declare class BinaryFuse8 extends BinaryFuse {
|
|
|
115
116
|
* filter.has("alice"); // true
|
|
116
117
|
* ```
|
|
117
118
|
*/
|
|
118
|
-
declare class BinaryFuse16 extends BinaryFuse {
|
|
119
|
+
export declare class BinaryFuse16 extends BinaryFuse {
|
|
119
120
|
/**
|
|
120
121
|
* Builds a filter from the given keys; duplicates are ignored.
|
|
121
122
|
*
|
|
@@ -140,4 +141,4 @@ declare class BinaryFuse16 extends BinaryFuse {
|
|
|
140
141
|
static fromJSON(value: unknown): BinaryFuse16;
|
|
141
142
|
}
|
|
142
143
|
//#endregion
|
|
143
|
-
export { BadMagicError,
|
|
144
|
+
export { BadMagicError, ChecksumError, type FilterJSON, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/fuse/index.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { i as hash128KeyInto, n as RLO, o as mul64, r as fmix64, t as RHI } from "../hasher-DVWXS_vJ.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError } from "../serialize-CMSYFym-.js";
|
|
2
3
|
//#region src/fuse/fuse.ts
|
|
3
4
|
const ARITY = 3;
|
|
4
5
|
const TYPE_FUSE8 = 3;
|
|
5
6
|
const TYPE_FUSE16 = 4;
|
|
7
|
+
const PARAMS_SIZE = 16;
|
|
6
8
|
var BinaryFuseBuildError = class extends Error {
|
|
7
9
|
name = "BinaryFuseBuildError";
|
|
8
10
|
};
|
|
@@ -43,10 +45,10 @@ function computeParams(size) {
|
|
|
43
45
|
const segMask = seg - 1;
|
|
44
46
|
const sizeFactor = Math.max(1.125, .875 + .25 * Math.log(1e6) / Math.log(Math.max(size, 2)));
|
|
45
47
|
const capacity = Math.round(size * sizeFactor);
|
|
46
|
-
let segCount = Math.floor((capacity + seg - 1) / seg) -
|
|
48
|
+
let segCount = Math.floor((capacity + seg - 1) / seg) - 2;
|
|
47
49
|
if (segCount <= 0) segCount = 1;
|
|
48
50
|
let arrayLength = (segCount + ARITY - 1) * seg;
|
|
49
|
-
segCount = Math.floor((arrayLength + seg - 1) / seg) -
|
|
51
|
+
segCount = Math.floor((arrayLength + seg - 1) / seg) - 2;
|
|
50
52
|
if (segCount <= 0) segCount = 1;
|
|
51
53
|
arrayLength = (segCount + ARITY - 1) * seg;
|
|
52
54
|
return {
|
|
@@ -160,7 +162,7 @@ function fuseStateFromBytes(bytes, expectedType) {
|
|
|
160
162
|
const { type, flags, body } = readHeader(bytes);
|
|
161
163
|
if (type !== expectedType) throw new SerializationError(`expected DSTL type ${String(expectedType)}, got ${String(type)}`);
|
|
162
164
|
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
163
|
-
assertMinBodyLength(body.length,
|
|
165
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "fuse");
|
|
164
166
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
165
167
|
const seed = dv.getUint32(0, true);
|
|
166
168
|
const seg = dv.getUint32(4, true);
|
|
@@ -170,8 +172,8 @@ function fuseStateFromBytes(bytes, expectedType) {
|
|
|
170
172
|
if (segCountLen % seg !== 0) throw new SerializationError(`fuse segment count length ${String(segCountLen)} is not a multiple of segment length ${String(seg)}`);
|
|
171
173
|
const bpe = expectedType === TYPE_FUSE8 ? 1 : 2;
|
|
172
174
|
const expectedArrayLength = size === 0 ? 0 : segCountLen + 2 * seg;
|
|
173
|
-
assertBodyLength(body.length,
|
|
174
|
-
const laneBytes = body.subarray(
|
|
175
|
+
assertBodyLength(body.length, PARAMS_SIZE + expectedArrayLength * bpe, "fuse");
|
|
176
|
+
const laneBytes = body.subarray(PARAMS_SIZE);
|
|
175
177
|
const arrayLength = laneBytes.length / bpe;
|
|
176
178
|
const fp = bpe === 1 ? new Uint8Array(arrayLength) : new Uint16Array(arrayLength);
|
|
177
179
|
new Uint8Array(fp.buffer).set(laneBytes);
|
|
@@ -214,16 +216,17 @@ var BinaryFuse = class {
|
|
|
214
216
|
}
|
|
215
217
|
toBytes() {
|
|
216
218
|
const laneBytes = new Uint8Array(this.#fp.buffer, this.#fp.byteOffset, this.#fp.byteLength);
|
|
219
|
+
const type = this.#fp.BYTES_PER_ELEMENT === 1 ? TYPE_FUSE8 : TYPE_FUSE16;
|
|
217
220
|
return writeFrame({
|
|
218
|
-
version:
|
|
219
|
-
type
|
|
221
|
+
version: 5,
|
|
222
|
+
type,
|
|
220
223
|
flags: 0
|
|
221
|
-
},
|
|
224
|
+
}, PARAMS_SIZE, laneBytes.length, (body, dv) => {
|
|
222
225
|
dv.setUint32(0, this.#seed, true);
|
|
223
226
|
dv.setUint32(4, this.#seg, true);
|
|
224
227
|
dv.setUint32(8, this.#segCountLen, true);
|
|
225
228
|
dv.setUint32(12, this.#size, true);
|
|
226
|
-
body.set(laneBytes,
|
|
229
|
+
body.set(laneBytes, PARAMS_SIZE);
|
|
227
230
|
});
|
|
228
231
|
}
|
|
229
232
|
has(key) {
|
|
@@ -269,4 +272,4 @@ var BinaryFuse16 = class BinaryFuse16 extends BinaryFuse {
|
|
|
269
272
|
}
|
|
270
273
|
};
|
|
271
274
|
//#endregion
|
|
272
|
-
export { BadMagicError, BinaryFuse16, BinaryFuse8, BinaryFuseBuildError, ChecksumError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, fuseBitsPerKey };
|
|
275
|
+
export { BadMagicError, BinaryFuse16, BinaryFuse8, BinaryFuseBuildError, ChecksumError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, fuseBitsPerKey };
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/core/hasher.ts
|
|
10
|
+
let RLO = 0;
|
|
11
|
+
let RHI = 0;
|
|
12
|
+
const LANES = {
|
|
13
|
+
w0: 0,
|
|
14
|
+
w1: 0,
|
|
15
|
+
w2: 0,
|
|
16
|
+
w3: 0
|
|
17
|
+
};
|
|
18
|
+
function mul64(alo, ahi, blo, bhi) {
|
|
19
|
+
const a0 = alo & 65535;
|
|
20
|
+
const a1 = alo >>> 16;
|
|
21
|
+
const a2 = ahi & 65535;
|
|
22
|
+
const a3 = ahi >>> 16;
|
|
23
|
+
const b0 = blo & 65535;
|
|
24
|
+
const b1 = blo >>> 16;
|
|
25
|
+
const b2 = bhi & 65535;
|
|
26
|
+
const b3 = bhi >>> 16;
|
|
27
|
+
let c0 = a0 * b0;
|
|
28
|
+
let c1 = c0 >>> 16;
|
|
29
|
+
c0 &= 65535;
|
|
30
|
+
c1 += a1 * b0;
|
|
31
|
+
let c2 = c1 >>> 16;
|
|
32
|
+
c1 &= 65535;
|
|
33
|
+
c1 += a0 * b1;
|
|
34
|
+
c2 += c1 >>> 16;
|
|
35
|
+
c1 &= 65535;
|
|
36
|
+
c2 += a2 * b0;
|
|
37
|
+
let c3 = c2 >>> 16;
|
|
38
|
+
c2 &= 65535;
|
|
39
|
+
c2 += a1 * b1;
|
|
40
|
+
c3 += c2 >>> 16;
|
|
41
|
+
c2 &= 65535;
|
|
42
|
+
c2 += a0 * b2;
|
|
43
|
+
c3 += c2 >>> 16;
|
|
44
|
+
c2 &= 65535;
|
|
45
|
+
c3 += a3 * b0 + a2 * b1 + a1 * b2 + a0 * b3;
|
|
46
|
+
c3 &= 65535;
|
|
47
|
+
RLO = (c1 << 16 | c0) >>> 0;
|
|
48
|
+
RHI = (c3 << 16 | c2) >>> 0;
|
|
49
|
+
}
|
|
50
|
+
function fmix64(lo, hi) {
|
|
51
|
+
let l = lo;
|
|
52
|
+
let h = hi;
|
|
53
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
54
|
+
mul64(l, h, 3981806797, 4283543511);
|
|
55
|
+
l = RLO;
|
|
56
|
+
h = RHI;
|
|
57
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
58
|
+
mul64(l, h, 444984403, 3301882366);
|
|
59
|
+
l = RLO;
|
|
60
|
+
h = RHI;
|
|
61
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
62
|
+
RLO = l;
|
|
63
|
+
RHI = h;
|
|
64
|
+
}
|
|
65
|
+
function rotl32(x, r) {
|
|
66
|
+
return (x << r | x >>> 32 - r) >>> 0;
|
|
67
|
+
}
|
|
68
|
+
function fmix32(h) {
|
|
69
|
+
let x = h;
|
|
70
|
+
x ^= x >>> 16;
|
|
71
|
+
x = Math.imul(x, 2246822507);
|
|
72
|
+
x ^= x >>> 13;
|
|
73
|
+
x = Math.imul(x, 3266489909);
|
|
74
|
+
x ^= x >>> 16;
|
|
75
|
+
return x >>> 0;
|
|
76
|
+
}
|
|
77
|
+
const K1 = 597399067;
|
|
78
|
+
const K2 = 2869860233;
|
|
79
|
+
const K3 = 951274213;
|
|
80
|
+
const K4 = 2716044179;
|
|
81
|
+
function computeLanes(bytes, seed, len) {
|
|
82
|
+
const nblocks = len >>> 4;
|
|
83
|
+
let h1 = seed >>> 0;
|
|
84
|
+
let h2 = seed >>> 0;
|
|
85
|
+
let h3 = seed >>> 0;
|
|
86
|
+
let h4 = seed >>> 0;
|
|
87
|
+
for (let i = 0; i < nblocks; i++) {
|
|
88
|
+
const b = i * 16;
|
|
89
|
+
let k1 = ((bytes[b] ?? 0) | (bytes[b + 1] ?? 0) << 8 | (bytes[b + 2] ?? 0) << 16 | (bytes[b + 3] ?? 0) << 24) >>> 0;
|
|
90
|
+
let k2 = ((bytes[b + 4] ?? 0) | (bytes[b + 5] ?? 0) << 8 | (bytes[b + 6] ?? 0) << 16 | (bytes[b + 7] ?? 0) << 24) >>> 0;
|
|
91
|
+
let k3 = ((bytes[b + 8] ?? 0) | (bytes[b + 9] ?? 0) << 8 | (bytes[b + 10] ?? 0) << 16 | (bytes[b + 11] ?? 0) << 24) >>> 0;
|
|
92
|
+
let k4 = ((bytes[b + 12] ?? 0) | (bytes[b + 13] ?? 0) << 8 | (bytes[b + 14] ?? 0) << 16 | (bytes[b + 15] ?? 0) << 24) >>> 0;
|
|
93
|
+
k1 = Math.imul(rotl32(Math.imul(k1, K1), 15), K2);
|
|
94
|
+
h1 ^= k1;
|
|
95
|
+
h1 = rotl32(h1, 19);
|
|
96
|
+
h1 = h1 + h2 >>> 0;
|
|
97
|
+
h1 = Math.imul(h1, 5) + 1444728091 >>> 0;
|
|
98
|
+
k2 = Math.imul(rotl32(Math.imul(k2, K2), 16), K3);
|
|
99
|
+
h2 ^= k2;
|
|
100
|
+
h2 = rotl32(h2, 17);
|
|
101
|
+
h2 = h2 + h3 >>> 0;
|
|
102
|
+
h2 = Math.imul(h2, 5) + 197830471 >>> 0;
|
|
103
|
+
k3 = Math.imul(rotl32(Math.imul(k3, K3), 17), K4);
|
|
104
|
+
h3 ^= k3;
|
|
105
|
+
h3 = rotl32(h3, 15);
|
|
106
|
+
h3 = h3 + h4 >>> 0;
|
|
107
|
+
h3 = Math.imul(h3, 5) + 2530024501 >>> 0;
|
|
108
|
+
k4 = Math.imul(rotl32(Math.imul(k4, K4), 18), K1);
|
|
109
|
+
h4 ^= k4;
|
|
110
|
+
h4 = rotl32(h4, 13);
|
|
111
|
+
h4 = h4 + h1 >>> 0;
|
|
112
|
+
h4 = Math.imul(h4, 5) + 850148119 >>> 0;
|
|
113
|
+
}
|
|
114
|
+
let k1 = 0;
|
|
115
|
+
let k2 = 0;
|
|
116
|
+
let k3 = 0;
|
|
117
|
+
let k4 = 0;
|
|
118
|
+
const tail = nblocks * 16;
|
|
119
|
+
const rem = len & 15;
|
|
120
|
+
if (rem >= 15) k4 ^= (bytes[tail + 14] ?? 0) << 16;
|
|
121
|
+
if (rem >= 14) k4 ^= (bytes[tail + 13] ?? 0) << 8;
|
|
122
|
+
if (rem >= 13) {
|
|
123
|
+
k4 ^= bytes[tail + 12] ?? 0;
|
|
124
|
+
k4 = Math.imul(rotl32(Math.imul(k4, K4), 18), K1);
|
|
125
|
+
h4 ^= k4;
|
|
126
|
+
}
|
|
127
|
+
if (rem >= 12) k3 ^= (bytes[tail + 11] ?? 0) << 24;
|
|
128
|
+
if (rem >= 11) k3 ^= (bytes[tail + 10] ?? 0) << 16;
|
|
129
|
+
if (rem >= 10) k3 ^= (bytes[tail + 9] ?? 0) << 8;
|
|
130
|
+
if (rem >= 9) {
|
|
131
|
+
k3 ^= bytes[tail + 8] ?? 0;
|
|
132
|
+
k3 = Math.imul(rotl32(Math.imul(k3, K3), 17), K4);
|
|
133
|
+
h3 ^= k3;
|
|
134
|
+
}
|
|
135
|
+
if (rem >= 8) k2 ^= (bytes[tail + 7] ?? 0) << 24;
|
|
136
|
+
if (rem >= 7) k2 ^= (bytes[tail + 6] ?? 0) << 16;
|
|
137
|
+
if (rem >= 6) k2 ^= (bytes[tail + 5] ?? 0) << 8;
|
|
138
|
+
if (rem >= 5) {
|
|
139
|
+
k2 ^= bytes[tail + 4] ?? 0;
|
|
140
|
+
k2 = Math.imul(rotl32(Math.imul(k2, K2), 16), K3);
|
|
141
|
+
h2 ^= k2;
|
|
142
|
+
}
|
|
143
|
+
if (rem >= 4) k1 ^= (bytes[tail + 3] ?? 0) << 24;
|
|
144
|
+
if (rem >= 3) k1 ^= (bytes[tail + 2] ?? 0) << 16;
|
|
145
|
+
if (rem >= 2) k1 ^= (bytes[tail + 1] ?? 0) << 8;
|
|
146
|
+
if (rem >= 1) {
|
|
147
|
+
k1 ^= bytes[tail] ?? 0;
|
|
148
|
+
k1 = Math.imul(rotl32(Math.imul(k1, K1), 15), K2);
|
|
149
|
+
h1 ^= k1;
|
|
150
|
+
}
|
|
151
|
+
h1 = (h1 ^ len) >>> 0;
|
|
152
|
+
h2 = (h2 ^ len) >>> 0;
|
|
153
|
+
h3 = (h3 ^ len) >>> 0;
|
|
154
|
+
h4 = (h4 ^ len) >>> 0;
|
|
155
|
+
h1 = h1 + h2 >>> 0;
|
|
156
|
+
h1 = h1 + h3 >>> 0;
|
|
157
|
+
h1 = h1 + h4 >>> 0;
|
|
158
|
+
h2 = h2 + h1 >>> 0;
|
|
159
|
+
h3 = h3 + h1 >>> 0;
|
|
160
|
+
h4 = h4 + h1 >>> 0;
|
|
161
|
+
h1 = fmix32(h1);
|
|
162
|
+
h2 = fmix32(h2);
|
|
163
|
+
h3 = fmix32(h3);
|
|
164
|
+
h4 = fmix32(h4);
|
|
165
|
+
h1 = h1 + h2 >>> 0;
|
|
166
|
+
h1 = h1 + h3 >>> 0;
|
|
167
|
+
h1 = h1 + h4 >>> 0;
|
|
168
|
+
h2 = h2 + h1 >>> 0;
|
|
169
|
+
h3 = h3 + h1 >>> 0;
|
|
170
|
+
h4 = h4 + h1 >>> 0;
|
|
171
|
+
LANES.w0 = h1;
|
|
172
|
+
LANES.w1 = h2;
|
|
173
|
+
LANES.w2 = h3;
|
|
174
|
+
LANES.w3 = h4;
|
|
175
|
+
}
|
|
176
|
+
const keyEncoder = new TextEncoder();
|
|
177
|
+
let keyBuf = new Uint8Array(256);
|
|
178
|
+
let encBytes = keyBuf;
|
|
179
|
+
let encLen = 0;
|
|
180
|
+
function encodeKey(key) {
|
|
181
|
+
if (typeof key === "string") {
|
|
182
|
+
const cap = key.length * 3;
|
|
183
|
+
if (keyBuf.length < cap) keyBuf = new Uint8Array(cap);
|
|
184
|
+
encLen = keyEncoder.encodeInto(key, keyBuf).written;
|
|
185
|
+
encBytes = keyBuf;
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
encBytes = normalize(key);
|
|
189
|
+
encLen = encBytes.length;
|
|
190
|
+
}
|
|
191
|
+
function keyToLanes(key, seed) {
|
|
192
|
+
encodeKey(key);
|
|
193
|
+
computeLanes(encBytes, seed, encLen);
|
|
194
|
+
}
|
|
195
|
+
function hash128KeyInto(key, seed, out) {
|
|
196
|
+
keyToLanes(key, seed);
|
|
197
|
+
out.w0 = LANES.w0;
|
|
198
|
+
out.w1 = LANES.w1;
|
|
199
|
+
out.w2 = LANES.w2;
|
|
200
|
+
out.w3 = LANES.w3;
|
|
201
|
+
}
|
|
202
|
+
function reduce(x, range) {
|
|
203
|
+
const xlo = x & 65535;
|
|
204
|
+
const xhi = x >>> 16;
|
|
205
|
+
const rlo = range & 65535;
|
|
206
|
+
const rhi = range >>> 16;
|
|
207
|
+
const lolo = xlo * rlo;
|
|
208
|
+
const lohi = xlo * rhi;
|
|
209
|
+
const hilo = xhi * rlo;
|
|
210
|
+
const hihi = xhi * rhi;
|
|
211
|
+
const carry = (lolo >>> 16) + (lohi & 65535) + (hilo & 65535);
|
|
212
|
+
return hihi + (lohi >>> 16) + (hilo >>> 16) + (carry >>> 16) >>> 0;
|
|
213
|
+
}
|
|
214
|
+
function hash32x2Into(key, seed, out) {
|
|
215
|
+
keyToLanes(key, seed);
|
|
216
|
+
out[0] = LANES.w0;
|
|
217
|
+
out[1] = LANES.w1;
|
|
218
|
+
}
|
|
219
|
+
function probeInto(key, count, range, seed, out) {
|
|
220
|
+
keyToLanes(key, seed);
|
|
221
|
+
const a = LANES.w0;
|
|
222
|
+
const b = LANES.w1;
|
|
223
|
+
for (let i = 0; i < count; i++) {
|
|
224
|
+
const x = a + Math.imul(i, b) + i * i >>> 0;
|
|
225
|
+
out[i] = reduce(x, range);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
//#endregion
|
|
229
|
+
Object.defineProperty(exports, "RHI", {
|
|
230
|
+
enumerable: true,
|
|
231
|
+
get: function() {
|
|
232
|
+
return RHI;
|
|
233
|
+
}
|
|
234
|
+
});
|
|
235
|
+
Object.defineProperty(exports, "RLO", {
|
|
236
|
+
enumerable: true,
|
|
237
|
+
get: function() {
|
|
238
|
+
return RLO;
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
Object.defineProperty(exports, "fmix64", {
|
|
242
|
+
enumerable: true,
|
|
243
|
+
get: function() {
|
|
244
|
+
return fmix64;
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
Object.defineProperty(exports, "hash128KeyInto", {
|
|
248
|
+
enumerable: true,
|
|
249
|
+
get: function() {
|
|
250
|
+
return hash128KeyInto;
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
Object.defineProperty(exports, "hash32x2Into", {
|
|
254
|
+
enumerable: true,
|
|
255
|
+
get: function() {
|
|
256
|
+
return hash32x2Into;
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
Object.defineProperty(exports, "mul64", {
|
|
260
|
+
enumerable: true,
|
|
261
|
+
get: function() {
|
|
262
|
+
return mul64;
|
|
263
|
+
}
|
|
264
|
+
});
|
|
265
|
+
Object.defineProperty(exports, "probeInto", {
|
|
266
|
+
enumerable: true,
|
|
267
|
+
get: function() {
|
|
268
|
+
return probeInto;
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
Object.defineProperty(exports, "reduce", {
|
|
272
|
+
enumerable: true,
|
|
273
|
+
get: function() {
|
|
274
|
+
return reduce;
|
|
275
|
+
}
|
|
276
|
+
});
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/core/hasher.ts
|
|
10
|
+
let RLO = 0;
|
|
11
|
+
let RHI = 0;
|
|
12
|
+
const LANES = {
|
|
13
|
+
w0: 0,
|
|
14
|
+
w1: 0,
|
|
15
|
+
w2: 0,
|
|
16
|
+
w3: 0
|
|
17
|
+
};
|
|
18
|
+
function mul64(alo, ahi, blo, bhi) {
|
|
19
|
+
const a0 = alo & 65535;
|
|
20
|
+
const a1 = alo >>> 16;
|
|
21
|
+
const a2 = ahi & 65535;
|
|
22
|
+
const a3 = ahi >>> 16;
|
|
23
|
+
const b0 = blo & 65535;
|
|
24
|
+
const b1 = blo >>> 16;
|
|
25
|
+
const b2 = bhi & 65535;
|
|
26
|
+
const b3 = bhi >>> 16;
|
|
27
|
+
let c0 = a0 * b0;
|
|
28
|
+
let c1 = c0 >>> 16;
|
|
29
|
+
c0 &= 65535;
|
|
30
|
+
c1 += a1 * b0;
|
|
31
|
+
let c2 = c1 >>> 16;
|
|
32
|
+
c1 &= 65535;
|
|
33
|
+
c1 += a0 * b1;
|
|
34
|
+
c2 += c1 >>> 16;
|
|
35
|
+
c1 &= 65535;
|
|
36
|
+
c2 += a2 * b0;
|
|
37
|
+
let c3 = c2 >>> 16;
|
|
38
|
+
c2 &= 65535;
|
|
39
|
+
c2 += a1 * b1;
|
|
40
|
+
c3 += c2 >>> 16;
|
|
41
|
+
c2 &= 65535;
|
|
42
|
+
c2 += a0 * b2;
|
|
43
|
+
c3 += c2 >>> 16;
|
|
44
|
+
c2 &= 65535;
|
|
45
|
+
c3 += a3 * b0 + a2 * b1 + a1 * b2 + a0 * b3;
|
|
46
|
+
c3 &= 65535;
|
|
47
|
+
RLO = (c1 << 16 | c0) >>> 0;
|
|
48
|
+
RHI = (c3 << 16 | c2) >>> 0;
|
|
49
|
+
}
|
|
50
|
+
function fmix64(lo, hi) {
|
|
51
|
+
let l = lo;
|
|
52
|
+
let h = hi;
|
|
53
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
54
|
+
mul64(l, h, 3981806797, 4283543511);
|
|
55
|
+
l = RLO;
|
|
56
|
+
h = RHI;
|
|
57
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
58
|
+
mul64(l, h, 444984403, 3301882366);
|
|
59
|
+
l = RLO;
|
|
60
|
+
h = RHI;
|
|
61
|
+
l = (l ^ h >>> 1) >>> 0;
|
|
62
|
+
RLO = l;
|
|
63
|
+
RHI = h;
|
|
64
|
+
}
|
|
65
|
+
function rotl32(x, r) {
|
|
66
|
+
return (x << r | x >>> 32 - r) >>> 0;
|
|
67
|
+
}
|
|
68
|
+
function fmix32(h) {
|
|
69
|
+
let x = h;
|
|
70
|
+
x ^= x >>> 16;
|
|
71
|
+
x = Math.imul(x, 2246822507);
|
|
72
|
+
x ^= x >>> 13;
|
|
73
|
+
x = Math.imul(x, 3266489909);
|
|
74
|
+
x ^= x >>> 16;
|
|
75
|
+
return x >>> 0;
|
|
76
|
+
}
|
|
77
|
+
const K1 = 597399067;
|
|
78
|
+
const K2 = 2869860233;
|
|
79
|
+
const K3 = 951274213;
|
|
80
|
+
const K4 = 2716044179;
|
|
81
|
+
function computeLanes(bytes, seed, len) {
|
|
82
|
+
const nblocks = len >>> 4;
|
|
83
|
+
let h1 = seed >>> 0;
|
|
84
|
+
let h2 = seed >>> 0;
|
|
85
|
+
let h3 = seed >>> 0;
|
|
86
|
+
let h4 = seed >>> 0;
|
|
87
|
+
for (let i = 0; i < nblocks; i++) {
|
|
88
|
+
const b = i * 16;
|
|
89
|
+
let k1 = ((bytes[b] ?? 0) | (bytes[b + 1] ?? 0) << 8 | (bytes[b + 2] ?? 0) << 16 | (bytes[b + 3] ?? 0) << 24) >>> 0;
|
|
90
|
+
let k2 = ((bytes[b + 4] ?? 0) | (bytes[b + 5] ?? 0) << 8 | (bytes[b + 6] ?? 0) << 16 | (bytes[b + 7] ?? 0) << 24) >>> 0;
|
|
91
|
+
let k3 = ((bytes[b + 8] ?? 0) | (bytes[b + 9] ?? 0) << 8 | (bytes[b + 10] ?? 0) << 16 | (bytes[b + 11] ?? 0) << 24) >>> 0;
|
|
92
|
+
let k4 = ((bytes[b + 12] ?? 0) | (bytes[b + 13] ?? 0) << 8 | (bytes[b + 14] ?? 0) << 16 | (bytes[b + 15] ?? 0) << 24) >>> 0;
|
|
93
|
+
k1 = Math.imul(rotl32(Math.imul(k1, K1), 15), K2);
|
|
94
|
+
h1 ^= k1;
|
|
95
|
+
h1 = rotl32(h1, 19);
|
|
96
|
+
h1 = h1 + h2 >>> 0;
|
|
97
|
+
h1 = Math.imul(h1, 5) + 1444728091 >>> 0;
|
|
98
|
+
k2 = Math.imul(rotl32(Math.imul(k2, K2), 16), K3);
|
|
99
|
+
h2 ^= k2;
|
|
100
|
+
h2 = rotl32(h2, 17);
|
|
101
|
+
h2 = h2 + h3 >>> 0;
|
|
102
|
+
h2 = Math.imul(h2, 5) + 197830471 >>> 0;
|
|
103
|
+
k3 = Math.imul(rotl32(Math.imul(k3, K3), 17), K4);
|
|
104
|
+
h3 ^= k3;
|
|
105
|
+
h3 = rotl32(h3, 15);
|
|
106
|
+
h3 = h3 + h4 >>> 0;
|
|
107
|
+
h3 = Math.imul(h3, 5) + 2530024501 >>> 0;
|
|
108
|
+
k4 = Math.imul(rotl32(Math.imul(k4, K4), 18), K1);
|
|
109
|
+
h4 ^= k4;
|
|
110
|
+
h4 = rotl32(h4, 13);
|
|
111
|
+
h4 = h4 + h1 >>> 0;
|
|
112
|
+
h4 = Math.imul(h4, 5) + 850148119 >>> 0;
|
|
113
|
+
}
|
|
114
|
+
let k1 = 0;
|
|
115
|
+
let k2 = 0;
|
|
116
|
+
let k3 = 0;
|
|
117
|
+
let k4 = 0;
|
|
118
|
+
const tail = nblocks * 16;
|
|
119
|
+
const rem = len & 15;
|
|
120
|
+
if (rem >= 15) k4 ^= (bytes[tail + 14] ?? 0) << 16;
|
|
121
|
+
if (rem >= 14) k4 ^= (bytes[tail + 13] ?? 0) << 8;
|
|
122
|
+
if (rem >= 13) {
|
|
123
|
+
k4 ^= bytes[tail + 12] ?? 0;
|
|
124
|
+
k4 = Math.imul(rotl32(Math.imul(k4, K4), 18), K1);
|
|
125
|
+
h4 ^= k4;
|
|
126
|
+
}
|
|
127
|
+
if (rem >= 12) k3 ^= (bytes[tail + 11] ?? 0) << 24;
|
|
128
|
+
if (rem >= 11) k3 ^= (bytes[tail + 10] ?? 0) << 16;
|
|
129
|
+
if (rem >= 10) k3 ^= (bytes[tail + 9] ?? 0) << 8;
|
|
130
|
+
if (rem >= 9) {
|
|
131
|
+
k3 ^= bytes[tail + 8] ?? 0;
|
|
132
|
+
k3 = Math.imul(rotl32(Math.imul(k3, K3), 17), K4);
|
|
133
|
+
h3 ^= k3;
|
|
134
|
+
}
|
|
135
|
+
if (rem >= 8) k2 ^= (bytes[tail + 7] ?? 0) << 24;
|
|
136
|
+
if (rem >= 7) k2 ^= (bytes[tail + 6] ?? 0) << 16;
|
|
137
|
+
if (rem >= 6) k2 ^= (bytes[tail + 5] ?? 0) << 8;
|
|
138
|
+
if (rem >= 5) {
|
|
139
|
+
k2 ^= bytes[tail + 4] ?? 0;
|
|
140
|
+
k2 = Math.imul(rotl32(Math.imul(k2, K2), 16), K3);
|
|
141
|
+
h2 ^= k2;
|
|
142
|
+
}
|
|
143
|
+
if (rem >= 4) k1 ^= (bytes[tail + 3] ?? 0) << 24;
|
|
144
|
+
if (rem >= 3) k1 ^= (bytes[tail + 2] ?? 0) << 16;
|
|
145
|
+
if (rem >= 2) k1 ^= (bytes[tail + 1] ?? 0) << 8;
|
|
146
|
+
if (rem >= 1) {
|
|
147
|
+
k1 ^= bytes[tail] ?? 0;
|
|
148
|
+
k1 = Math.imul(rotl32(Math.imul(k1, K1), 15), K2);
|
|
149
|
+
h1 ^= k1;
|
|
150
|
+
}
|
|
151
|
+
h1 = (h1 ^ len) >>> 0;
|
|
152
|
+
h2 = (h2 ^ len) >>> 0;
|
|
153
|
+
h3 = (h3 ^ len) >>> 0;
|
|
154
|
+
h4 = (h4 ^ len) >>> 0;
|
|
155
|
+
h1 = h1 + h2 >>> 0;
|
|
156
|
+
h1 = h1 + h3 >>> 0;
|
|
157
|
+
h1 = h1 + h4 >>> 0;
|
|
158
|
+
h2 = h2 + h1 >>> 0;
|
|
159
|
+
h3 = h3 + h1 >>> 0;
|
|
160
|
+
h4 = h4 + h1 >>> 0;
|
|
161
|
+
h1 = fmix32(h1);
|
|
162
|
+
h2 = fmix32(h2);
|
|
163
|
+
h3 = fmix32(h3);
|
|
164
|
+
h4 = fmix32(h4);
|
|
165
|
+
h1 = h1 + h2 >>> 0;
|
|
166
|
+
h1 = h1 + h3 >>> 0;
|
|
167
|
+
h1 = h1 + h4 >>> 0;
|
|
168
|
+
h2 = h2 + h1 >>> 0;
|
|
169
|
+
h3 = h3 + h1 >>> 0;
|
|
170
|
+
h4 = h4 + h1 >>> 0;
|
|
171
|
+
LANES.w0 = h1;
|
|
172
|
+
LANES.w1 = h2;
|
|
173
|
+
LANES.w2 = h3;
|
|
174
|
+
LANES.w3 = h4;
|
|
175
|
+
}
|
|
176
|
+
const keyEncoder = new TextEncoder();
|
|
177
|
+
let keyBuf = new Uint8Array(256);
|
|
178
|
+
let encBytes = keyBuf;
|
|
179
|
+
let encLen = 0;
|
|
180
|
+
function encodeKey(key) {
|
|
181
|
+
if (typeof key === "string") {
|
|
182
|
+
const cap = key.length * 3;
|
|
183
|
+
if (keyBuf.length < cap) keyBuf = new Uint8Array(cap);
|
|
184
|
+
encLen = keyEncoder.encodeInto(key, keyBuf).written;
|
|
185
|
+
encBytes = keyBuf;
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
encBytes = normalize(key);
|
|
189
|
+
encLen = encBytes.length;
|
|
190
|
+
}
|
|
191
|
+
function keyToLanes(key, seed) {
|
|
192
|
+
encodeKey(key);
|
|
193
|
+
computeLanes(encBytes, seed, encLen);
|
|
194
|
+
}
|
|
195
|
+
function hash128KeyInto(key, seed, out) {
|
|
196
|
+
keyToLanes(key, seed);
|
|
197
|
+
out.w0 = LANES.w0;
|
|
198
|
+
out.w1 = LANES.w1;
|
|
199
|
+
out.w2 = LANES.w2;
|
|
200
|
+
out.w3 = LANES.w3;
|
|
201
|
+
}
|
|
202
|
+
function reduce(x, range) {
|
|
203
|
+
const xlo = x & 65535;
|
|
204
|
+
const xhi = x >>> 16;
|
|
205
|
+
const rlo = range & 65535;
|
|
206
|
+
const rhi = range >>> 16;
|
|
207
|
+
const lolo = xlo * rlo;
|
|
208
|
+
const lohi = xlo * rhi;
|
|
209
|
+
const hilo = xhi * rlo;
|
|
210
|
+
const hihi = xhi * rhi;
|
|
211
|
+
const carry = (lolo >>> 16) + (lohi & 65535) + (hilo & 65535);
|
|
212
|
+
return hihi + (lohi >>> 16) + (hilo >>> 16) + (carry >>> 16) >>> 0;
|
|
213
|
+
}
|
|
214
|
+
function hash32x2Into(key, seed, out) {
|
|
215
|
+
keyToLanes(key, seed);
|
|
216
|
+
out[0] = LANES.w0;
|
|
217
|
+
out[1] = LANES.w1;
|
|
218
|
+
}
|
|
219
|
+
function probeInto(key, count, range, seed, out) {
|
|
220
|
+
keyToLanes(key, seed);
|
|
221
|
+
const a = LANES.w0;
|
|
222
|
+
const b = LANES.w1;
|
|
223
|
+
for (let i = 0; i < count; i++) {
|
|
224
|
+
const x = a + Math.imul(i, b) + i * i >>> 0;
|
|
225
|
+
out[i] = reduce(x, range);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
//#endregion
|
|
229
|
+
export { hash32x2Into as a, reduce as c, hash128KeyInto as i, RLO as n, mul64 as o, fmix64 as r, probeInto as s, RHI as t };
|
package/dist/hll/index.cjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
2
|
+
const require_hasher = require("../hasher-D8LCBTOM.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
3
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
4
5
|
const require_sizing = require("../sizing-BLzZk2zr.cjs");
|
|
5
6
|
//#region src/hll/estimate.ts
|
|
@@ -100,7 +101,7 @@ var Registers = class {
|
|
|
100
101
|
}
|
|
101
102
|
};
|
|
102
103
|
const RHO_BITS = 6;
|
|
103
|
-
const RHO_MASK =
|
|
104
|
+
const RHO_MASK = 63;
|
|
104
105
|
function encodeSparse(index, rho) {
|
|
105
106
|
return index << RHO_BITS | rho;
|
|
106
107
|
}
|
|
@@ -138,7 +139,8 @@ function compact(buf, len) {
|
|
|
138
139
|
//#region src/hll/hll.ts
|
|
139
140
|
const ERROR_CONSTANT = 1.04;
|
|
140
141
|
const TYPE = 5;
|
|
141
|
-
const PARAMS_SIZE =
|
|
142
|
+
const PARAMS_SIZE = 8;
|
|
143
|
+
const PARAMS_FIELDS_END = 6;
|
|
142
144
|
const DENSE = 0;
|
|
143
145
|
const SPARSE = 1;
|
|
144
146
|
const ENTRY_SIZE = 4;
|
|
@@ -181,6 +183,7 @@ var HyperLogLog = class HyperLogLog {
|
|
|
181
183
|
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
182
184
|
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
183
185
|
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "hll");
|
|
186
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "hll");
|
|
184
187
|
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
185
188
|
const p = body[0] ?? 0;
|
|
186
189
|
const encoding = body[1] ?? 0;
|
|
@@ -221,10 +224,10 @@ var HyperLogLog = class HyperLogLog {
|
|
|
221
224
|
const entries = compact(sparse, this.#len);
|
|
222
225
|
this.#len = entries;
|
|
223
226
|
return require_serialize.writeFrame({
|
|
224
|
-
version:
|
|
227
|
+
version: 5,
|
|
225
228
|
type: TYPE,
|
|
226
229
|
flags: 0
|
|
227
|
-
}, PARAMS_SIZE
|
|
230
|
+
}, PARAMS_SIZE, entries * ENTRY_SIZE, (body, dv) => {
|
|
228
231
|
body[0] = this.#p;
|
|
229
232
|
body[1] = SPARSE;
|
|
230
233
|
dv.setUint32(2, this.#seed, true);
|
|
@@ -233,10 +236,10 @@ var HyperLogLog = class HyperLogLog {
|
|
|
233
236
|
}
|
|
234
237
|
const payload = this.#registers.bytes;
|
|
235
238
|
return require_serialize.writeFrame({
|
|
236
|
-
version:
|
|
239
|
+
version: 5,
|
|
237
240
|
type: TYPE,
|
|
238
241
|
flags: 0
|
|
239
|
-
}, PARAMS_SIZE
|
|
242
|
+
}, PARAMS_SIZE, payload.length, (body, dv) => {
|
|
240
243
|
body[0] = this.#p;
|
|
241
244
|
body[1] = DENSE;
|
|
242
245
|
dv.setUint32(2, this.#seed, true);
|
|
@@ -260,7 +263,7 @@ var HyperLogLog = class HyperLogLog {
|
|
|
260
263
|
}
|
|
261
264
|
add(key) {
|
|
262
265
|
const s = this.#scratch;
|
|
263
|
-
|
|
266
|
+
require_hasher.hash128KeyInto(key, this.#seed, s);
|
|
264
267
|
const p = this.#p;
|
|
265
268
|
const tail = s.w0 << p >>> 0;
|
|
266
269
|
const rho = tail !== 0 ? Math.clz32(tail) + 1 : 32 - p + Math.clz32(s.w1) + 1;
|
|
@@ -355,6 +358,7 @@ exports.BadMagicError = require_serialize.BadMagicError;
|
|
|
355
358
|
exports.ChecksumError = require_serialize.ChecksumError;
|
|
356
359
|
exports.HyperLogLog = HyperLogLog;
|
|
357
360
|
exports.ParamError = require_params.ParamError;
|
|
361
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
358
362
|
exports.SerializationError = require_serialize.SerializationError;
|
|
359
363
|
exports.TruncatedError = require_serialize.TruncatedError;
|
|
360
364
|
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|