distillate 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -4
- package/dist/blocked/index.cjs +1 -1
- package/dist/blocked/index.js +1 -1
- package/dist/bloom/index.cjs +6 -4
- package/dist/bloom/index.js +6 -4
- package/dist/bytes-DWGupBUE.cjs +45 -0
- package/dist/bytes-oOWlu7N5.js +34 -0
- package/dist/countmin/index.cjs +1 -1
- package/dist/countmin/index.js +1 -1
- package/dist/cuckoo/index.cjs +1 -1
- package/dist/cuckoo/index.js +1 -1
- package/dist/ddsketch/index.cjs +23 -0
- package/dist/ddsketch/index.d.cts +4 -0
- package/dist/ddsketch/index.d.ts +4 -0
- package/dist/ddsketch/index.js +4 -0
- package/dist/ddsketch/protobuf/index.cjs +297 -0
- package/dist/ddsketch/protobuf/index.d.cts +43 -0
- package/dist/ddsketch/protobuf/index.d.ts +43 -0
- package/dist/ddsketch/protobuf/index.js +294 -0
- package/dist/ddsketch-1Buc9M3P.d.cts +293 -0
- package/dist/ddsketch-Ao9louSV.cjs +930 -0
- package/dist/ddsketch-CvwWk86v.d.ts +293 -0
- package/dist/ddsketch-D3Sy8CAU.js +841 -0
- package/dist/fuse/index.cjs +1 -1
- package/dist/fuse/index.js +1 -1
- package/dist/{hasher-DOdmZ-LI.js → hasher-BQxNrCQC.js} +3 -18
- package/dist/{hasher-DPXmP7SH.cjs → hasher-CKLP9Oqe.cjs} +3 -18
- package/dist/hll/index.cjs +1 -1
- package/dist/hll/index.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +1 -1
- package/dist/scalable/index.js +1 -1
- package/dist/topk/index.cjs +19 -20
- package/dist/topk/index.js +19 -20
- package/package.json +17 -1
- package/dist/bytes-Be7tq1F7.js +0 -9
- package/dist/bytes-D2duG1jq.cjs +0 -14
package/README.md
CHANGED
|
@@ -52,12 +52,13 @@ Each structure ships as its own subpath, so you only bundle what you import.
|
|
|
52
52
|
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
53
53
|
| `distillate/countmin` | Count-Min | how many times? | Counting events per key in fixed space |
|
|
54
54
|
| `distillate/topk` | Top-K | which are heaviest? | Finding the most frequent keys in fixed space |
|
|
55
|
+
| `distillate/ddsketch` | DDSketch | what is the p99? | Latency and size percentiles, mergeable, in fixed space |
|
|
55
56
|
|
|
56
57
|
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
57
|
-
key set. HyperLogLog, Count-Min
|
|
58
|
-
none reports whether it saw any particular key. HyperLogLog counts
|
|
59
|
-
keys, Count-Min counts how often each key appeared,
|
|
60
|
-
keys appeared most.
|
|
58
|
+
key set. HyperLogLog, Count-Min, Top-K and DDSketch are sketches rather than
|
|
59
|
+
filters: none reports whether it saw any particular key. HyperLogLog counts
|
|
60
|
+
distinct keys, Count-Min counts how often each key appeared, Top-K reports which
|
|
61
|
+
keys appeared most, and DDSketch reports quantiles of numbers rather than keys.
|
|
61
62
|
|
|
62
63
|
### Classic Bloom (`distillate/bloom`)
|
|
63
64
|
|
|
@@ -215,6 +216,24 @@ Keys come back as `Uint8Array`, exactly as recorded, so a serialized Top-K sketc
|
|
|
215
216
|
|
|
216
217
|
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `total`, `error()`, `epsilon`, `capacity`, `topKSizing(epsilon)`, and a low-level `new TopK({ capacity, seed })`.
|
|
217
218
|
|
|
219
|
+
### DDSketch (`distillate/ddsketch`)
|
|
220
|
+
|
|
221
|
+
A **sketch** of numbers rather than keys: every quantile it reports is within a relative error of the true value, tails included, and two sketches merge exactly. Full API, mappings, stores and the bound: [DDSketch guide](https://distillate.akxp.net/guides/ddsketch/).
|
|
222
|
+
|
|
223
|
+
```ts
|
|
224
|
+
import { DDSketch } from "distillate/ddsketch";
|
|
225
|
+
|
|
226
|
+
const latency = DDSketch.create(0.01); // within 1%
|
|
227
|
+
for (let ms = 1; ms <= 1000; ms++) latency.add(ms);
|
|
228
|
+
|
|
229
|
+
latency.quantile(0.99); // ~992
|
|
230
|
+
latency.max; // 1000
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
By default it bins with the bitwise mapping Datadog's `DDSketch.fast()` uses, about 1.75 times as fast as the log mapping, and keeps at most 4096 bins per sign, enough for about 19 orders of magnitude at 1%; at a tighter accuracy, raise `maxBins` through `collapsingLowestStore`. `distillate/ddsketch/protobuf` converts to and from the protobuf message Datadog's libraries exchange.
|
|
234
|
+
|
|
235
|
+
Also: `add(value, count)`, `quantiles`, `count`, `sum`, `min`, `union`, `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, and the `bitwiseMapping`, `logMapping`, `linearMapping`, `quadraticMapping`, `cubicMapping`, `denseStore`, `sparseStore`, `collapsingLowestStore` and `collapsingHighestStore` factories.
|
|
236
|
+
|
|
218
237
|
## Performance
|
|
219
238
|
|
|
220
239
|
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (cross-library harness, node v24.14.1, Apple M5):
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-CKLP9Oqe.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
5
|
//#region src/blocked/blocked.ts
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as hash32x2Into, u as reduce } from "../hasher-
|
|
1
|
+
import { a as hash32x2Into, u as reduce } from "../hasher-BQxNrCQC.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
4
|
//#region src/blocked/blocked.ts
|
package/dist/bloom/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_sizing = require("../sizing-Ch0SDdza.cjs");
|
|
3
|
-
const require_hasher = require("../hasher-
|
|
3
|
+
const require_hasher = require("../hasher-CKLP9Oqe.cjs");
|
|
4
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
5
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
6
6
|
//#region src/bloom/bloom.ts
|
|
@@ -67,7 +67,7 @@ var BloomFilter = class BloomFilter {
|
|
|
67
67
|
this.#m = m;
|
|
68
68
|
this.#k = k;
|
|
69
69
|
this.#seed = seed;
|
|
70
|
-
this.#scratch = new Uint32Array(k);
|
|
70
|
+
this.#scratch = new Uint32Array(Math.max(k, 2));
|
|
71
71
|
this.#n = n ?? Math.max(1, Math.round(m * Math.LN2 / k));
|
|
72
72
|
}
|
|
73
73
|
get m() {
|
|
@@ -131,8 +131,10 @@ var BloomFilter = class BloomFilter {
|
|
|
131
131
|
for (let i = 0; i < this.#k; i++) this.#bits.set(this.#scratch[i] ?? 0);
|
|
132
132
|
}
|
|
133
133
|
has(key) {
|
|
134
|
-
require_hasher.
|
|
135
|
-
|
|
134
|
+
require_hasher.hash32x2Into(key, this.#seed, this.#scratch);
|
|
135
|
+
const a = this.#scratch[0] ?? 0;
|
|
136
|
+
const b = this.#scratch[1] ?? 0;
|
|
137
|
+
for (let i = 0; i < this.#k; i++) if (!this.#bits.get(require_hasher.probeAt(a, b, i, this.#m))) return false;
|
|
136
138
|
return true;
|
|
137
139
|
}
|
|
138
140
|
};
|
package/dist/bloom/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as BitSet, t as bloomSizing } from "../sizing-zxCBlyhO.js";
|
|
2
|
-
import { c as probeInto } from "../hasher-
|
|
2
|
+
import { a as hash32x2Into, c as probeInto, s as probeAt } from "../hasher-BQxNrCQC.js";
|
|
3
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
4
4
|
import { a as assertUint16, i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
5
5
|
//#region src/bloom/bloom.ts
|
|
@@ -66,7 +66,7 @@ var BloomFilter = class BloomFilter {
|
|
|
66
66
|
this.#m = m;
|
|
67
67
|
this.#k = k;
|
|
68
68
|
this.#seed = seed;
|
|
69
|
-
this.#scratch = new Uint32Array(k);
|
|
69
|
+
this.#scratch = new Uint32Array(Math.max(k, 2));
|
|
70
70
|
this.#n = n ?? Math.max(1, Math.round(m * Math.LN2 / k));
|
|
71
71
|
}
|
|
72
72
|
get m() {
|
|
@@ -130,8 +130,10 @@ var BloomFilter = class BloomFilter {
|
|
|
130
130
|
for (let i = 0; i < this.#k; i++) this.#bits.set(this.#scratch[i] ?? 0);
|
|
131
131
|
}
|
|
132
132
|
has(key) {
|
|
133
|
-
|
|
134
|
-
|
|
133
|
+
hash32x2Into(key, this.#seed, this.#scratch);
|
|
134
|
+
const a = this.#scratch[0] ?? 0;
|
|
135
|
+
const b = this.#scratch[1] ?? 0;
|
|
136
|
+
for (let i = 0; i < this.#k; i++) if (!this.#bits.get(probeAt(a, b, i, this.#m))) return false;
|
|
135
137
|
return true;
|
|
136
138
|
}
|
|
137
139
|
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
const ASCII_FAST_MAX = 12;
|
|
9
|
+
let keyBuf = new Uint8Array(256);
|
|
10
|
+
let keyBytes = keyBuf;
|
|
11
|
+
function encodedBytes() {
|
|
12
|
+
return keyBytes;
|
|
13
|
+
}
|
|
14
|
+
function encodeKey(key) {
|
|
15
|
+
if (typeof key === "string") {
|
|
16
|
+
const n = key.length;
|
|
17
|
+
if (keyBuf.length < n * 3) keyBuf = new Uint8Array(n * 3);
|
|
18
|
+
keyBytes = keyBuf;
|
|
19
|
+
if (n <= ASCII_FAST_MAX) {
|
|
20
|
+
let i = 0;
|
|
21
|
+
for (; i < n; i++) {
|
|
22
|
+
const c = key.charCodeAt(i);
|
|
23
|
+
if (c > 127) break;
|
|
24
|
+
keyBuf[i] = c;
|
|
25
|
+
}
|
|
26
|
+
if (i === n) return n;
|
|
27
|
+
}
|
|
28
|
+
return encoder.encodeInto(key, keyBuf).written;
|
|
29
|
+
}
|
|
30
|
+
keyBytes = normalize(key);
|
|
31
|
+
return keyBytes.length;
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
34
|
+
Object.defineProperty(exports, "encodeKey", {
|
|
35
|
+
enumerable: true,
|
|
36
|
+
get: function() {
|
|
37
|
+
return encodeKey;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
Object.defineProperty(exports, "encodedBytes", {
|
|
41
|
+
enumerable: true,
|
|
42
|
+
get: function() {
|
|
43
|
+
return encodedBytes;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
const ASCII_FAST_MAX = 12;
|
|
9
|
+
let keyBuf = new Uint8Array(256);
|
|
10
|
+
let keyBytes = keyBuf;
|
|
11
|
+
function encodedBytes() {
|
|
12
|
+
return keyBytes;
|
|
13
|
+
}
|
|
14
|
+
function encodeKey(key) {
|
|
15
|
+
if (typeof key === "string") {
|
|
16
|
+
const n = key.length;
|
|
17
|
+
if (keyBuf.length < n * 3) keyBuf = new Uint8Array(n * 3);
|
|
18
|
+
keyBytes = keyBuf;
|
|
19
|
+
if (n <= ASCII_FAST_MAX) {
|
|
20
|
+
let i = 0;
|
|
21
|
+
for (; i < n; i++) {
|
|
22
|
+
const c = key.charCodeAt(i);
|
|
23
|
+
if (c > 127) break;
|
|
24
|
+
keyBuf[i] = c;
|
|
25
|
+
}
|
|
26
|
+
if (i === n) return n;
|
|
27
|
+
}
|
|
28
|
+
return encoder.encodeInto(key, keyBuf).written;
|
|
29
|
+
}
|
|
30
|
+
keyBytes = normalize(key);
|
|
31
|
+
return keyBytes.length;
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
34
|
+
export { encodedBytes as n, encodeKey as t };
|
package/dist/countmin/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-CKLP9Oqe.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
5
|
//#region src/countmin/sizing.ts
|
package/dist/countmin/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as hash32x2Into, s as probeAt } from "../hasher-
|
|
1
|
+
import { a as hash32x2Into, s as probeAt } from "../hasher-BQxNrCQC.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
4
|
//#region src/countmin/sizing.ts
|
package/dist/cuckoo/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-CKLP9Oqe.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
5
|
const require_distinct = require("../distinct-Ck5PKuCS.cjs");
|
package/dist/cuckoo/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { i as hash128KeyInto, u as reduce } from "../hasher-
|
|
1
|
+
import { i as hash128KeyInto, u as reduce } from "../hasher-BQxNrCQC.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
4
|
import { t as DistinctHashes } from "../distinct-BhxDB8mW.js";
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
3
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
4
|
+
const require_ddsketch = require("../ddsketch-Ao9louSV.cjs");
|
|
5
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
6
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
7
|
+
exports.DDSketch = require_ddsketch.DDSketch;
|
|
8
|
+
exports.DDSketchParamMismatchError = require_ddsketch.DDSketchParamMismatchError;
|
|
9
|
+
exports.ParamError = require_params.ParamError;
|
|
10
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
11
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
12
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
13
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
14
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
15
|
+
exports.bitwiseMapping = require_ddsketch.bitwiseMapping;
|
|
16
|
+
exports.collapsingHighestStore = require_ddsketch.collapsingHighestStore;
|
|
17
|
+
exports.collapsingLowestStore = require_ddsketch.collapsingLowestStore;
|
|
18
|
+
exports.cubicMapping = require_ddsketch.cubicMapping;
|
|
19
|
+
exports.denseStore = require_ddsketch.denseStore;
|
|
20
|
+
exports.linearMapping = require_ddsketch.linearMapping;
|
|
21
|
+
exports.logMapping = require_ddsketch.logMapping;
|
|
22
|
+
exports.quadraticMapping = require_ddsketch.quadraticMapping;
|
|
23
|
+
exports.sparseStore = require_ddsketch.sparseStore;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
2
|
+
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
3
|
+
import { _ as quadraticMapping, a as StoreFactory, c as collapsingLowestStore, d as IndexMapping, f as MappingKind, g as logMapping, h as linearMapping, i as Store, l as denseStore, m as cubicMapping, n as DDSketchOptions, o as StoreKind, p as bitwiseMapping, r as DDSketchParamMismatchError, s as collapsingHighestStore, t as DDSketch, u as sparseStore } from "../ddsketch-1Buc9M3P.cjs";
|
|
4
|
+
export { BadMagicError, ChecksumError, DDSketch, type DDSketchOptions, DDSketchParamMismatchError, type FilterJSON, type IndexMapping, type MappingKind, ParamError, ReservedBitsError, SerializationError, type Store, type StoreFactory, type StoreKind, TruncatedError, UnknownHashVariantError, UnknownVersionError, bitwiseMapping, collapsingHighestStore, collapsingLowestStore, cubicMapping, denseStore, linearMapping, logMapping, quadraticMapping, sparseStore };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
2
|
+
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
3
|
+
import { _ as quadraticMapping, a as StoreFactory, c as collapsingLowestStore, d as IndexMapping, f as MappingKind, g as logMapping, h as linearMapping, i as Store, l as denseStore, m as cubicMapping, n as DDSketchOptions, o as StoreKind, p as bitwiseMapping, r as DDSketchParamMismatchError, s as collapsingHighestStore, t as DDSketch, u as sparseStore } from "../ddsketch-CvwWk86v.js";
|
|
4
|
+
export { BadMagicError, ChecksumError, DDSketch, type DDSketchOptions, DDSketchParamMismatchError, type FilterJSON, type IndexMapping, type MappingKind, ParamError, ReservedBitsError, SerializationError, type Store, type StoreFactory, type StoreKind, TruncatedError, UnknownHashVariantError, UnknownVersionError, bitwiseMapping, collapsingHighestStore, collapsingLowestStore, cubicMapping, denseStore, linearMapping, logMapping, quadraticMapping, sparseStore };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { a as TruncatedError, i as SerializationError, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError } from "../serialize-CMSYFym-.js";
|
|
2
|
+
import { t as ParamError } from "../params-CeajSrwv.js";
|
|
3
|
+
import { a as collapsingHighestStore, c as sparseStore, f as linearMapping, h as quadraticMapping, l as bitwiseMapping, n as DDSketchParamMismatchError, o as collapsingLowestStore, p as logMapping, s as denseStore, t as DDSketch, u as cubicMapping } from "../ddsketch-D3Sy8CAU.js";
|
|
4
|
+
export { BadMagicError, ChecksumError, DDSketch, DDSketchParamMismatchError, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, bitwiseMapping, collapsingHighestStore, collapsingLowestStore, cubicMapping, denseStore, linearMapping, logMapping, quadraticMapping, sparseStore };
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_serialize = require("../../serialize-CHikZ4GH.cjs");
|
|
3
|
+
const require_ddsketch = require("../../ddsketch-Ao9louSV.cjs");
|
|
4
|
+
//#region src/ddsketch/protobuf/protobuf.ts
|
|
5
|
+
const KINDS = [
|
|
6
|
+
"log",
|
|
7
|
+
"linear",
|
|
8
|
+
"quadratic",
|
|
9
|
+
"cubic"
|
|
10
|
+
];
|
|
11
|
+
const VARINT = 0;
|
|
12
|
+
const FIXED64 = 1;
|
|
13
|
+
const LENGTH = 2;
|
|
14
|
+
const FIXED32 = 5;
|
|
15
|
+
var Reader = class Reader {
|
|
16
|
+
#bytes;
|
|
17
|
+
#view;
|
|
18
|
+
#at;
|
|
19
|
+
#end;
|
|
20
|
+
constructor(bytes, at = 0, end = bytes.length) {
|
|
21
|
+
this.#bytes = bytes;
|
|
22
|
+
this.#view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
23
|
+
this.#at = at;
|
|
24
|
+
this.#end = end;
|
|
25
|
+
}
|
|
26
|
+
get done() {
|
|
27
|
+
return this.#at >= this.#end;
|
|
28
|
+
}
|
|
29
|
+
varint() {
|
|
30
|
+
let value = 0;
|
|
31
|
+
for (let shift = 0; shift < 70; shift += 7) {
|
|
32
|
+
if (this.#at >= this.#end) throw new require_serialize.SerializationError("protobuf: truncated varint");
|
|
33
|
+
const byte = this.#bytes[this.#at++] ?? 0;
|
|
34
|
+
value += (byte & 127) * 2 ** shift;
|
|
35
|
+
if (byte < 128) return value;
|
|
36
|
+
}
|
|
37
|
+
throw new require_serialize.SerializationError("protobuf: varint longer than ten bytes");
|
|
38
|
+
}
|
|
39
|
+
sint32() {
|
|
40
|
+
const u = this.varint() % 2 ** 32;
|
|
41
|
+
return u >>> 1 ^ -(u & 1);
|
|
42
|
+
}
|
|
43
|
+
double() {
|
|
44
|
+
this.#need(8);
|
|
45
|
+
const value = this.#view.getFloat64(this.#at, true);
|
|
46
|
+
this.#at += 8;
|
|
47
|
+
return value;
|
|
48
|
+
}
|
|
49
|
+
message() {
|
|
50
|
+
const length = this.varint();
|
|
51
|
+
this.#need(length);
|
|
52
|
+
const sub = new Reader(this.#bytes, this.#at, this.#at + length);
|
|
53
|
+
this.#at += length;
|
|
54
|
+
return sub;
|
|
55
|
+
}
|
|
56
|
+
tag() {
|
|
57
|
+
const tag = this.varint();
|
|
58
|
+
return [Math.floor(tag / 8), tag % 8];
|
|
59
|
+
}
|
|
60
|
+
skip(wire) {
|
|
61
|
+
if (wire === VARINT) this.varint();
|
|
62
|
+
else if (wire === FIXED64) this.#skipBytes(8);
|
|
63
|
+
else if (wire === LENGTH) this.message();
|
|
64
|
+
else if (wire === FIXED32) this.#skipBytes(4);
|
|
65
|
+
else throw new require_serialize.SerializationError(`protobuf: unsupported wire type ${String(wire)}`);
|
|
66
|
+
}
|
|
67
|
+
#skipBytes(n) {
|
|
68
|
+
this.#need(n);
|
|
69
|
+
this.#at += n;
|
|
70
|
+
}
|
|
71
|
+
#need(n) {
|
|
72
|
+
if (this.#at + n > this.#end) throw new require_serialize.SerializationError("protobuf: field runs past the end of its message");
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
function expectWire(field, wire, want) {
|
|
76
|
+
if (wire !== want) throw new require_serialize.SerializationError(`protobuf: ${field} has wire type ${String(wire)}, want ${String(want)}`);
|
|
77
|
+
}
|
|
78
|
+
function readMapping(r, into) {
|
|
79
|
+
while (!r.done) {
|
|
80
|
+
const [field, wire] = r.tag();
|
|
81
|
+
if (field === 1) {
|
|
82
|
+
expectWire("gamma", wire, FIXED64);
|
|
83
|
+
into.gamma = r.double();
|
|
84
|
+
} else if (field === 2) {
|
|
85
|
+
expectWire("indexOffset", wire, FIXED64);
|
|
86
|
+
into.indexOffset = r.double();
|
|
87
|
+
} else if (field === 3) {
|
|
88
|
+
expectWire("interpolation", wire, VARINT);
|
|
89
|
+
into.interpolation = r.varint();
|
|
90
|
+
} else r.skip(wire);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function readStore(r, bins) {
|
|
94
|
+
const contiguous = [];
|
|
95
|
+
let offset = 0;
|
|
96
|
+
while (!r.done) {
|
|
97
|
+
const [field, wire] = r.tag();
|
|
98
|
+
if (field === 1) {
|
|
99
|
+
expectWire("binCounts", wire, LENGTH);
|
|
100
|
+
const entry = r.message();
|
|
101
|
+
let index = 0;
|
|
102
|
+
let count = 0;
|
|
103
|
+
while (!entry.done) {
|
|
104
|
+
const [f, w] = entry.tag();
|
|
105
|
+
if (f === 1) {
|
|
106
|
+
expectWire("binCounts key", w, VARINT);
|
|
107
|
+
index = entry.sint32();
|
|
108
|
+
} else if (f === 2) {
|
|
109
|
+
expectWire("binCounts value", w, FIXED64);
|
|
110
|
+
count = entry.double();
|
|
111
|
+
} else entry.skip(w);
|
|
112
|
+
}
|
|
113
|
+
bins.set(index, (bins.get(index) ?? 0) + count);
|
|
114
|
+
} else if (field === 2) {
|
|
115
|
+
if (wire === LENGTH) {
|
|
116
|
+
const packed = r.message();
|
|
117
|
+
while (!packed.done) contiguous.push(packed.double());
|
|
118
|
+
} else {
|
|
119
|
+
expectWire("contiguousBinCounts", wire, FIXED64);
|
|
120
|
+
contiguous.push(r.double());
|
|
121
|
+
}
|
|
122
|
+
} else if (field === 3) {
|
|
123
|
+
expectWire("contiguousBinIndexOffset", wire, VARINT);
|
|
124
|
+
offset = r.sint32();
|
|
125
|
+
} else r.skip(wire);
|
|
126
|
+
}
|
|
127
|
+
contiguous.forEach((count, i) => {
|
|
128
|
+
if (count !== 0) bins.set(offset + i, (bins.get(offset + i) ?? 0) + count);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
function fromProtobuf(bytes, options = {}) {
|
|
132
|
+
const { store = require_ddsketch.collapsingLowestStore(4096) } = options;
|
|
133
|
+
const r = new Reader(bytes);
|
|
134
|
+
const mapped = {
|
|
135
|
+
gamma: 0,
|
|
136
|
+
indexOffset: 0,
|
|
137
|
+
interpolation: 0
|
|
138
|
+
};
|
|
139
|
+
const positive = new Map();
|
|
140
|
+
const negative = new Map();
|
|
141
|
+
let zeroCount = 0;
|
|
142
|
+
while (!r.done) {
|
|
143
|
+
const [field, wire] = r.tag();
|
|
144
|
+
if (field === 1) {
|
|
145
|
+
expectWire("mapping", wire, LENGTH);
|
|
146
|
+
readMapping(r.message(), mapped);
|
|
147
|
+
} else if (field === 2) {
|
|
148
|
+
expectWire("positiveValues", wire, LENGTH);
|
|
149
|
+
readStore(r.message(), positive);
|
|
150
|
+
} else if (field === 3) {
|
|
151
|
+
expectWire("negativeValues", wire, LENGTH);
|
|
152
|
+
readStore(r.message(), negative);
|
|
153
|
+
} else if (field === 4) {
|
|
154
|
+
expectWire("zeroCount", wire, FIXED64);
|
|
155
|
+
zeroCount = r.double();
|
|
156
|
+
} else r.skip(wire);
|
|
157
|
+
}
|
|
158
|
+
const kind = KINDS[mapped.interpolation];
|
|
159
|
+
if (kind === void 0) throw new require_serialize.SerializationError(`protobuf: unknown interpolation ${String(mapped.interpolation)}`);
|
|
160
|
+
const { gamma, indexOffset } = mapped;
|
|
161
|
+
if (!(gamma > 1 && gamma < Infinity) || !Number.isFinite(indexOffset)) throw new require_serialize.SerializationError("protobuf: gamma must be finite and above 1, and indexOffset finite");
|
|
162
|
+
if (!(zeroCount >= 0 && zeroCount < Infinity)) throw new require_serialize.SerializationError(`protobuf: zeroCount must be finite and not negative, got ${String(zeroCount)}`);
|
|
163
|
+
const mapping = require_ddsketch.mappingWithGamma(kind, gamma, indexOffset);
|
|
164
|
+
if (!require_ddsketch.isIndexable(mapping)) throw new require_serialize.SerializationError("protobuf: the mapping leaves no value indexable");
|
|
165
|
+
const lo = mapping.index(mapping.minIndexableValue);
|
|
166
|
+
const hi = mapping.index(mapping.maxIndexableValue);
|
|
167
|
+
for (const bins of [positive, negative]) for (const [index, count] of bins) if (count === 0) bins.delete(index);
|
|
168
|
+
else if (!(count > 0 && count < Infinity) || index < lo || index > hi) throw new require_serialize.SerializationError(`protobuf: bin ${String(index)} with count ${String(count)} is not one a sketch could hold`);
|
|
169
|
+
return require_ddsketch.sketchFromBins(mapping, store, positive, negative, zeroCount);
|
|
170
|
+
}
|
|
171
|
+
function varintSize(value) {
|
|
172
|
+
let size = 1;
|
|
173
|
+
while (value >= 128) {
|
|
174
|
+
value = Math.floor(value / 128);
|
|
175
|
+
size++;
|
|
176
|
+
}
|
|
177
|
+
return size;
|
|
178
|
+
}
|
|
179
|
+
function zigzag(value) {
|
|
180
|
+
return (value << 1 ^ value >> 31) >>> 0;
|
|
181
|
+
}
|
|
182
|
+
function present(value) {
|
|
183
|
+
return value !== 0 || Object.is(value, -0);
|
|
184
|
+
}
|
|
185
|
+
function storeBody(store, sparse) {
|
|
186
|
+
const indices = [];
|
|
187
|
+
const counts = [];
|
|
188
|
+
store.forEach((index, count) => {
|
|
189
|
+
indices.push(index);
|
|
190
|
+
counts.push(count);
|
|
191
|
+
});
|
|
192
|
+
let size = 0;
|
|
193
|
+
const first = indices[0];
|
|
194
|
+
if (sparse) for (const index of indices) {
|
|
195
|
+
const entry = 1 + varintSize(zigzag(index)) + 1 + 8;
|
|
196
|
+
size += 1 + varintSize(entry) + entry;
|
|
197
|
+
}
|
|
198
|
+
else if (first !== void 0) {
|
|
199
|
+
const packed = 8 * ((indices.at(-1) ?? first) - first + 1);
|
|
200
|
+
size = 1 + varintSize(packed) + packed;
|
|
201
|
+
if (first !== 0) size += 1 + varintSize(zigzag(first));
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
indices,
|
|
205
|
+
counts,
|
|
206
|
+
sparse,
|
|
207
|
+
size
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
var Writer = class {
|
|
211
|
+
bytes;
|
|
212
|
+
#view;
|
|
213
|
+
#at = 0;
|
|
214
|
+
constructor(size) {
|
|
215
|
+
this.bytes = new Uint8Array(size);
|
|
216
|
+
this.#view = new DataView(this.bytes.buffer);
|
|
217
|
+
}
|
|
218
|
+
tag(field, wire) {
|
|
219
|
+
this.bytes[this.#at++] = field * 8 + wire;
|
|
220
|
+
}
|
|
221
|
+
varint(value) {
|
|
222
|
+
while (value >= 128) {
|
|
223
|
+
this.bytes[this.#at++] = value % 128 | 128;
|
|
224
|
+
value = Math.floor(value / 128);
|
|
225
|
+
}
|
|
226
|
+
this.bytes[this.#at++] = value;
|
|
227
|
+
}
|
|
228
|
+
double(value) {
|
|
229
|
+
this.#view.setFloat64(this.#at, value, true);
|
|
230
|
+
this.#at += 8;
|
|
231
|
+
}
|
|
232
|
+
store(field, body) {
|
|
233
|
+
this.tag(field, LENGTH);
|
|
234
|
+
this.varint(body.size);
|
|
235
|
+
const { indices, counts } = body;
|
|
236
|
+
const first = indices[0];
|
|
237
|
+
if (first === void 0) return;
|
|
238
|
+
if (body.sparse) {
|
|
239
|
+
indices.forEach((index, i) => {
|
|
240
|
+
const key = zigzag(index);
|
|
241
|
+
this.tag(1, LENGTH);
|
|
242
|
+
this.varint(1 + varintSize(key) + 1 + 8);
|
|
243
|
+
this.tag(1, VARINT);
|
|
244
|
+
this.varint(key);
|
|
245
|
+
this.tag(2, FIXED64);
|
|
246
|
+
this.double(counts[i] ?? 0);
|
|
247
|
+
});
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const span = (indices.at(-1) ?? first) - first + 1;
|
|
251
|
+
this.tag(2, LENGTH);
|
|
252
|
+
this.varint(8 * span);
|
|
253
|
+
indices.forEach((index, i) => {
|
|
254
|
+
this.#view.setFloat64(this.#at + 8 * (index - first), counts[i] ?? 0, true);
|
|
255
|
+
});
|
|
256
|
+
this.#at += 8 * span;
|
|
257
|
+
if (first !== 0) {
|
|
258
|
+
this.tag(3, VARINT);
|
|
259
|
+
this.varint(zigzag(first));
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
function toProtobuf(sketch) {
|
|
264
|
+
const { mapping } = sketch;
|
|
265
|
+
const { positive, negative, zeroCount } = require_ddsketch.sketchBins(sketch);
|
|
266
|
+
const sparse = sketch.store.kind === "sparse";
|
|
267
|
+
const pos = storeBody(positive, sparse);
|
|
268
|
+
const neg = storeBody(negative, sparse);
|
|
269
|
+
const interpolation = KINDS.indexOf(mapping.kind);
|
|
270
|
+
const mappingSize = (present(mapping.gamma) ? 9 : 0) + (present(mapping.indexOffset) ? 9 : 0) + (interpolation !== 0 ? 1 + varintSize(interpolation) : 0);
|
|
271
|
+
const w = new Writer(1 + varintSize(mappingSize) + mappingSize + (1 + varintSize(pos.size) + pos.size) + (1 + varintSize(neg.size) + neg.size) + (present(zeroCount) ? 9 : 0));
|
|
272
|
+
w.tag(1, LENGTH);
|
|
273
|
+
w.varint(mappingSize);
|
|
274
|
+
if (present(mapping.gamma)) {
|
|
275
|
+
w.tag(1, FIXED64);
|
|
276
|
+
w.double(mapping.gamma);
|
|
277
|
+
}
|
|
278
|
+
if (present(mapping.indexOffset)) {
|
|
279
|
+
w.tag(2, FIXED64);
|
|
280
|
+
w.double(mapping.indexOffset);
|
|
281
|
+
}
|
|
282
|
+
if (interpolation !== 0) {
|
|
283
|
+
w.tag(3, VARINT);
|
|
284
|
+
w.varint(interpolation);
|
|
285
|
+
}
|
|
286
|
+
w.store(2, pos);
|
|
287
|
+
w.store(3, neg);
|
|
288
|
+
if (present(zeroCount)) {
|
|
289
|
+
w.tag(4, FIXED64);
|
|
290
|
+
w.double(zeroCount);
|
|
291
|
+
}
|
|
292
|
+
return w.bytes;
|
|
293
|
+
}
|
|
294
|
+
//#endregion
|
|
295
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
296
|
+
exports.fromProtobuf = fromProtobuf;
|
|
297
|
+
exports.toProtobuf = toProtobuf;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { o as SerializationError } from "../../serialize--xR6vGSW.cjs";
|
|
2
|
+
import { a as StoreFactory, t as DDSketch } from "../../ddsketch-1Buc9M3P.cjs";
|
|
3
|
+
//#region src/ddsketch/protobuf/protobuf.d.ts
|
|
4
|
+
/** Options for {@link fromProtobuf}. */
|
|
5
|
+
interface ProtobufOptions {
|
|
6
|
+
/**
|
|
7
|
+
* The store the decoded sketch keeps its bins in. Defaults to
|
|
8
|
+
* {@link collapsingLowestStore} at 4096 bins, where sketches-go's FromProto
|
|
9
|
+
* takes a dense store: a message of a few dozen bytes can name two bins
|
|
10
|
+
* billions of indices apart, and a dense store would allocate every slot
|
|
11
|
+
* between them. Pass `denseStore()` for sketches-go's behavior on
|
|
12
|
+
* trusted input.
|
|
13
|
+
*/
|
|
14
|
+
store?: StoreFactory;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Decodes Datadog's `DDSketch` protobuf message, as sketches-go, sketches-java
|
|
18
|
+
* and the Datadog agent write it.
|
|
19
|
+
*
|
|
20
|
+
* The message carries no sum, min or max, so the decoded sketch derives them
|
|
21
|
+
* from its bins: each is within `relativeAccuracy` of the truth for min and
|
|
22
|
+
* max, and approximate for sum.
|
|
23
|
+
*
|
|
24
|
+
* @param bytes - The serialized message.
|
|
25
|
+
* @param options - Optional store for the decoded bins.
|
|
26
|
+
* @returns The decoded sketch.
|
|
27
|
+
* @throws {@link SerializationError} when the message is malformed.
|
|
28
|
+
*/
|
|
29
|
+
export declare function fromProtobuf(bytes: Uint8Array, options?: ProtobufOptions): DDSketch;
|
|
30
|
+
/**
|
|
31
|
+
* Encodes a sketch as Datadog's `DDSketch` protobuf message, the form the
|
|
32
|
+
* Datadog API and sketches-go read. For a dense or collapsing store the bytes
|
|
33
|
+
* are exactly those sketches-go writes for the same bins.
|
|
34
|
+
*
|
|
35
|
+
* The message has no fields for sum, min or max, so those do not survive the
|
|
36
|
+
* trip; {@link fromProtobuf} derives them from the bins.
|
|
37
|
+
*
|
|
38
|
+
* @param sketch - The sketch to encode.
|
|
39
|
+
* @returns The serialized message.
|
|
40
|
+
*/
|
|
41
|
+
export declare function toProtobuf(sketch: DDSketch): Uint8Array;
|
|
42
|
+
//#endregion
|
|
43
|
+
export { type ProtobufOptions, SerializationError };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { o as SerializationError } from "../../serialize--xR6vGSW.js";
|
|
2
|
+
import { a as StoreFactory, t as DDSketch } from "../../ddsketch-CvwWk86v.js";
|
|
3
|
+
//#region src/ddsketch/protobuf/protobuf.d.ts
|
|
4
|
+
/** Options for {@link fromProtobuf}. */
|
|
5
|
+
interface ProtobufOptions {
|
|
6
|
+
/**
|
|
7
|
+
* The store the decoded sketch keeps its bins in. Defaults to
|
|
8
|
+
* {@link collapsingLowestStore} at 4096 bins, where sketches-go's FromProto
|
|
9
|
+
* takes a dense store: a message of a few dozen bytes can name two bins
|
|
10
|
+
* billions of indices apart, and a dense store would allocate every slot
|
|
11
|
+
* between them. Pass `denseStore()` for sketches-go's behavior on
|
|
12
|
+
* trusted input.
|
|
13
|
+
*/
|
|
14
|
+
store?: StoreFactory;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Decodes Datadog's `DDSketch` protobuf message, as sketches-go, sketches-java
|
|
18
|
+
* and the Datadog agent write it.
|
|
19
|
+
*
|
|
20
|
+
* The message carries no sum, min or max, so the decoded sketch derives them
|
|
21
|
+
* from its bins: each is within `relativeAccuracy` of the truth for min and
|
|
22
|
+
* max, and approximate for sum.
|
|
23
|
+
*
|
|
24
|
+
* @param bytes - The serialized message.
|
|
25
|
+
* @param options - Optional store for the decoded bins.
|
|
26
|
+
* @returns The decoded sketch.
|
|
27
|
+
* @throws {@link SerializationError} when the message is malformed.
|
|
28
|
+
*/
|
|
29
|
+
export declare function fromProtobuf(bytes: Uint8Array, options?: ProtobufOptions): DDSketch;
|
|
30
|
+
/**
|
|
31
|
+
* Encodes a sketch as Datadog's `DDSketch` protobuf message, the form the
|
|
32
|
+
* Datadog API and sketches-go read. For a dense or collapsing store the bytes
|
|
33
|
+
* are exactly those sketches-go writes for the same bins.
|
|
34
|
+
*
|
|
35
|
+
* The message has no fields for sum, min or max, so those do not survive the
|
|
36
|
+
* trip; {@link fromProtobuf} derives them from the bins.
|
|
37
|
+
*
|
|
38
|
+
* @param sketch - The sketch to encode.
|
|
39
|
+
* @returns The serialized message.
|
|
40
|
+
*/
|
|
41
|
+
export declare function toProtobuf(sketch: DDSketch): Uint8Array;
|
|
42
|
+
//#endregion
|
|
43
|
+
export { type ProtobufOptions, SerializationError };
|