distillate 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -10
- package/dist/blocked/index.cjs +1 -1
- package/dist/blocked/index.js +1 -1
- package/dist/bloom/index.cjs +3 -4
- package/dist/bloom/index.d.cts +12 -2
- package/dist/bloom/index.d.ts +12 -2
- package/dist/bloom/index.js +2 -3
- package/dist/bytes-Be7tq1F7.js +9 -0
- package/dist/bytes-D2duG1jq.cjs +14 -0
- package/dist/countmin/index.cjs +203 -0
- package/dist/countmin/index.d.cts +191 -0
- package/dist/countmin/index.d.ts +191 -0
- package/dist/countmin/index.js +191 -0
- package/dist/cuckoo/index.cjs +2 -2
- package/dist/cuckoo/index.js +2 -2
- package/dist/fuse/index.cjs +1 -1
- package/dist/fuse/index.js +1 -1
- package/dist/{hasher-3Cyn7Yg-.js → hasher-DOdmZ-LI.js} +1 -8
- package/dist/{hasher-MhY7aQ8F.cjs → hasher-DPXmP7SH.cjs} +2 -9
- package/dist/hll/index.cjs +10 -4
- package/dist/hll/index.d.cts +13 -2
- package/dist/hll/index.d.ts +13 -2
- package/dist/hll/index.js +9 -3
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +5 -6
- package/dist/scalable/index.js +2 -3
- package/dist/{bitset-QYGqxnEf.cjs → sizing-Ch0SDdza.cjs} +15 -0
- package/dist/{bitset-Cf2jwRJX.js → sizing-zxCBlyhO.js} +10 -1
- package/dist/topk/index.cjs +423 -0
- package/dist/topk/index.d.cts +207 -0
- package/dist/topk/index.d.ts +207 -0
- package/dist/topk/index.js +411 -0
- package/package.json +17 -1
- package/dist/sizing-BLzZk2zr.cjs +0 -28
- package/dist/sizing-BtUrtvF_.js +0 -17
- package/dist/sizing-wYOW27eY.d.cts +0 -22
- package/dist/sizing-wYOW27eY.d.ts +0 -22
package/README.md
CHANGED
|
@@ -42,18 +42,22 @@ Every push runs a CI smoke matrix that imports the built package on Node 22/24,
|
|
|
42
42
|
|
|
43
43
|
Each structure ships as its own subpath, so you only bundle what you import.
|
|
44
44
|
|
|
45
|
-
| Import | Structure | Answers
|
|
46
|
-
| --------------------- | -------------- |
|
|
47
|
-
| `distillate/bloom` | Classic Bloom | seen this key?
|
|
48
|
-
| `distillate/scalable` | Scalable Bloom | seen this key?
|
|
49
|
-
| `distillate/blocked` | Blocked Bloom | seen this key?
|
|
50
|
-
| `distillate/fuse` | Binary Fuse | seen this key?
|
|
51
|
-
| `distillate/cuckoo` | Cuckoo | seen this key?
|
|
52
|
-
| `distillate/hll` | HyperLogLog | how many distinct?
|
|
45
|
+
| Import | Structure | Answers | Use for |
|
|
46
|
+
| --------------------- | -------------- | ------------------- | -------------------------------------------------------- |
|
|
47
|
+
| `distillate/bloom` | Classic Bloom | seen this key? | Familiar default, migration from `bloom-filters` |
|
|
48
|
+
| `distillate/scalable` | Scalable Bloom | seen this key? | Key count unknown or growing; keeps its FPR bound |
|
|
49
|
+
| `distillate/blocked` | Blocked Bloom | seen this key? | Faster lookups and a lower FPR for a small space premium |
|
|
50
|
+
| `distillate/fuse` | Binary Fuse | seen this key? | Static set built once and queried a lot; least space |
|
|
51
|
+
| `distillate/cuckoo` | Cuckoo | seen this key? | Keys come and go; the one filter with delete |
|
|
52
|
+
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
53
|
+
| `distillate/countmin` | Count-Min | how many times? | Counting events per key in fixed space |
|
|
54
|
+
| `distillate/topk` | Top-K | which are heaviest? | Finding the most frequent keys in fixed space |
|
|
53
55
|
|
|
54
56
|
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
55
|
-
key set. HyperLogLog
|
|
56
|
-
whether it saw any particular
|
|
57
|
+
key set. HyperLogLog, Count-Min and Top-K are sketches rather than filters:
|
|
58
|
+
none reports whether it saw any particular key. HyperLogLog counts distinct
|
|
59
|
+
keys, Count-Min counts how often each key appeared, and Top-K reports which
|
|
60
|
+
keys appeared most.
|
|
57
61
|
|
|
58
62
|
### Classic Bloom (`distillate/bloom`)
|
|
59
63
|
|
|
@@ -163,6 +167,54 @@ Below a few thousand distinct keys the sketch counts rather than estimates, so s
|
|
|
163
167
|
|
|
164
168
|
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `standardError`, `hllSizing(relativeError)`, and a low-level `new HyperLogLog({ p, seed })`.
|
|
165
169
|
|
|
170
|
+
### Count-Min (`distillate/countmin`)
|
|
171
|
+
|
|
172
|
+
A **sketch**, not a filter: it estimates how many times each key appeared, in space fixed by the error you ask for rather than by how many keys arrive. The estimate is never below the truth, and at most `epsilon` of the total above it. Full API and sizing: [Count-Min guide](https://distillate.akxp.net/guides/countmin/).
|
|
173
|
+
|
|
174
|
+
```ts
|
|
175
|
+
import { CountMinSketch } from "distillate/countmin";
|
|
176
|
+
|
|
177
|
+
const hits = CountMinSketch.create(0.001, 0.001); // error factor, failure probability
|
|
178
|
+
hits.add("/login");
|
|
179
|
+
hits.add("/login", 4);
|
|
180
|
+
hits.add("/signup");
|
|
181
|
+
|
|
182
|
+
hits.count("/login"); // 5
|
|
183
|
+
hits.total; // 6
|
|
184
|
+
|
|
185
|
+
// Combine per-shard sketches: counts add, exactly as if one sketch saw both streams.
|
|
186
|
+
const other = CountMinSketch.from(["/login", "/login"], 0.001, 0.001);
|
|
187
|
+
hits.union(other).count("/login"); // 7
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
`from` counts repeats, unlike `CuckooFilter.from` which ignores them: for a frequency sketch the repeats are the measurement. A counter that would pass `2^32 - 1` throws `CountMinOverflowError` rather than wrapping, since a wrapped counter would read as an underestimate.
|
|
191
|
+
|
|
192
|
+
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `error()`, `epsilon`, `delta`, `width`, `depth`, `countMinSizing(epsilon, delta)`, and a low-level `new CountMinSketch({ width, depth, seed })`.
|
|
193
|
+
|
|
194
|
+
### Top-K (`distillate/topk`)
|
|
195
|
+
|
|
196
|
+
A **sketch** that lists the keys seen most often, in a table fixed by the error you ask for rather than by how many keys arrive. A key it holds never reads below its true count, and any key heavier than `error()` is guaranteed to be held. Full API and the bound: [Top-K guide](https://distillate.akxp.net/guides/topk/).
|
|
197
|
+
|
|
198
|
+
```ts
|
|
199
|
+
import { TopK } from "distillate/topk";
|
|
200
|
+
|
|
201
|
+
const routes = TopK.create(0.01); // error factor
|
|
202
|
+
routes.add("/login", 5);
|
|
203
|
+
routes.add("/signup", 2);
|
|
204
|
+
routes.add("/about");
|
|
205
|
+
|
|
206
|
+
const decoder = new TextDecoder();
|
|
207
|
+
routes
|
|
208
|
+
.top(2)
|
|
209
|
+
.map((e) => decoder.decode(e.key))
|
|
210
|
+
.join(); // "/login,/signup"
|
|
211
|
+
routes.count("/login"); // 5
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Keys come back as `Uint8Array`, exactly as recorded, so a serialized Top-K sketch holds your keys verbatim, unlike every other frame here. `union` combines sketches soundly but, unlike Count-Min's, not byte for byte as one sketch fed both streams.
|
|
215
|
+
|
|
216
|
+
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `total`, `error()`, `epsilon`, `capacity`, `topKSizing(epsilon)`, and a low-level `new TopK({ capacity, seed })`.
|
|
217
|
+
|
|
166
218
|
## Performance
|
|
167
219
|
|
|
168
220
|
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (cross-library harness, node v24.14.1, Apple M5):
|
package/dist/blocked/index.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_hasher = require("../hasher-DPXmP7SH.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
5
|
//#region src/blocked/blocked.ts
|
package/dist/blocked/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as hash32x2Into, u as reduce } from "../hasher-
|
|
1
|
+
import { a as hash32x2Into, u as reduce } from "../hasher-DOdmZ-LI.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { i as assertProbability, n as assertPositiveFinite, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
4
|
//#region src/blocked/blocked.ts
|
package/dist/bloom/index.cjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const
|
|
3
|
-
const require_hasher = require("../hasher-
|
|
2
|
+
const require_sizing = require("../sizing-Ch0SDdza.cjs");
|
|
3
|
+
const require_hasher = require("../hasher-DPXmP7SH.cjs");
|
|
4
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
5
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
6
|
-
const require_sizing = require("../sizing-BLzZk2zr.cjs");
|
|
7
6
|
//#region src/bloom/bloom.ts
|
|
8
7
|
const TYPE = 1;
|
|
9
8
|
const PARAMS_SIZE = 16;
|
|
@@ -64,7 +63,7 @@ var BloomFilter = class BloomFilter {
|
|
|
64
63
|
require_params.assertPositiveInt(n, "n");
|
|
65
64
|
require_params.assertUint32(n, "n");
|
|
66
65
|
}
|
|
67
|
-
this.#bits = new
|
|
66
|
+
this.#bits = new require_sizing.BitSet(m);
|
|
68
67
|
this.#m = m;
|
|
69
68
|
this.#k = k;
|
|
70
69
|
this.#seed = seed;
|
package/dist/bloom/index.d.cts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
-
import { r as bloomSizing, t as BloomSizing } from "../sizing-wYOW27eY.cjs";
|
|
5
4
|
//#region src/bloom/bloom.d.ts
|
|
6
5
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
7
6
|
export declare class BloomParamMismatchError extends Error {
|
|
@@ -144,4 +143,15 @@ export declare class BloomFilter {
|
|
|
144
143
|
has(key: BytesLike): boolean;
|
|
145
144
|
}
|
|
146
145
|
//#endregion
|
|
147
|
-
|
|
146
|
+
//#region src/core/sizing.d.ts
|
|
147
|
+
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
148
|
+
interface BloomSizing {
|
|
149
|
+
/** Number of bits in the filter. */
|
|
150
|
+
m: number;
|
|
151
|
+
/** Number of hash probes per key. */
|
|
152
|
+
k: number;
|
|
153
|
+
}
|
|
154
|
+
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
155
|
+
export declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
156
|
+
//#endregion
|
|
157
|
+
export { BadMagicError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/bloom/index.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
-
import { r as bloomSizing, t as BloomSizing } from "../sizing-wYOW27eY.js";
|
|
5
4
|
//#region src/bloom/bloom.d.ts
|
|
6
5
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
7
6
|
export declare class BloomParamMismatchError extends Error {
|
|
@@ -144,4 +143,15 @@ export declare class BloomFilter {
|
|
|
144
143
|
has(key: BytesLike): boolean;
|
|
145
144
|
}
|
|
146
145
|
//#endregion
|
|
147
|
-
|
|
146
|
+
//#region src/core/sizing.d.ts
|
|
147
|
+
/** Bloom filter geometry: the `BloomParams` fields a sizing solve determines. */
|
|
148
|
+
interface BloomSizing {
|
|
149
|
+
/** Number of bits in the filter. */
|
|
150
|
+
m: number;
|
|
151
|
+
/** Number of hash probes per key. */
|
|
152
|
+
k: number;
|
|
153
|
+
}
|
|
154
|
+
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
155
|
+
export declare function bloomSizing(n: number, epsilon: number): BloomSizing;
|
|
156
|
+
//#endregion
|
|
157
|
+
export { BadMagicError, type BloomParams, type BloomSizing, ChecksumError, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|
package/dist/bloom/index.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { c as probeInto } from "../hasher-
|
|
1
|
+
import { n as BitSet, t as bloomSizing } from "../sizing-zxCBlyhO.js";
|
|
2
|
+
import { c as probeInto } from "../hasher-DOdmZ-LI.js";
|
|
3
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
4
4
|
import { a as assertUint16, i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
5
|
-
import { t as bloomSizing } from "../sizing-BtUrtvF_.js";
|
|
6
5
|
//#region src/bloom/bloom.ts
|
|
7
6
|
const TYPE = 1;
|
|
8
7
|
const PARAMS_SIZE = 16;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
//#endregion
|
|
9
|
+
export { normalize as t };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
//#region src/core/bytes.ts
|
|
2
|
+
const encoder = new TextEncoder();
|
|
3
|
+
function normalize(input) {
|
|
4
|
+
if (typeof input === "string") return encoder.encode(input);
|
|
5
|
+
if (input instanceof Uint8Array) return input;
|
|
6
|
+
return new Uint8Array(input);
|
|
7
|
+
}
|
|
8
|
+
//#endregion
|
|
9
|
+
Object.defineProperty(exports, "normalize", {
|
|
10
|
+
enumerable: true,
|
|
11
|
+
get: function() {
|
|
12
|
+
return normalize;
|
|
13
|
+
}
|
|
14
|
+
});
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_hasher = require("../hasher-DPXmP7SH.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
//#region src/countmin/sizing.ts
|
|
6
|
+
function countMinSizing(epsilon, delta) {
|
|
7
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
8
|
+
require_params.assertProbability(delta, "delta");
|
|
9
|
+
return {
|
|
10
|
+
width: Math.ceil(Math.E / epsilon),
|
|
11
|
+
depth: Math.ceil(Math.log(1 / delta))
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
//#endregion
|
|
15
|
+
//#region src/countmin/countmin.ts
|
|
16
|
+
const TYPE = 8;
|
|
17
|
+
const PARAMS_SIZE = 16;
|
|
18
|
+
const PARAMS_FIELDS_END = 12;
|
|
19
|
+
const counterBytes = (counters) => new Uint8Array(counters.buffer, counters.byteOffset, counters.byteLength);
|
|
20
|
+
function rowSum(counters, width, depth) {
|
|
21
|
+
let expected = 0;
|
|
22
|
+
for (let r = 0; r < depth; r++) {
|
|
23
|
+
let sum = 0;
|
|
24
|
+
const from = r * width;
|
|
25
|
+
for (let c = from; c < from + width; c++) sum += counters[c] ?? 0;
|
|
26
|
+
if (sum > Number.MAX_SAFE_INTEGER) throw new require_serialize.SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, past the safe integer range`);
|
|
27
|
+
if (r === 0) expected = sum;
|
|
28
|
+
else if (sum !== expected) throw new require_serialize.SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, but row 0 sums to ${String(expected)}`);
|
|
29
|
+
}
|
|
30
|
+
return expected;
|
|
31
|
+
}
|
|
32
|
+
var CountMinParamMismatchError = class extends Error {
|
|
33
|
+
name = "CountMinParamMismatchError";
|
|
34
|
+
};
|
|
35
|
+
var CountMinOverflowError = class extends RangeError {
|
|
36
|
+
name = "CountMinOverflowError";
|
|
37
|
+
};
|
|
38
|
+
var CountMinSketch = class CountMinSketch {
|
|
39
|
+
#counters;
|
|
40
|
+
#width;
|
|
41
|
+
#depth;
|
|
42
|
+
#seed;
|
|
43
|
+
#words = new Uint32Array(2);
|
|
44
|
+
#positions;
|
|
45
|
+
#total = 0;
|
|
46
|
+
static create(epsilon, delta, options = {}) {
|
|
47
|
+
return new CountMinSketch({
|
|
48
|
+
...options,
|
|
49
|
+
...countMinSizing(epsilon, delta)
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
static from(keys, epsilon, delta, options = {}) {
|
|
53
|
+
const sketch = CountMinSketch.create(epsilon, delta, options);
|
|
54
|
+
for (const key of keys) sketch.add(key);
|
|
55
|
+
return sketch;
|
|
56
|
+
}
|
|
57
|
+
static fromBytes(bytes) {
|
|
58
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
59
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
60
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
61
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "countmin");
|
|
62
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "countmin");
|
|
63
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
64
|
+
const width = view.getUint32(0, true);
|
|
65
|
+
const depth = view.getUint32(4, true);
|
|
66
|
+
const seed = view.getUint32(8, true);
|
|
67
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + 4 * width * depth, "countmin");
|
|
68
|
+
let sketch;
|
|
69
|
+
try {
|
|
70
|
+
sketch = new CountMinSketch({
|
|
71
|
+
width,
|
|
72
|
+
depth,
|
|
73
|
+
seed
|
|
74
|
+
});
|
|
75
|
+
} catch (err) {
|
|
76
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`countmin: ${err.message}`);
|
|
77
|
+
throw err;
|
|
78
|
+
}
|
|
79
|
+
const counters = sketch.#counters;
|
|
80
|
+
for (let i = 0; i < counters.length; i++) counters[i] = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
81
|
+
sketch.#total = rowSum(counters, width, depth);
|
|
82
|
+
return sketch;
|
|
83
|
+
}
|
|
84
|
+
constructor({ width, depth, seed = 0 }) {
|
|
85
|
+
require_params.assertPositiveInt(width, "width");
|
|
86
|
+
require_params.assertUint32(width, "width");
|
|
87
|
+
require_params.assertPositiveInt(depth, "depth");
|
|
88
|
+
require_params.assertUint32(depth, "depth");
|
|
89
|
+
require_params.assertUint32(seed, "seed");
|
|
90
|
+
if (width * depth > 4294967295) throw new require_params.ParamError(`width ${String(width)} by depth ${String(depth)} needs ${String(width * depth)} counters, above the maximum ${String(4294967295)}`);
|
|
91
|
+
this.#counters = new Uint32Array(width * depth);
|
|
92
|
+
this.#positions = new Uint32Array(depth);
|
|
93
|
+
this.#width = width;
|
|
94
|
+
this.#depth = depth;
|
|
95
|
+
this.#seed = seed;
|
|
96
|
+
}
|
|
97
|
+
get width() {
|
|
98
|
+
return this.#width;
|
|
99
|
+
}
|
|
100
|
+
get depth() {
|
|
101
|
+
return this.#depth;
|
|
102
|
+
}
|
|
103
|
+
get seed() {
|
|
104
|
+
return this.#seed;
|
|
105
|
+
}
|
|
106
|
+
get total() {
|
|
107
|
+
return this.#total;
|
|
108
|
+
}
|
|
109
|
+
get epsilon() {
|
|
110
|
+
return Math.E / this.#width;
|
|
111
|
+
}
|
|
112
|
+
get delta() {
|
|
113
|
+
return Math.exp(-this.#depth);
|
|
114
|
+
}
|
|
115
|
+
error() {
|
|
116
|
+
return this.epsilon * this.#total;
|
|
117
|
+
}
|
|
118
|
+
#positionAt(row) {
|
|
119
|
+
return row * this.#width + require_hasher.probeAt(this.#words[0] ?? 0, this.#words[1] ?? 0, row, this.#width);
|
|
120
|
+
}
|
|
121
|
+
add(key, count = 1) {
|
|
122
|
+
require_params.assertPositiveInt(count, "count");
|
|
123
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
124
|
+
require_hasher.hash32x2Into(key, this.#seed, this.#words);
|
|
125
|
+
const headroom = 4294967295 - count;
|
|
126
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
127
|
+
const at = this.#positionAt(r);
|
|
128
|
+
if ((this.#counters[at] ?? 0) > headroom) throw new CountMinOverflowError(`adding ${String(count)} would carry a counter past ${String(4294967295)}`);
|
|
129
|
+
this.#positions[r] = at;
|
|
130
|
+
}
|
|
131
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
132
|
+
const at = this.#positions[r] ?? 0;
|
|
133
|
+
this.#counters[at] = (this.#counters[at] ?? 0) + count;
|
|
134
|
+
}
|
|
135
|
+
this.#total += count;
|
|
136
|
+
}
|
|
137
|
+
union(other) {
|
|
138
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) throw new CountMinParamMismatchError("cannot union Count-Min sketches whose parameters do not match");
|
|
139
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
140
|
+
const mine = this.#counters;
|
|
141
|
+
const theirs = other.#counters;
|
|
142
|
+
const merged = new CountMinSketch({
|
|
143
|
+
width: this.#width,
|
|
144
|
+
depth: this.#depth,
|
|
145
|
+
seed: this.#seed
|
|
146
|
+
});
|
|
147
|
+
const into = merged.#counters;
|
|
148
|
+
for (let i = 0; i < into.length; i++) {
|
|
149
|
+
const sum = (mine[i] ?? 0) + (theirs[i] ?? 0);
|
|
150
|
+
if (sum > 4294967295) throw new CountMinOverflowError(`union would carry a counter to ${String(sum)}, past ${String(4294967295)}`);
|
|
151
|
+
into[i] = sum;
|
|
152
|
+
}
|
|
153
|
+
merged.#total = this.#total + other.#total;
|
|
154
|
+
return merged;
|
|
155
|
+
}
|
|
156
|
+
toBytes() {
|
|
157
|
+
const counters = this.#counters;
|
|
158
|
+
return require_serialize.writeFrame({
|
|
159
|
+
version: 5,
|
|
160
|
+
type: TYPE,
|
|
161
|
+
flags: 0
|
|
162
|
+
}, PARAMS_SIZE, 4 * counters.length, (_, view) => {
|
|
163
|
+
view.setUint32(0, this.#width, true);
|
|
164
|
+
view.setUint32(4, this.#depth, true);
|
|
165
|
+
view.setUint32(8, this.#seed, true);
|
|
166
|
+
counters.forEach((c, i) => {
|
|
167
|
+
view.setUint32(PARAMS_SIZE + 4 * i, c, true);
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
equals(other) {
|
|
172
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) return false;
|
|
173
|
+
return require_serialize.bytesEqual(counterBytes(this.#counters), counterBytes(other.#counters));
|
|
174
|
+
}
|
|
175
|
+
toJSON() {
|
|
176
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
177
|
+
}
|
|
178
|
+
static fromJSON(value) {
|
|
179
|
+
return CountMinSketch.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
180
|
+
}
|
|
181
|
+
count(key) {
|
|
182
|
+
require_hasher.hash32x2Into(key, this.#seed, this.#words);
|
|
183
|
+
let min = Infinity;
|
|
184
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
185
|
+
const at = this.#counters[this.#positionAt(r)] ?? 0;
|
|
186
|
+
if (at < min) min = at;
|
|
187
|
+
}
|
|
188
|
+
return min;
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
//#endregion
|
|
192
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
193
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
194
|
+
exports.CountMinOverflowError = CountMinOverflowError;
|
|
195
|
+
exports.CountMinParamMismatchError = CountMinParamMismatchError;
|
|
196
|
+
exports.CountMinSketch = CountMinSketch;
|
|
197
|
+
exports.ParamError = require_params.ParamError;
|
|
198
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
199
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
200
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
201
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
202
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
203
|
+
exports.countMinSizing = countMinSizing;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
+
//#region src/countmin/countmin.d.ts
|
|
5
|
+
/** Thrown when an operation requires two sketches built with identical parameters. */
|
|
6
|
+
export declare class CountMinParamMismatchError extends Error {
|
|
7
|
+
/** Discriminates this error from other `Error`s. */
|
|
8
|
+
override readonly name = "CountMinParamMismatchError";
|
|
9
|
+
}
|
|
10
|
+
/** Thrown when an add would carry a counter past what a u32 holds. */
|
|
11
|
+
export declare class CountMinOverflowError extends RangeError {
|
|
12
|
+
/** Discriminates this error from other `Error`s. */
|
|
13
|
+
override readonly name = "CountMinOverflowError";
|
|
14
|
+
}
|
|
15
|
+
/** Options accepted alongside a sizing solve: everything but the geometry. */
|
|
16
|
+
type CountMinOptions = Omit<CountMinParams, "width" | "depth">;
|
|
17
|
+
/** Low-level Count-Min sketch parameters. */
|
|
18
|
+
interface CountMinParams {
|
|
19
|
+
/** Counters per row. */
|
|
20
|
+
width: number;
|
|
21
|
+
/** Number of rows, one probe each. */
|
|
22
|
+
depth: number;
|
|
23
|
+
/** Hash seed; defaults to `0`. */
|
|
24
|
+
seed?: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* A Count-Min sketch: a fixed-size frequency estimate with a tunable
|
|
28
|
+
* overestimate and no underestimate.
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```ts
|
|
32
|
+
* const sketch = CountMinSketch.create(0.001, 0.001);
|
|
33
|
+
* sketch.add("alice");
|
|
34
|
+
* sketch.add("alice", 3);
|
|
35
|
+
* sketch.count("alice"); // 4 (or a rare overestimate)
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export declare class CountMinSketch {
|
|
39
|
+
#private;
|
|
40
|
+
/**
|
|
41
|
+
* Creates a sketch whose estimate is at most `epsilon * total` above the
|
|
42
|
+
* true count, with probability `1 - delta`.
|
|
43
|
+
*
|
|
44
|
+
* Unlike a filter, a sketch is sized by the error it targets rather than by
|
|
45
|
+
* how many keys it will see, so no key count is needed.
|
|
46
|
+
*
|
|
47
|
+
* @param epsilon - Error factor relative to the total recorded, e.g. `0.001`.
|
|
48
|
+
* @param delta - Probability the bound is exceeded, e.g. `0.001`.
|
|
49
|
+
* @param options - Optional seed.
|
|
50
|
+
* @returns A new, empty sketch.
|
|
51
|
+
*/
|
|
52
|
+
static create(epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
53
|
+
/**
|
|
54
|
+
* Builds a sketch recording `keys`, at the target error bound. The
|
|
55
|
+
* ergonomic entry point when the stream is already in hand.
|
|
56
|
+
*
|
|
57
|
+
* A repeated key is counted once per occurrence, which is the whole point of
|
|
58
|
+
* a frequency sketch. This is the opposite of `CuckooFilter.from`, where a
|
|
59
|
+
* repeat costs a slot and is therefore dropped.
|
|
60
|
+
*
|
|
61
|
+
* @param keys - The keys to record, repeats included.
|
|
62
|
+
* @param epsilon - Error factor relative to the total recorded.
|
|
63
|
+
* @param delta - Probability the bound is exceeded.
|
|
64
|
+
* @param options - Optional seed.
|
|
65
|
+
* @returns A new sketch holding every occurrence.
|
|
66
|
+
*/
|
|
67
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
68
|
+
/**
|
|
69
|
+
* Restores a sketch from its {@link CountMinSketch.toBytes} serialization.
|
|
70
|
+
*
|
|
71
|
+
* Built from the geometry the frame stores, never from a sizing solve: the
|
|
72
|
+
* constructor takes `width` and `depth` directly, so there is nothing to
|
|
73
|
+
* re-derive and nothing to disagree about.
|
|
74
|
+
*
|
|
75
|
+
* @param bytes - The serialized sketch.
|
|
76
|
+
* @returns The reconstructed sketch.
|
|
77
|
+
*/
|
|
78
|
+
static fromBytes(bytes: Uint8Array): CountMinSketch;
|
|
79
|
+
/**
|
|
80
|
+
* Constructs a sketch from low-level {@link CountMinParams}. Prefer
|
|
81
|
+
* {@link CountMinSketch.create} unless restoring a specific geometry.
|
|
82
|
+
*/
|
|
83
|
+
constructor({ width, depth, seed }: CountMinParams);
|
|
84
|
+
/** Counters per row. */
|
|
85
|
+
get width(): number;
|
|
86
|
+
/** Number of rows, one probe each. */
|
|
87
|
+
get depth(): number;
|
|
88
|
+
/** Hash seed. */
|
|
89
|
+
get seed(): number;
|
|
90
|
+
/** Sum of every count recorded, the denominator of the error bound. */
|
|
91
|
+
get total(): number;
|
|
92
|
+
/**
|
|
93
|
+
* Error factor the geometry implements, `e / width`.
|
|
94
|
+
*
|
|
95
|
+
* Sizing rounds `width` up, so this is at or below the `epsilon` passed to
|
|
96
|
+
* {@link CountMinSketch.create}: the sketch reports what it delivers rather
|
|
97
|
+
* than what was asked for.
|
|
98
|
+
*/
|
|
99
|
+
get epsilon(): number;
|
|
100
|
+
/**
|
|
101
|
+
* Probability the error bound is exceeded, `e ** -depth`. At or below the
|
|
102
|
+
* `delta` passed to {@link CountMinSketch.create}, for the same reason.
|
|
103
|
+
*/
|
|
104
|
+
get delta(): number;
|
|
105
|
+
/**
|
|
106
|
+
* The additive error bound right now, `epsilon * total`. A count is at most
|
|
107
|
+
* this far above the truth, with probability `1 - delta`.
|
|
108
|
+
*
|
|
109
|
+
* It grows with what the sketch has recorded, so an estimate means less as
|
|
110
|
+
* the stream goes on, the way a filter's `rate()` rises as it fills.
|
|
111
|
+
*
|
|
112
|
+
* @returns The bound, `0` for an empty sketch.
|
|
113
|
+
*/
|
|
114
|
+
error(): number;
|
|
115
|
+
/**
|
|
116
|
+
* Records `count` occurrences of a key.
|
|
117
|
+
*
|
|
118
|
+
* @param key - The key to record, as a string or bytes.
|
|
119
|
+
* @param count - How many occurrences to record; defaults to `1`.
|
|
120
|
+
*/
|
|
121
|
+
add(key: BytesLike, count?: number): void;
|
|
122
|
+
/**
|
|
123
|
+
* Returns a new sketch counting everything both this one and `other` have
|
|
124
|
+
* recorded, as if one sketch had seen both streams.
|
|
125
|
+
*
|
|
126
|
+
* Both operands are left untouched.
|
|
127
|
+
*
|
|
128
|
+
* Unlike the filters, this is not idempotent: `a.union(a)` doubles every
|
|
129
|
+
* count, because counts add where bits only ever turn on.
|
|
130
|
+
*
|
|
131
|
+
* @param other - A sketch built with an identical geometry and seed.
|
|
132
|
+
* @returns A new sketch holding both streams.
|
|
133
|
+
*/
|
|
134
|
+
union(other: CountMinSketch): CountMinSketch;
|
|
135
|
+
/**
|
|
136
|
+
* Serializes the sketch to a portable little-endian byte layout.
|
|
137
|
+
*
|
|
138
|
+
* The total is not stored. Under plain increment every row sums to it, so
|
|
139
|
+
* the field would be redundant, and a reader deriving it instead gets an
|
|
140
|
+
* integrity check on the counters for free.
|
|
141
|
+
*
|
|
142
|
+
* @returns The serialized sketch, readable by {@link CountMinSketch.fromBytes}.
|
|
143
|
+
*/
|
|
144
|
+
toBytes(): Uint8Array;
|
|
145
|
+
/**
|
|
146
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
147
|
+
* bytes, meaning identical geometry, seed and counters.
|
|
148
|
+
*
|
|
149
|
+
* @param other - The sketch to compare against.
|
|
150
|
+
* @returns `true` if the two sketches are byte-for-byte identical.
|
|
151
|
+
*/
|
|
152
|
+
equals(other: CountMinSketch): boolean;
|
|
153
|
+
/**
|
|
154
|
+
* Serializes the sketch to a JSON-friendly envelope wrapping the base64 of
|
|
155
|
+
* {@link CountMinSketch.toBytes}.
|
|
156
|
+
*
|
|
157
|
+
* @returns The envelope, readable by {@link CountMinSketch.fromJSON}.
|
|
158
|
+
*/
|
|
159
|
+
toJSON(): FilterJSON;
|
|
160
|
+
/**
|
|
161
|
+
* Restores a sketch from its {@link CountMinSketch.toJSON} envelope.
|
|
162
|
+
*
|
|
163
|
+
* @param value - The JSON envelope.
|
|
164
|
+
* @returns The reconstructed sketch.
|
|
165
|
+
*/
|
|
166
|
+
static fromJSON(value: unknown): CountMinSketch;
|
|
167
|
+
/**
|
|
168
|
+
* Estimates how many times a key was added. The estimate is never below the
|
|
169
|
+
* true count and is at most {@link CountMinSketch.error} above it.
|
|
170
|
+
*
|
|
171
|
+
* @param key - The key to estimate.
|
|
172
|
+
* @returns The estimated count, `0` for a key the sketch has not seen.
|
|
173
|
+
*/
|
|
174
|
+
count(key: BytesLike): number;
|
|
175
|
+
}
|
|
176
|
+
//#endregion
|
|
177
|
+
//#region src/countmin/sizing.d.ts
|
|
178
|
+
/** Count-Min geometry: the `CountMinParams` fields a sizing solve determines. */
|
|
179
|
+
interface CountMinSizing {
|
|
180
|
+
/** Counters per row. */
|
|
181
|
+
width: number;
|
|
182
|
+
/** Number of rows, one probe each. */
|
|
183
|
+
depth: number;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Count-Min sizing: `width` columns and `depth` rows for an estimate at most
|
|
187
|
+
* `epsilon * total` above the true count, with probability `1 - delta`.
|
|
188
|
+
*/
|
|
189
|
+
export declare function countMinSizing(epsilon: number, delta: number): CountMinSizing;
|
|
190
|
+
//#endregion
|
|
191
|
+
export { BadMagicError, ChecksumError, type CountMinOptions, type CountMinParams, type CountMinSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError };
|