distillate 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -2
- package/dist/bloom/index.cjs +1 -1
- package/dist/bloom/index.d.cts +1 -1
- package/dist/bloom/index.d.ts +1 -1
- package/dist/bloom/index.js +1 -1
- package/dist/countmin/index.cjs +194 -0
- package/dist/countmin/index.d.cts +178 -0
- package/dist/countmin/index.d.ts +178 -0
- package/dist/countmin/index.js +182 -0
- package/dist/cuckoo/index.cjs +1 -1
- package/dist/cuckoo/index.js +1 -1
- package/dist/hll/index.cjs +1 -1
- package/dist/hll/index.d.cts +1 -1
- package/dist/hll/index.d.ts +1 -1
- package/dist/hll/index.js +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/dist/scalable/index.cjs +1 -1
- package/dist/scalable/index.js +1 -1
- package/dist/{sizing-wYOW27eY.d.cts → sizing-BmKfHf_V.d.cts} +13 -1
- package/dist/{sizing-wYOW27eY.d.ts → sizing-BmKfHf_V.d.ts} +13 -1
- package/dist/{sizing-BtUrtvF_.js → sizing-D3-_7pdP.js} +9 -1
- package/dist/{sizing-BLzZk2zr.cjs → sizing-R1pwbzjC.cjs} +14 -0
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -50,10 +50,12 @@ Each structure ships as its own subpath, so you only bundle what you import.
|
|
|
50
50
|
| `distillate/fuse` | Binary Fuse | seen this key? | Static set built once and queried a lot; least space |
|
|
51
51
|
| `distillate/cuckoo` | Cuckoo | seen this key? | Keys come and go; the one filter with delete |
|
|
52
52
|
| `distillate/hll` | HyperLogLog | how many distinct? | Counting distinct users, IPs, or keys in fixed space |
|
|
53
|
+
| `distillate/countmin` | Count-Min | how many times? | Counting events per key in fixed space |
|
|
53
54
|
|
|
54
55
|
The filters are mutable except Binary Fuse, which is built once from the whole
|
|
55
|
-
key set. HyperLogLog
|
|
56
|
-
whether it saw any particular
|
|
56
|
+
key set. HyperLogLog and Count-Min are sketches rather than filters: neither
|
|
57
|
+
reports whether it saw any particular key. HyperLogLog counts distinct keys,
|
|
58
|
+
Count-Min counts how often each key appeared.
|
|
57
59
|
|
|
58
60
|
### Classic Bloom (`distillate/bloom`)
|
|
59
61
|
|
|
@@ -163,6 +165,30 @@ Below a few thousand distinct keys the sketch counts rather than estimates, so s
|
|
|
163
165
|
|
|
164
166
|
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `standardError`, `hllSizing(relativeError)`, and a low-level `new HyperLogLog({ p, seed })`.
|
|
165
167
|
|
|
168
|
+
### Count-Min (`distillate/countmin`)
|
|
169
|
+
|
|
170
|
+
A **sketch**, not a filter: it estimates how many times each key appeared, in space fixed by the error you ask for rather than by how many keys arrive. The estimate is never below the truth, and at most `epsilon` of the total above it. Full API and sizing: [Count-Min guide](https://distillate.akxp.net/guides/countmin/).
|
|
171
|
+
|
|
172
|
+
```ts
|
|
173
|
+
import { CountMinSketch } from "distillate/countmin";
|
|
174
|
+
|
|
175
|
+
const hits = CountMinSketch.create(0.001, 0.001); // error factor, failure probability
|
|
176
|
+
hits.add("/login");
|
|
177
|
+
hits.add("/login", 4);
|
|
178
|
+
hits.add("/signup");
|
|
179
|
+
|
|
180
|
+
hits.count("/login"); // 5
|
|
181
|
+
hits.total; // 6
|
|
182
|
+
|
|
183
|
+
// Combine per-shard sketches: counts add, exactly as if one sketch saw both streams.
|
|
184
|
+
const other = CountMinSketch.from(["/login", "/login"], 0.001, 0.001);
|
|
185
|
+
hits.union(other).count("/login"); // 7
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`from` counts repeats, unlike `CuckooFilter.from` which ignores them: for a frequency sketch the repeats are the measurement. A counter that would pass `2^32 - 1` throws `CountMinOverflowError` rather than wrapping, since a wrapped counter would read as an underestimate.
|
|
189
|
+
|
|
190
|
+
Also: `equals`, `toBytes` / `fromBytes`, `toJSON` / `fromJSON`, `error()`, `epsilon`, `delta`, `width`, `depth`, `countMinSizing(epsilon, delta)`, and a low-level `new CountMinSketch({ width, depth, seed })`.
|
|
191
|
+
|
|
166
192
|
## Performance
|
|
167
193
|
|
|
168
194
|
Classic Bloom head-to-head at a **matched 1% false-positive rate** over the same 100k keys, measured by identical code (cross-library harness, node v24.14.1, Apple M5):
|
package/dist/bloom/index.cjs
CHANGED
|
@@ -3,7 +3,7 @@ const require_bitset = require("../bitset-QYGqxnEf.cjs");
|
|
|
3
3
|
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
4
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
5
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
6
|
-
const require_sizing = require("../sizing-
|
|
6
|
+
const require_sizing = require("../sizing-R1pwbzjC.cjs");
|
|
7
7
|
//#region src/bloom/bloom.ts
|
|
8
8
|
const TYPE = 1;
|
|
9
9
|
const PARAMS_SIZE = 16;
|
package/dist/bloom/index.d.cts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
-
import {
|
|
4
|
+
import { i as bloomSizing, t as BloomSizing } from "../sizing-BmKfHf_V.cjs";
|
|
5
5
|
//#region src/bloom/bloom.d.ts
|
|
6
6
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
7
7
|
export declare class BloomParamMismatchError extends Error {
|
package/dist/bloom/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
-
import {
|
|
4
|
+
import { i as bloomSizing, t as BloomSizing } from "../sizing-BmKfHf_V.js";
|
|
5
5
|
//#region src/bloom/bloom.d.ts
|
|
6
6
|
/** Thrown when an operation requires two filters built with identical parameters. */
|
|
7
7
|
export declare class BloomParamMismatchError extends Error {
|
package/dist/bloom/index.js
CHANGED
|
@@ -2,7 +2,7 @@ import { t as BitSet } from "../bitset-Cf2jwRJX.js";
|
|
|
2
2
|
import { c as probeInto } from "../hasher-3Cyn7Yg-.js";
|
|
3
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
4
4
|
import { a as assertUint16, i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
5
|
-
import { t as bloomSizing } from "../sizing-
|
|
5
|
+
import { t as bloomSizing } from "../sizing-D3-_7pdP.js";
|
|
6
6
|
//#region src/bloom/bloom.ts
|
|
7
7
|
const TYPE = 1;
|
|
8
8
|
const PARAMS_SIZE = 16;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
|
+
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
|
+
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
+
const require_sizing = require("../sizing-R1pwbzjC.cjs");
|
|
6
|
+
//#region src/countmin/countmin.ts
|
|
7
|
+
const TYPE = 8;
|
|
8
|
+
const PARAMS_SIZE = 16;
|
|
9
|
+
const PARAMS_FIELDS_END = 12;
|
|
10
|
+
const counterBytes = (counters) => new Uint8Array(counters.buffer, counters.byteOffset, counters.byteLength);
|
|
11
|
+
function rowSum(counters, width, depth) {
|
|
12
|
+
let expected = 0;
|
|
13
|
+
for (let r = 0; r < depth; r++) {
|
|
14
|
+
let sum = 0;
|
|
15
|
+
const from = r * width;
|
|
16
|
+
for (let c = from; c < from + width; c++) sum += counters[c] ?? 0;
|
|
17
|
+
if (sum > Number.MAX_SAFE_INTEGER) throw new require_serialize.SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, past the safe integer range`);
|
|
18
|
+
if (r === 0) expected = sum;
|
|
19
|
+
else if (sum !== expected) throw new require_serialize.SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, but row 0 sums to ${String(expected)}`);
|
|
20
|
+
}
|
|
21
|
+
return expected;
|
|
22
|
+
}
|
|
23
|
+
var CountMinParamMismatchError = class extends Error {
|
|
24
|
+
name = "CountMinParamMismatchError";
|
|
25
|
+
};
|
|
26
|
+
var CountMinOverflowError = class extends RangeError {
|
|
27
|
+
name = "CountMinOverflowError";
|
|
28
|
+
};
|
|
29
|
+
var CountMinSketch = class CountMinSketch {
|
|
30
|
+
#counters;
|
|
31
|
+
#width;
|
|
32
|
+
#depth;
|
|
33
|
+
#seed;
|
|
34
|
+
#words = new Uint32Array(2);
|
|
35
|
+
#positions;
|
|
36
|
+
#total = 0;
|
|
37
|
+
static create(epsilon, delta, options = {}) {
|
|
38
|
+
return new CountMinSketch({
|
|
39
|
+
...options,
|
|
40
|
+
...require_sizing.countMinSizing(epsilon, delta)
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
static from(keys, epsilon, delta, options = {}) {
|
|
44
|
+
const sketch = CountMinSketch.create(epsilon, delta, options);
|
|
45
|
+
for (const key of keys) sketch.add(key);
|
|
46
|
+
return sketch;
|
|
47
|
+
}
|
|
48
|
+
static fromBytes(bytes) {
|
|
49
|
+
const { type, flags, body } = require_serialize.readHeader(bytes);
|
|
50
|
+
if (type !== TYPE) throw new require_serialize.SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
51
|
+
if ((flags & 15) !== 0) throw new require_serialize.UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
52
|
+
require_serialize.assertMinBodyLength(body.length, PARAMS_SIZE, "countmin");
|
|
53
|
+
require_serialize.assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "countmin");
|
|
54
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
55
|
+
const width = view.getUint32(0, true);
|
|
56
|
+
const depth = view.getUint32(4, true);
|
|
57
|
+
const seed = view.getUint32(8, true);
|
|
58
|
+
require_serialize.assertBodyLength(body.length, PARAMS_SIZE + 4 * width * depth, "countmin");
|
|
59
|
+
let sketch;
|
|
60
|
+
try {
|
|
61
|
+
sketch = new CountMinSketch({
|
|
62
|
+
width,
|
|
63
|
+
depth,
|
|
64
|
+
seed
|
|
65
|
+
});
|
|
66
|
+
} catch (err) {
|
|
67
|
+
if (err instanceof require_params.ParamError) throw new require_serialize.SerializationError(`countmin: ${err.message}`);
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
const counters = sketch.#counters;
|
|
71
|
+
for (let i = 0; i < counters.length; i++) counters[i] = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
72
|
+
sketch.#total = rowSum(counters, width, depth);
|
|
73
|
+
return sketch;
|
|
74
|
+
}
|
|
75
|
+
constructor({ width, depth, seed = 0 }) {
|
|
76
|
+
require_params.assertPositiveInt(width, "width");
|
|
77
|
+
require_params.assertUint32(width, "width");
|
|
78
|
+
require_params.assertPositiveInt(depth, "depth");
|
|
79
|
+
require_params.assertUint32(depth, "depth");
|
|
80
|
+
require_params.assertUint32(seed, "seed");
|
|
81
|
+
if (width * depth > 4294967295) throw new require_params.ParamError(`width ${String(width)} by depth ${String(depth)} needs ${String(width * depth)} counters, above the maximum ${String(4294967295)}`);
|
|
82
|
+
this.#counters = new Uint32Array(width * depth);
|
|
83
|
+
this.#positions = new Uint32Array(depth);
|
|
84
|
+
this.#width = width;
|
|
85
|
+
this.#depth = depth;
|
|
86
|
+
this.#seed = seed;
|
|
87
|
+
}
|
|
88
|
+
get width() {
|
|
89
|
+
return this.#width;
|
|
90
|
+
}
|
|
91
|
+
get depth() {
|
|
92
|
+
return this.#depth;
|
|
93
|
+
}
|
|
94
|
+
get seed() {
|
|
95
|
+
return this.#seed;
|
|
96
|
+
}
|
|
97
|
+
get total() {
|
|
98
|
+
return this.#total;
|
|
99
|
+
}
|
|
100
|
+
get epsilon() {
|
|
101
|
+
return Math.E / this.#width;
|
|
102
|
+
}
|
|
103
|
+
get delta() {
|
|
104
|
+
return Math.exp(-this.#depth);
|
|
105
|
+
}
|
|
106
|
+
error() {
|
|
107
|
+
return this.epsilon * this.#total;
|
|
108
|
+
}
|
|
109
|
+
#positionAt(row) {
|
|
110
|
+
return row * this.#width + require_hasher.probeAt(this.#words[0] ?? 0, this.#words[1] ?? 0, row, this.#width);
|
|
111
|
+
}
|
|
112
|
+
add(key, count = 1) {
|
|
113
|
+
require_params.assertPositiveInt(count, "count");
|
|
114
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
115
|
+
require_hasher.hash32x2Into(key, this.#seed, this.#words);
|
|
116
|
+
const headroom = 4294967295 - count;
|
|
117
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
118
|
+
const at = this.#positionAt(r);
|
|
119
|
+
if ((this.#counters[at] ?? 0) > headroom) throw new CountMinOverflowError(`adding ${String(count)} would carry a counter past ${String(4294967295)}`);
|
|
120
|
+
this.#positions[r] = at;
|
|
121
|
+
}
|
|
122
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
123
|
+
const at = this.#positions[r] ?? 0;
|
|
124
|
+
this.#counters[at] = (this.#counters[at] ?? 0) + count;
|
|
125
|
+
}
|
|
126
|
+
this.#total += count;
|
|
127
|
+
}
|
|
128
|
+
union(other) {
|
|
129
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) throw new CountMinParamMismatchError("cannot union Count-Min sketches whose parameters do not match");
|
|
130
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
131
|
+
const mine = this.#counters;
|
|
132
|
+
const theirs = other.#counters;
|
|
133
|
+
const merged = new CountMinSketch({
|
|
134
|
+
width: this.#width,
|
|
135
|
+
depth: this.#depth,
|
|
136
|
+
seed: this.#seed
|
|
137
|
+
});
|
|
138
|
+
const into = merged.#counters;
|
|
139
|
+
for (let i = 0; i < into.length; i++) {
|
|
140
|
+
const sum = (mine[i] ?? 0) + (theirs[i] ?? 0);
|
|
141
|
+
if (sum > 4294967295) throw new CountMinOverflowError(`union would carry a counter to ${String(sum)}, past ${String(4294967295)}`);
|
|
142
|
+
into[i] = sum;
|
|
143
|
+
}
|
|
144
|
+
merged.#total = this.#total + other.#total;
|
|
145
|
+
return merged;
|
|
146
|
+
}
|
|
147
|
+
toBytes() {
|
|
148
|
+
const counters = this.#counters;
|
|
149
|
+
return require_serialize.writeFrame({
|
|
150
|
+
version: 5,
|
|
151
|
+
type: TYPE,
|
|
152
|
+
flags: 0
|
|
153
|
+
}, PARAMS_SIZE, 4 * counters.length, (_, view) => {
|
|
154
|
+
view.setUint32(0, this.#width, true);
|
|
155
|
+
view.setUint32(4, this.#depth, true);
|
|
156
|
+
view.setUint32(8, this.#seed, true);
|
|
157
|
+
counters.forEach((c, i) => {
|
|
158
|
+
view.setUint32(PARAMS_SIZE + 4 * i, c, true);
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
equals(other) {
|
|
163
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) return false;
|
|
164
|
+
return require_serialize.bytesEqual(counterBytes(this.#counters), counterBytes(other.#counters));
|
|
165
|
+
}
|
|
166
|
+
toJSON() {
|
|
167
|
+
return require_serialize.toJSONEnvelope(this.toBytes());
|
|
168
|
+
}
|
|
169
|
+
static fromJSON(value) {
|
|
170
|
+
return CountMinSketch.fromBytes(require_serialize.fromJSONEnvelope(value));
|
|
171
|
+
}
|
|
172
|
+
count(key) {
|
|
173
|
+
require_hasher.hash32x2Into(key, this.#seed, this.#words);
|
|
174
|
+
let min = Infinity;
|
|
175
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
176
|
+
const at = this.#counters[this.#positionAt(r)] ?? 0;
|
|
177
|
+
if (at < min) min = at;
|
|
178
|
+
}
|
|
179
|
+
return min;
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
//#endregion
|
|
183
|
+
exports.BadMagicError = require_serialize.BadMagicError;
|
|
184
|
+
exports.ChecksumError = require_serialize.ChecksumError;
|
|
185
|
+
exports.CountMinOverflowError = CountMinOverflowError;
|
|
186
|
+
exports.CountMinParamMismatchError = CountMinParamMismatchError;
|
|
187
|
+
exports.CountMinSketch = CountMinSketch;
|
|
188
|
+
exports.ParamError = require_params.ParamError;
|
|
189
|
+
exports.ReservedBitsError = require_serialize.ReservedBitsError;
|
|
190
|
+
exports.SerializationError = require_serialize.SerializationError;
|
|
191
|
+
exports.TruncatedError = require_serialize.TruncatedError;
|
|
192
|
+
exports.UnknownHashVariantError = require_serialize.UnknownHashVariantError;
|
|
193
|
+
exports.UnknownVersionError = require_serialize.UnknownVersionError;
|
|
194
|
+
exports.countMinSizing = require_sizing.countMinSizing;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
+
import { a as countMinSizing, n as CountMinSizing } from "../sizing-BmKfHf_V.cjs";
|
|
5
|
+
//#region src/countmin/countmin.d.ts
|
|
6
|
+
/** Thrown when an operation requires two sketches built with identical parameters. */
|
|
7
|
+
export declare class CountMinParamMismatchError extends Error {
|
|
8
|
+
/** Discriminates this error from other `Error`s. */
|
|
9
|
+
override readonly name = "CountMinParamMismatchError";
|
|
10
|
+
}
|
|
11
|
+
/** Thrown when an add would carry a counter past what a u32 holds. */
|
|
12
|
+
export declare class CountMinOverflowError extends RangeError {
|
|
13
|
+
/** Discriminates this error from other `Error`s. */
|
|
14
|
+
override readonly name = "CountMinOverflowError";
|
|
15
|
+
}
|
|
16
|
+
/** Options accepted alongside a sizing solve: everything but the geometry. */
|
|
17
|
+
type CountMinOptions = Omit<CountMinParams, "width" | "depth">;
|
|
18
|
+
/** Low-level Count-Min sketch parameters. */
|
|
19
|
+
interface CountMinParams {
|
|
20
|
+
/** Counters per row. */
|
|
21
|
+
width: number;
|
|
22
|
+
/** Number of rows, one probe each. */
|
|
23
|
+
depth: number;
|
|
24
|
+
/** Hash seed; defaults to `0`. */
|
|
25
|
+
seed?: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A Count-Min sketch: a fixed-size frequency estimate with a tunable
|
|
29
|
+
* overestimate and no underestimate.
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```ts
|
|
33
|
+
* const sketch = CountMinSketch.create(0.001, 0.001);
|
|
34
|
+
* sketch.add("alice");
|
|
35
|
+
* sketch.add("alice", 3);
|
|
36
|
+
* sketch.count("alice"); // 4 (or a rare overestimate)
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
export declare class CountMinSketch {
|
|
40
|
+
#private;
|
|
41
|
+
/**
|
|
42
|
+
* Creates a sketch whose estimate is at most `epsilon * total` above the
|
|
43
|
+
* true count, with probability `1 - delta`.
|
|
44
|
+
*
|
|
45
|
+
* Unlike a filter, a sketch is sized by the error it targets rather than by
|
|
46
|
+
* how many keys it will see, so no key count is needed.
|
|
47
|
+
*
|
|
48
|
+
* @param epsilon - Error factor relative to the total recorded, e.g. `0.001`.
|
|
49
|
+
* @param delta - Probability the bound is exceeded, e.g. `0.001`.
|
|
50
|
+
* @param options - Optional seed.
|
|
51
|
+
* @returns A new, empty sketch.
|
|
52
|
+
*/
|
|
53
|
+
static create(epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
54
|
+
/**
|
|
55
|
+
* Builds a sketch recording `keys`, at the target error bound. The
|
|
56
|
+
* ergonomic entry point when the stream is already in hand.
|
|
57
|
+
*
|
|
58
|
+
* A repeated key is counted once per occurrence, which is the whole point of
|
|
59
|
+
* a frequency sketch. This is the opposite of `CuckooFilter.from`, where a
|
|
60
|
+
* repeat costs a slot and is therefore dropped.
|
|
61
|
+
*
|
|
62
|
+
* @param keys - The keys to record, repeats included.
|
|
63
|
+
* @param epsilon - Error factor relative to the total recorded.
|
|
64
|
+
* @param delta - Probability the bound is exceeded.
|
|
65
|
+
* @param options - Optional seed.
|
|
66
|
+
* @returns A new sketch holding every occurrence.
|
|
67
|
+
*/
|
|
68
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
69
|
+
/**
|
|
70
|
+
* Restores a sketch from its {@link CountMinSketch.toBytes} serialization.
|
|
71
|
+
*
|
|
72
|
+
* Built from the geometry the frame stores, never from a sizing solve: the
|
|
73
|
+
* constructor takes `width` and `depth` directly, so there is nothing to
|
|
74
|
+
* re-derive and nothing to disagree about.
|
|
75
|
+
*
|
|
76
|
+
* @param bytes - The serialized sketch.
|
|
77
|
+
* @returns The reconstructed sketch.
|
|
78
|
+
*/
|
|
79
|
+
static fromBytes(bytes: Uint8Array): CountMinSketch;
|
|
80
|
+
/**
|
|
81
|
+
* Constructs a sketch from low-level {@link CountMinParams}. Prefer
|
|
82
|
+
* {@link CountMinSketch.create} unless restoring a specific geometry.
|
|
83
|
+
*/
|
|
84
|
+
constructor({ width, depth, seed }: CountMinParams);
|
|
85
|
+
/** Counters per row. */
|
|
86
|
+
get width(): number;
|
|
87
|
+
/** Number of rows, one probe each. */
|
|
88
|
+
get depth(): number;
|
|
89
|
+
/** Hash seed. */
|
|
90
|
+
get seed(): number;
|
|
91
|
+
/** Sum of every count recorded, the denominator of the error bound. */
|
|
92
|
+
get total(): number;
|
|
93
|
+
/**
|
|
94
|
+
* Error factor the geometry implements, `e / width`.
|
|
95
|
+
*
|
|
96
|
+
* Sizing rounds `width` up, so this is at or below the `epsilon` passed to
|
|
97
|
+
* {@link CountMinSketch.create}: the sketch reports what it delivers rather
|
|
98
|
+
* than what was asked for.
|
|
99
|
+
*/
|
|
100
|
+
get epsilon(): number;
|
|
101
|
+
/**
|
|
102
|
+
* Probability the error bound is exceeded, `e ** -depth`. At or below the
|
|
103
|
+
* `delta` passed to {@link CountMinSketch.create}, for the same reason.
|
|
104
|
+
*/
|
|
105
|
+
get delta(): number;
|
|
106
|
+
/**
|
|
107
|
+
* The additive error bound right now, `epsilon * total`. A count is at most
|
|
108
|
+
* this far above the truth, with probability `1 - delta`.
|
|
109
|
+
*
|
|
110
|
+
* It grows with what the sketch has recorded, so an estimate means less as
|
|
111
|
+
* the stream goes on, the way a filter's `rate()` rises as it fills.
|
|
112
|
+
*
|
|
113
|
+
* @returns The bound, `0` for an empty sketch.
|
|
114
|
+
*/
|
|
115
|
+
error(): number;
|
|
116
|
+
/**
|
|
117
|
+
* Records `count` occurrences of a key.
|
|
118
|
+
*
|
|
119
|
+
* @param key - The key to record, as a string or bytes.
|
|
120
|
+
* @param count - How many occurrences to record; defaults to `1`.
|
|
121
|
+
*/
|
|
122
|
+
add(key: BytesLike, count?: number): void;
|
|
123
|
+
/**
|
|
124
|
+
* Returns a new sketch counting everything both this one and `other` have
|
|
125
|
+
* recorded, as if one sketch had seen both streams.
|
|
126
|
+
*
|
|
127
|
+
* Both operands are left untouched.
|
|
128
|
+
*
|
|
129
|
+
* Unlike the filters, this is not idempotent: `a.union(a)` doubles every
|
|
130
|
+
* count, because counts add where bits only ever turn on.
|
|
131
|
+
*
|
|
132
|
+
* @param other - A sketch built with an identical geometry and seed.
|
|
133
|
+
* @returns A new sketch holding both streams.
|
|
134
|
+
*/
|
|
135
|
+
union(other: CountMinSketch): CountMinSketch;
|
|
136
|
+
/**
|
|
137
|
+
* Serializes the sketch to a portable little-endian byte layout.
|
|
138
|
+
*
|
|
139
|
+
* The total is not stored. Under plain increment every row sums to it, so
|
|
140
|
+
* the field would be redundant, and a reader deriving it instead gets an
|
|
141
|
+
* integrity check on the counters for free.
|
|
142
|
+
*
|
|
143
|
+
* @returns The serialized sketch, readable by {@link CountMinSketch.fromBytes}.
|
|
144
|
+
*/
|
|
145
|
+
toBytes(): Uint8Array;
|
|
146
|
+
/**
|
|
147
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
148
|
+
* bytes, meaning identical geometry, seed and counters.
|
|
149
|
+
*
|
|
150
|
+
* @param other - The sketch to compare against.
|
|
151
|
+
* @returns `true` if the two sketches are byte-for-byte identical.
|
|
152
|
+
*/
|
|
153
|
+
equals(other: CountMinSketch): boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Serializes the sketch to a JSON-friendly envelope wrapping the base64 of
|
|
156
|
+
* {@link CountMinSketch.toBytes}.
|
|
157
|
+
*
|
|
158
|
+
* @returns The envelope, readable by {@link CountMinSketch.fromJSON}.
|
|
159
|
+
*/
|
|
160
|
+
toJSON(): FilterJSON;
|
|
161
|
+
/**
|
|
162
|
+
* Restores a sketch from its {@link CountMinSketch.toJSON} envelope.
|
|
163
|
+
*
|
|
164
|
+
* @param value - The JSON envelope.
|
|
165
|
+
* @returns The reconstructed sketch.
|
|
166
|
+
*/
|
|
167
|
+
static fromJSON(value: unknown): CountMinSketch;
|
|
168
|
+
/**
|
|
169
|
+
* Estimates how many times a key was added. The estimate is never below the
|
|
170
|
+
* true count and is at most {@link CountMinSketch.error} above it.
|
|
171
|
+
*
|
|
172
|
+
* @param key - The key to estimate.
|
|
173
|
+
* @returns The estimated count, `0` for a key the sketch has not seen.
|
|
174
|
+
*/
|
|
175
|
+
count(key: BytesLike): number;
|
|
176
|
+
}
|
|
177
|
+
//#endregion
|
|
178
|
+
export { BadMagicError, ChecksumError, type CountMinOptions, type CountMinParams, type CountMinSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, countMinSizing };
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
|
+
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
+
import { a as countMinSizing, n as CountMinSizing } from "../sizing-BmKfHf_V.js";
|
|
5
|
+
//#region src/countmin/countmin.d.ts
|
|
6
|
+
/** Thrown when an operation requires two sketches built with identical parameters. */
|
|
7
|
+
export declare class CountMinParamMismatchError extends Error {
|
|
8
|
+
/** Discriminates this error from other `Error`s. */
|
|
9
|
+
override readonly name = "CountMinParamMismatchError";
|
|
10
|
+
}
|
|
11
|
+
/** Thrown when an add would carry a counter past what a u32 holds. */
|
|
12
|
+
export declare class CountMinOverflowError extends RangeError {
|
|
13
|
+
/** Discriminates this error from other `Error`s. */
|
|
14
|
+
override readonly name = "CountMinOverflowError";
|
|
15
|
+
}
|
|
16
|
+
/** Options accepted alongside a sizing solve: everything but the geometry. */
|
|
17
|
+
type CountMinOptions = Omit<CountMinParams, "width" | "depth">;
|
|
18
|
+
/** Low-level Count-Min sketch parameters. */
|
|
19
|
+
interface CountMinParams {
|
|
20
|
+
/** Counters per row. */
|
|
21
|
+
width: number;
|
|
22
|
+
/** Number of rows, one probe each. */
|
|
23
|
+
depth: number;
|
|
24
|
+
/** Hash seed; defaults to `0`. */
|
|
25
|
+
seed?: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A Count-Min sketch: a fixed-size frequency estimate with a tunable
|
|
29
|
+
* overestimate and no underestimate.
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```ts
|
|
33
|
+
* const sketch = CountMinSketch.create(0.001, 0.001);
|
|
34
|
+
* sketch.add("alice");
|
|
35
|
+
* sketch.add("alice", 3);
|
|
36
|
+
* sketch.count("alice"); // 4 (or a rare overestimate)
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
export declare class CountMinSketch {
|
|
40
|
+
#private;
|
|
41
|
+
/**
|
|
42
|
+
* Creates a sketch whose estimate is at most `epsilon * total` above the
|
|
43
|
+
* true count, with probability `1 - delta`.
|
|
44
|
+
*
|
|
45
|
+
* Unlike a filter, a sketch is sized by the error it targets rather than by
|
|
46
|
+
* how many keys it will see, so no key count is needed.
|
|
47
|
+
*
|
|
48
|
+
* @param epsilon - Error factor relative to the total recorded, e.g. `0.001`.
|
|
49
|
+
* @param delta - Probability the bound is exceeded, e.g. `0.001`.
|
|
50
|
+
* @param options - Optional seed.
|
|
51
|
+
* @returns A new, empty sketch.
|
|
52
|
+
*/
|
|
53
|
+
static create(epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
54
|
+
/**
|
|
55
|
+
* Builds a sketch recording `keys`, at the target error bound. The
|
|
56
|
+
* ergonomic entry point when the stream is already in hand.
|
|
57
|
+
*
|
|
58
|
+
* A repeated key is counted once per occurrence, which is the whole point of
|
|
59
|
+
* a frequency sketch. This is the opposite of `CuckooFilter.from`, where a
|
|
60
|
+
* repeat costs a slot and is therefore dropped.
|
|
61
|
+
*
|
|
62
|
+
* @param keys - The keys to record, repeats included.
|
|
63
|
+
* @param epsilon - Error factor relative to the total recorded.
|
|
64
|
+
* @param delta - Probability the bound is exceeded.
|
|
65
|
+
* @param options - Optional seed.
|
|
66
|
+
* @returns A new sketch holding every occurrence.
|
|
67
|
+
*/
|
|
68
|
+
static from(keys: Iterable<BytesLike>, epsilon: number, delta: number, options?: CountMinOptions): CountMinSketch;
|
|
69
|
+
/**
|
|
70
|
+
* Restores a sketch from its {@link CountMinSketch.toBytes} serialization.
|
|
71
|
+
*
|
|
72
|
+
* Built from the geometry the frame stores, never from a sizing solve: the
|
|
73
|
+
* constructor takes `width` and `depth` directly, so there is nothing to
|
|
74
|
+
* re-derive and nothing to disagree about.
|
|
75
|
+
*
|
|
76
|
+
* @param bytes - The serialized sketch.
|
|
77
|
+
* @returns The reconstructed sketch.
|
|
78
|
+
*/
|
|
79
|
+
static fromBytes(bytes: Uint8Array): CountMinSketch;
|
|
80
|
+
/**
|
|
81
|
+
* Constructs a sketch from low-level {@link CountMinParams}. Prefer
|
|
82
|
+
* {@link CountMinSketch.create} unless restoring a specific geometry.
|
|
83
|
+
*/
|
|
84
|
+
constructor({ width, depth, seed }: CountMinParams);
|
|
85
|
+
/** Counters per row. */
|
|
86
|
+
get width(): number;
|
|
87
|
+
/** Number of rows, one probe each. */
|
|
88
|
+
get depth(): number;
|
|
89
|
+
/** Hash seed. */
|
|
90
|
+
get seed(): number;
|
|
91
|
+
/** Sum of every count recorded, the denominator of the error bound. */
|
|
92
|
+
get total(): number;
|
|
93
|
+
/**
|
|
94
|
+
* Error factor the geometry implements, `e / width`.
|
|
95
|
+
*
|
|
96
|
+
* Sizing rounds `width` up, so this is at or below the `epsilon` passed to
|
|
97
|
+
* {@link CountMinSketch.create}: the sketch reports what it delivers rather
|
|
98
|
+
* than what was asked for.
|
|
99
|
+
*/
|
|
100
|
+
get epsilon(): number;
|
|
101
|
+
/**
|
|
102
|
+
* Probability the error bound is exceeded, `e ** -depth`. At or below the
|
|
103
|
+
* `delta` passed to {@link CountMinSketch.create}, for the same reason.
|
|
104
|
+
*/
|
|
105
|
+
get delta(): number;
|
|
106
|
+
/**
|
|
107
|
+
* The additive error bound right now, `epsilon * total`. A count is at most
|
|
108
|
+
* this far above the truth, with probability `1 - delta`.
|
|
109
|
+
*
|
|
110
|
+
* It grows with what the sketch has recorded, so an estimate means less as
|
|
111
|
+
* the stream goes on, the way a filter's `rate()` rises as it fills.
|
|
112
|
+
*
|
|
113
|
+
* @returns The bound, `0` for an empty sketch.
|
|
114
|
+
*/
|
|
115
|
+
error(): number;
|
|
116
|
+
/**
|
|
117
|
+
* Records `count` occurrences of a key.
|
|
118
|
+
*
|
|
119
|
+
* @param key - The key to record, as a string or bytes.
|
|
120
|
+
* @param count - How many occurrences to record; defaults to `1`.
|
|
121
|
+
*/
|
|
122
|
+
add(key: BytesLike, count?: number): void;
|
|
123
|
+
/**
|
|
124
|
+
* Returns a new sketch counting everything both this one and `other` have
|
|
125
|
+
* recorded, as if one sketch had seen both streams.
|
|
126
|
+
*
|
|
127
|
+
* Both operands are left untouched.
|
|
128
|
+
*
|
|
129
|
+
* Unlike the filters, this is not idempotent: `a.union(a)` doubles every
|
|
130
|
+
* count, because counts add where bits only ever turn on.
|
|
131
|
+
*
|
|
132
|
+
* @param other - A sketch built with an identical geometry and seed.
|
|
133
|
+
* @returns A new sketch holding both streams.
|
|
134
|
+
*/
|
|
135
|
+
union(other: CountMinSketch): CountMinSketch;
|
|
136
|
+
/**
|
|
137
|
+
* Serializes the sketch to a portable little-endian byte layout.
|
|
138
|
+
*
|
|
139
|
+
* The total is not stored. Under plain increment every row sums to it, so
|
|
140
|
+
* the field would be redundant, and a reader deriving it instead gets an
|
|
141
|
+
* integrity check on the counters for free.
|
|
142
|
+
*
|
|
143
|
+
* @returns The serialized sketch, readable by {@link CountMinSketch.fromBytes}.
|
|
144
|
+
*/
|
|
145
|
+
toBytes(): Uint8Array;
|
|
146
|
+
/**
|
|
147
|
+
* Tests structural equality: `true` when `other` serializes to identical
|
|
148
|
+
* bytes, meaning identical geometry, seed and counters.
|
|
149
|
+
*
|
|
150
|
+
* @param other - The sketch to compare against.
|
|
151
|
+
* @returns `true` if the two sketches are byte-for-byte identical.
|
|
152
|
+
*/
|
|
153
|
+
equals(other: CountMinSketch): boolean;
|
|
154
|
+
/**
|
|
155
|
+
* Serializes the sketch to a JSON-friendly envelope wrapping the base64 of
|
|
156
|
+
* {@link CountMinSketch.toBytes}.
|
|
157
|
+
*
|
|
158
|
+
* @returns The envelope, readable by {@link CountMinSketch.fromJSON}.
|
|
159
|
+
*/
|
|
160
|
+
toJSON(): FilterJSON;
|
|
161
|
+
/**
|
|
162
|
+
* Restores a sketch from its {@link CountMinSketch.toJSON} envelope.
|
|
163
|
+
*
|
|
164
|
+
* @param value - The JSON envelope.
|
|
165
|
+
* @returns The reconstructed sketch.
|
|
166
|
+
*/
|
|
167
|
+
static fromJSON(value: unknown): CountMinSketch;
|
|
168
|
+
/**
|
|
169
|
+
* Estimates how many times a key was added. The estimate is never below the
|
|
170
|
+
* true count and is at most {@link CountMinSketch.error} above it.
|
|
171
|
+
*
|
|
172
|
+
* @param key - The key to estimate.
|
|
173
|
+
* @returns The estimated count, `0` for a key the sketch has not seen.
|
|
174
|
+
*/
|
|
175
|
+
count(key: BytesLike): number;
|
|
176
|
+
}
|
|
177
|
+
//#endregion
|
|
178
|
+
export { BadMagicError, ChecksumError, type CountMinOptions, type CountMinParams, type CountMinSizing, type FilterJSON, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, countMinSizing };
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { a as hash32x2Into, s as probeAt } from "../hasher-3Cyn7Yg-.js";
|
|
2
|
+
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
|
+
import { o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
+
import { n as countMinSizing } from "../sizing-D3-_7pdP.js";
|
|
5
|
+
//#region src/countmin/countmin.ts
|
|
6
|
+
const TYPE = 8;
|
|
7
|
+
const PARAMS_SIZE = 16;
|
|
8
|
+
const PARAMS_FIELDS_END = 12;
|
|
9
|
+
const counterBytes = (counters) => new Uint8Array(counters.buffer, counters.byteOffset, counters.byteLength);
|
|
10
|
+
function rowSum(counters, width, depth) {
|
|
11
|
+
let expected = 0;
|
|
12
|
+
for (let r = 0; r < depth; r++) {
|
|
13
|
+
let sum = 0;
|
|
14
|
+
const from = r * width;
|
|
15
|
+
for (let c = from; c < from + width; c++) sum += counters[c] ?? 0;
|
|
16
|
+
if (sum > Number.MAX_SAFE_INTEGER) throw new SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, past the safe integer range`);
|
|
17
|
+
if (r === 0) expected = sum;
|
|
18
|
+
else if (sum !== expected) throw new SerializationError(`countmin: row ${String(r)} sums to ${String(sum)}, but row 0 sums to ${String(expected)}`);
|
|
19
|
+
}
|
|
20
|
+
return expected;
|
|
21
|
+
}
|
|
22
|
+
var CountMinParamMismatchError = class extends Error {
|
|
23
|
+
name = "CountMinParamMismatchError";
|
|
24
|
+
};
|
|
25
|
+
var CountMinOverflowError = class extends RangeError {
|
|
26
|
+
name = "CountMinOverflowError";
|
|
27
|
+
};
|
|
28
|
+
var CountMinSketch = class CountMinSketch {
|
|
29
|
+
#counters;
|
|
30
|
+
#width;
|
|
31
|
+
#depth;
|
|
32
|
+
#seed;
|
|
33
|
+
#words = new Uint32Array(2);
|
|
34
|
+
#positions;
|
|
35
|
+
#total = 0;
|
|
36
|
+
static create(epsilon, delta, options = {}) {
|
|
37
|
+
return new CountMinSketch({
|
|
38
|
+
...options,
|
|
39
|
+
...countMinSizing(epsilon, delta)
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
static from(keys, epsilon, delta, options = {}) {
|
|
43
|
+
const sketch = CountMinSketch.create(epsilon, delta, options);
|
|
44
|
+
for (const key of keys) sketch.add(key);
|
|
45
|
+
return sketch;
|
|
46
|
+
}
|
|
47
|
+
static fromBytes(bytes) {
|
|
48
|
+
const { type, flags, body } = readHeader(bytes);
|
|
49
|
+
if (type !== TYPE) throw new SerializationError(`expected DSTL type ${String(TYPE)}, got ${String(type)}`);
|
|
50
|
+
if ((flags & 15) !== 0) throw new UnknownHashVariantError(`unsupported hash variant ${String(flags & 15)}`);
|
|
51
|
+
assertMinBodyLength(body.length, PARAMS_SIZE, "countmin");
|
|
52
|
+
assertParamsPadding(body, PARAMS_FIELDS_END, PARAMS_SIZE, "countmin");
|
|
53
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
54
|
+
const width = view.getUint32(0, true);
|
|
55
|
+
const depth = view.getUint32(4, true);
|
|
56
|
+
const seed = view.getUint32(8, true);
|
|
57
|
+
assertBodyLength(body.length, PARAMS_SIZE + 4 * width * depth, "countmin");
|
|
58
|
+
let sketch;
|
|
59
|
+
try {
|
|
60
|
+
sketch = new CountMinSketch({
|
|
61
|
+
width,
|
|
62
|
+
depth,
|
|
63
|
+
seed
|
|
64
|
+
});
|
|
65
|
+
} catch (err) {
|
|
66
|
+
if (err instanceof ParamError) throw new SerializationError(`countmin: ${err.message}`);
|
|
67
|
+
throw err;
|
|
68
|
+
}
|
|
69
|
+
const counters = sketch.#counters;
|
|
70
|
+
for (let i = 0; i < counters.length; i++) counters[i] = view.getUint32(PARAMS_SIZE + 4 * i, true);
|
|
71
|
+
sketch.#total = rowSum(counters, width, depth);
|
|
72
|
+
return sketch;
|
|
73
|
+
}
|
|
74
|
+
constructor({ width, depth, seed = 0 }) {
|
|
75
|
+
assertPositiveInt(width, "width");
|
|
76
|
+
assertUint32(width, "width");
|
|
77
|
+
assertPositiveInt(depth, "depth");
|
|
78
|
+
assertUint32(depth, "depth");
|
|
79
|
+
assertUint32(seed, "seed");
|
|
80
|
+
if (width * depth > 4294967295) throw new ParamError(`width ${String(width)} by depth ${String(depth)} needs ${String(width * depth)} counters, above the maximum ${String(4294967295)}`);
|
|
81
|
+
this.#counters = new Uint32Array(width * depth);
|
|
82
|
+
this.#positions = new Uint32Array(depth);
|
|
83
|
+
this.#width = width;
|
|
84
|
+
this.#depth = depth;
|
|
85
|
+
this.#seed = seed;
|
|
86
|
+
}
|
|
87
|
+
get width() {
|
|
88
|
+
return this.#width;
|
|
89
|
+
}
|
|
90
|
+
get depth() {
|
|
91
|
+
return this.#depth;
|
|
92
|
+
}
|
|
93
|
+
get seed() {
|
|
94
|
+
return this.#seed;
|
|
95
|
+
}
|
|
96
|
+
get total() {
|
|
97
|
+
return this.#total;
|
|
98
|
+
}
|
|
99
|
+
get epsilon() {
|
|
100
|
+
return Math.E / this.#width;
|
|
101
|
+
}
|
|
102
|
+
get delta() {
|
|
103
|
+
return Math.exp(-this.#depth);
|
|
104
|
+
}
|
|
105
|
+
error() {
|
|
106
|
+
return this.epsilon * this.#total;
|
|
107
|
+
}
|
|
108
|
+
#positionAt(row) {
|
|
109
|
+
return row * this.#width + probeAt(this.#words[0] ?? 0, this.#words[1] ?? 0, row, this.#width);
|
|
110
|
+
}
|
|
111
|
+
add(key, count = 1) {
|
|
112
|
+
assertPositiveInt(count, "count");
|
|
113
|
+
if (this.#total + count > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`adding ${String(count)} would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
114
|
+
hash32x2Into(key, this.#seed, this.#words);
|
|
115
|
+
const headroom = 4294967295 - count;
|
|
116
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
117
|
+
const at = this.#positionAt(r);
|
|
118
|
+
if ((this.#counters[at] ?? 0) > headroom) throw new CountMinOverflowError(`adding ${String(count)} would carry a counter past ${String(4294967295)}`);
|
|
119
|
+
this.#positions[r] = at;
|
|
120
|
+
}
|
|
121
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
122
|
+
const at = this.#positions[r] ?? 0;
|
|
123
|
+
this.#counters[at] = (this.#counters[at] ?? 0) + count;
|
|
124
|
+
}
|
|
125
|
+
this.#total += count;
|
|
126
|
+
}
|
|
127
|
+
union(other) {
|
|
128
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) throw new CountMinParamMismatchError("cannot union Count-Min sketches whose parameters do not match");
|
|
129
|
+
if (this.#total + other.#total > Number.MAX_SAFE_INTEGER) throw new CountMinOverflowError(`union would carry the total past ${String(Number.MAX_SAFE_INTEGER)}`);
|
|
130
|
+
const mine = this.#counters;
|
|
131
|
+
const theirs = other.#counters;
|
|
132
|
+
const merged = new CountMinSketch({
|
|
133
|
+
width: this.#width,
|
|
134
|
+
depth: this.#depth,
|
|
135
|
+
seed: this.#seed
|
|
136
|
+
});
|
|
137
|
+
const into = merged.#counters;
|
|
138
|
+
for (let i = 0; i < into.length; i++) {
|
|
139
|
+
const sum = (mine[i] ?? 0) + (theirs[i] ?? 0);
|
|
140
|
+
if (sum > 4294967295) throw new CountMinOverflowError(`union would carry a counter to ${String(sum)}, past ${String(4294967295)}`);
|
|
141
|
+
into[i] = sum;
|
|
142
|
+
}
|
|
143
|
+
merged.#total = this.#total + other.#total;
|
|
144
|
+
return merged;
|
|
145
|
+
}
|
|
146
|
+
toBytes() {
|
|
147
|
+
const counters = this.#counters;
|
|
148
|
+
return writeFrame({
|
|
149
|
+
version: 5,
|
|
150
|
+
type: TYPE,
|
|
151
|
+
flags: 0
|
|
152
|
+
}, PARAMS_SIZE, 4 * counters.length, (_, view) => {
|
|
153
|
+
view.setUint32(0, this.#width, true);
|
|
154
|
+
view.setUint32(4, this.#depth, true);
|
|
155
|
+
view.setUint32(8, this.#seed, true);
|
|
156
|
+
counters.forEach((c, i) => {
|
|
157
|
+
view.setUint32(PARAMS_SIZE + 4 * i, c, true);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
equals(other) {
|
|
162
|
+
if (this.#width !== other.#width || this.#depth !== other.#depth || this.#seed !== other.#seed) return false;
|
|
163
|
+
return bytesEqual(counterBytes(this.#counters), counterBytes(other.#counters));
|
|
164
|
+
}
|
|
165
|
+
toJSON() {
|
|
166
|
+
return toJSONEnvelope(this.toBytes());
|
|
167
|
+
}
|
|
168
|
+
static fromJSON(value) {
|
|
169
|
+
return CountMinSketch.fromBytes(fromJSONEnvelope(value));
|
|
170
|
+
}
|
|
171
|
+
count(key) {
|
|
172
|
+
hash32x2Into(key, this.#seed, this.#words);
|
|
173
|
+
let min = Infinity;
|
|
174
|
+
for (let r = 0; r < this.#depth; r++) {
|
|
175
|
+
const at = this.#counters[this.#positionAt(r)] ?? 0;
|
|
176
|
+
if (at < min) min = at;
|
|
177
|
+
}
|
|
178
|
+
return min;
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
//#endregion
|
|
182
|
+
export { BadMagicError, ChecksumError, CountMinOverflowError, CountMinParamMismatchError, CountMinSketch, ParamError, ReservedBitsError, SerializationError, TruncatedError, UnknownHashVariantError, UnknownVersionError, countMinSizing };
|
package/dist/cuckoo/index.cjs
CHANGED
|
@@ -195,7 +195,7 @@ var CuckooFilter = class CuckooFilter {
|
|
|
195
195
|
});
|
|
196
196
|
}
|
|
197
197
|
equals(other) {
|
|
198
|
-
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && require_serialize.bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
198
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && this.#f === other.#f && this.#buckets === other.#buckets && require_serialize.bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
199
199
|
}
|
|
200
200
|
toJSON() {
|
|
201
201
|
return require_serialize.toJSONEnvelope(this.toBytes());
|
package/dist/cuckoo/index.js
CHANGED
|
@@ -194,7 +194,7 @@ var CuckooFilter = class CuckooFilter {
|
|
|
194
194
|
});
|
|
195
195
|
}
|
|
196
196
|
equals(other) {
|
|
197
|
-
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
197
|
+
return this.#n === other.#n && this.#epsilon === other.#epsilon && this.#seed === other.#seed && this.#count === other.#count && this.#f === other.#f && this.#buckets === other.#buckets && bytesEqual(wordBytes(this.#words), wordBytes(other.#words));
|
|
198
198
|
}
|
|
199
199
|
toJSON() {
|
|
200
200
|
return toJSONEnvelope(this.toBytes());
|
package/dist/hll/index.cjs
CHANGED
|
@@ -2,7 +2,7 @@ Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
|
2
2
|
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
3
3
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
4
4
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
5
|
-
const require_sizing = require("../sizing-
|
|
5
|
+
const require_sizing = require("../sizing-R1pwbzjC.cjs");
|
|
6
6
|
//#region src/hll/estimate.ts
|
|
7
7
|
const ALPHA_INF = .5 / Math.LN2;
|
|
8
8
|
function sigma(x) {
|
package/dist/hll/index.d.cts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.cjs";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.cjs";
|
|
4
|
-
import {
|
|
4
|
+
import { o as hllSizing, r as HllSizing } from "../sizing-BmKfHf_V.cjs";
|
|
5
5
|
//#region src/hll/hll.d.ts
|
|
6
6
|
/** Construction parameters for a {@link HyperLogLog} sketch. */
|
|
7
7
|
interface HllParams {
|
package/dist/hll/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
2
|
import { a as ReservedBitsError, c as UnknownHashVariantError, l as UnknownVersionError, n as ChecksumError, o as SerializationError, r as FilterJSON, s as TruncatedError, t as BadMagicError } from "../serialize--xR6vGSW.js";
|
|
3
3
|
import { t as ParamError } from "../params-DnqJBqLS.js";
|
|
4
|
-
import {
|
|
4
|
+
import { o as hllSizing, r as HllSizing } from "../sizing-BmKfHf_V.js";
|
|
5
5
|
//#region src/hll/hll.d.ts
|
|
6
6
|
/** Construction parameters for a {@link HyperLogLog} sketch. */
|
|
7
7
|
interface HllParams {
|
package/dist/hll/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { i as hash128KeyInto } from "../hasher-3Cyn7Yg-.js";
|
|
2
2
|
import { a as TruncatedError, c as assertBodyLength, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
3
3
|
import { o as assertUint32, t as ParamError } from "../params-CeajSrwv.js";
|
|
4
|
-
import {
|
|
4
|
+
import { r as hllSizing } from "../sizing-D3-_7pdP.js";
|
|
5
5
|
//#region src/hll/estimate.ts
|
|
6
6
|
const ALPHA_INF = .5 / Math.LN2;
|
|
7
7
|
function sigma(x) {
|
package/dist/index.cjs
CHANGED
package/dist/index.js
CHANGED
package/dist/scalable/index.cjs
CHANGED
|
@@ -3,7 +3,7 @@ const require_bitset = require("../bitset-QYGqxnEf.cjs");
|
|
|
3
3
|
const require_hasher = require("../hasher-MhY7aQ8F.cjs");
|
|
4
4
|
const require_serialize = require("../serialize-CHikZ4GH.cjs");
|
|
5
5
|
const require_params = require("../params-UTZbJ22c.cjs");
|
|
6
|
-
const require_sizing = require("../sizing-
|
|
6
|
+
const require_sizing = require("../sizing-R1pwbzjC.cjs");
|
|
7
7
|
//#region src/scalable/scalable.ts
|
|
8
8
|
const MAX_BITS = 4294967295;
|
|
9
9
|
const TYPE = 6;
|
package/dist/scalable/index.js
CHANGED
|
@@ -2,7 +2,7 @@ import { t as BitSet } from "../bitset-Cf2jwRJX.js";
|
|
|
2
2
|
import { i as hash128KeyInto, l as probeLanesInto, s as probeAt } from "../hasher-3Cyn7Yg-.js";
|
|
3
3
|
import { a as TruncatedError, c as assertBodyLength, d as bytesEqual, f as fromJSONEnvelope, g as writeFrame, h as toJSONEnvelope, i as SerializationError, l as assertMinBodyLength, m as readHeader, n as ChecksumError, o as UnknownHashVariantError, r as ReservedBitsError, s as UnknownVersionError, t as BadMagicError, u as assertParamsPadding } from "../serialize-CMSYFym-.js";
|
|
4
4
|
import { i as assertProbability, o as assertUint32, r as assertPositiveInt, t as ParamError } from "../params-CeajSrwv.js";
|
|
5
|
-
import { t as bloomSizing } from "../sizing-
|
|
5
|
+
import { t as bloomSizing } from "../sizing-D3-_7pdP.js";
|
|
6
6
|
//#region src/scalable/scalable.ts
|
|
7
7
|
const MAX_BITS = 4294967295;
|
|
8
8
|
const TYPE = 6;
|
|
@@ -18,5 +18,17 @@ interface HllSizing {
|
|
|
18
18
|
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
19
19
|
*/
|
|
20
20
|
declare function hllSizing(relativeError: number): HllSizing;
|
|
21
|
+
/** Count-Min geometry: the `CountMinParams` fields a sizing solve determines. */
|
|
22
|
+
interface CountMinSizing {
|
|
23
|
+
/** Counters per row. */
|
|
24
|
+
width: number;
|
|
25
|
+
/** Number of rows, one probe each. */
|
|
26
|
+
depth: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Count-Min sizing: `width` columns and `depth` rows for an estimate at most
|
|
30
|
+
* `epsilon * total` above the true count, with probability `1 - delta`.
|
|
31
|
+
*/
|
|
32
|
+
declare function countMinSizing(epsilon: number, delta: number): CountMinSizing;
|
|
21
33
|
//#endregion
|
|
22
|
-
export {
|
|
34
|
+
export { countMinSizing as a, bloomSizing as i, CountMinSizing as n, hllSizing as o, HllSizing as r, BloomSizing as t };
|
|
@@ -18,5 +18,17 @@ interface HllSizing {
|
|
|
18
18
|
* `relativeError`. Throws {@link ParamError} when even `HLL_MAX_P` cannot.
|
|
19
19
|
*/
|
|
20
20
|
declare function hllSizing(relativeError: number): HllSizing;
|
|
21
|
+
/** Count-Min geometry: the `CountMinParams` fields a sizing solve determines. */
|
|
22
|
+
interface CountMinSizing {
|
|
23
|
+
/** Counters per row. */
|
|
24
|
+
width: number;
|
|
25
|
+
/** Number of rows, one probe each. */
|
|
26
|
+
depth: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Count-Min sizing: `width` columns and `depth` rows for an estimate at most
|
|
30
|
+
* `epsilon * total` above the true count, with probability `1 - delta`.
|
|
31
|
+
*/
|
|
32
|
+
declare function countMinSizing(epsilon: number, delta: number): CountMinSizing;
|
|
21
33
|
//#endregion
|
|
22
|
-
export {
|
|
34
|
+
export { countMinSizing as a, bloomSizing as i, CountMinSizing as n, hllSizing as o, HllSizing as r, BloomSizing as t };
|
|
@@ -13,5 +13,13 @@ function hllSizing(relativeError) {
|
|
|
13
13
|
if (p > 18) throw new ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
14
14
|
return { p };
|
|
15
15
|
}
|
|
16
|
+
function countMinSizing(epsilon, delta) {
|
|
17
|
+
assertProbability(epsilon, "epsilon");
|
|
18
|
+
assertProbability(delta, "delta");
|
|
19
|
+
return {
|
|
20
|
+
width: Math.ceil(Math.E / epsilon),
|
|
21
|
+
depth: Math.ceil(Math.log(1 / delta))
|
|
22
|
+
};
|
|
23
|
+
}
|
|
16
24
|
//#endregion
|
|
17
|
-
export {
|
|
25
|
+
export { countMinSizing as n, hllSizing as r, bloomSizing as t };
|
|
@@ -13,6 +13,14 @@ function hllSizing(relativeError) {
|
|
|
13
13
|
if (p > 18) throw new require_params.ParamError(`relativeError ${String(relativeError)} needs precision ${String(p)}, above the maximum ${String(18)}`);
|
|
14
14
|
return { p };
|
|
15
15
|
}
|
|
16
|
+
function countMinSizing(epsilon, delta) {
|
|
17
|
+
require_params.assertProbability(epsilon, "epsilon");
|
|
18
|
+
require_params.assertProbability(delta, "delta");
|
|
19
|
+
return {
|
|
20
|
+
width: Math.ceil(Math.E / epsilon),
|
|
21
|
+
depth: Math.ceil(Math.log(1 / delta))
|
|
22
|
+
};
|
|
23
|
+
}
|
|
16
24
|
//#endregion
|
|
17
25
|
Object.defineProperty(exports, "bloomSizing", {
|
|
18
26
|
enumerable: true,
|
|
@@ -20,6 +28,12 @@ Object.defineProperty(exports, "bloomSizing", {
|
|
|
20
28
|
return bloomSizing;
|
|
21
29
|
}
|
|
22
30
|
});
|
|
31
|
+
Object.defineProperty(exports, "countMinSizing", {
|
|
32
|
+
enumerable: true,
|
|
33
|
+
get: function() {
|
|
34
|
+
return countMinSizing;
|
|
35
|
+
}
|
|
36
|
+
});
|
|
23
37
|
Object.defineProperty(exports, "hllSizing", {
|
|
24
38
|
enumerable: true,
|
|
25
39
|
get: function() {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "distillate",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Probabilistic data structures for JavaScript. Space-efficient, approximate-membership filters (Bloom, Blocked Bloom, Binary Fuse) with tunable error and a portable binary format; zero dependencies, universal.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bloom-filter",
|
|
@@ -87,6 +87,14 @@
|
|
|
87
87
|
"import": "./dist/cuckoo/index.js",
|
|
88
88
|
"require": "./dist/cuckoo/index.cjs"
|
|
89
89
|
},
|
|
90
|
+
"./countmin": {
|
|
91
|
+
"types": {
|
|
92
|
+
"import": "./dist/countmin/index.d.ts",
|
|
93
|
+
"require": "./dist/countmin/index.d.cts"
|
|
94
|
+
},
|
|
95
|
+
"import": "./dist/countmin/index.js",
|
|
96
|
+
"require": "./dist/countmin/index.cjs"
|
|
97
|
+
},
|
|
90
98
|
"./package.json": "./package.json"
|
|
91
99
|
},
|
|
92
100
|
"main": "./dist/index.cjs",
|