distillate 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +105 -0
- package/dist/blocked/index.cjs +116 -0
- package/dist/blocked/index.d.cts +29 -0
- package/dist/blocked/index.d.ts +29 -0
- package/dist/blocked/index.js +114 -0
- package/dist/bloom/index.cjs +132 -0
- package/dist/bloom/index.d.cts +24 -0
- package/dist/bloom/index.d.ts +24 -0
- package/dist/bloom/index.js +130 -0
- package/dist/bytes-DCuYtUVS.d.cts +4 -0
- package/dist/bytes-DCuYtUVS.d.ts +4 -0
- package/dist/fuse/index.cjs +298 -0
- package/dist/fuse/index.d.cts +35 -0
- package/dist/fuse/index.d.ts +35 -0
- package/dist/fuse/index.js +295 -0
- package/dist/index.cjs +5 -0
- package/dist/index.d.cts +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/serialize-C5rwVPvv.cjs +312 -0
- package/dist/serialize-E_4WyueA.js +271 -0
- package/package.json +114 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Akshay
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# distillate
|
|
2
|
+
|
|
3
|
+
[](https://github.com/akshay-xp/distillate/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/distillate)
|
|
5
|
+
[](./LICENSE)
|
|
6
|
+
|
|
7
|
+
Probabilistic data structures for JavaScript: space-efficient, approximate answers with tunable error and zero false negatives. TypeScript-first, zero dependencies, and the right structure per workload. It opens with a family of membership filters (the next-generation successor to Bloom filter packages) and is built to grow into other sketches.
|
|
8
|
+
|
|
9
|
+
> **Pre-release (0.x).** The published structures are correct, tested, and benchmarked, but the public API may still change before `1.0`, and more structures (Cuckoo, Scalable Bloom) are on the way. Pin a version if you depend on it.
|
|
10
|
+
|
|
11
|
+
## Why
|
|
12
|
+
|
|
13
|
+
An approximate-membership query (AMQ) filter answers "is this in the set?" with a tunable false-positive rate and **zero false negatives**, in a fraction of the space of storing the set itself.
|
|
14
|
+
|
|
15
|
+
- **Runs anywhere**: Node, Bun, Deno, browsers, and Cloudflare/Vercel edge. No `eval`, no required WASM compile.
|
|
16
|
+
- **Correct**: no false negatives (property-tested); false-positive rates validated against theory.
|
|
17
|
+
- **Small**: per-structure subpath imports, `sideEffects: false`, zero runtime dependencies.
|
|
18
|
+
- **Portable**: a versioned little-endian binary format (`toBytes` / `fromBytes`) for persistence and cross-language reads.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
npm install distillate
|
|
24
|
+
# or: pnpm add distillate / bun add distillate / deno add npm:distillate
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Requires Node 20+ (or any modern Bun/Deno/browser/edge runtime).
|
|
28
|
+
|
|
29
|
+
## Runtime support
|
|
30
|
+
|
|
31
|
+
`distillate` targets ES2022 with zero runtime dependencies and no `eval`, so it runs on every modern JavaScript runtime:
|
|
32
|
+
|
|
33
|
+
- **Node.js** 20, 22, 24 (LTS and current)
|
|
34
|
+
- **Bun** and **Deno**
|
|
35
|
+
- Browsers and Cloudflare/Vercel edge
|
|
36
|
+
|
|
37
|
+
Every push runs a CI smoke matrix that imports the built package on Node 20/22/24, Bun, and Deno, so cross-runtime support is verified, not assumed.
|
|
38
|
+
|
|
39
|
+
## Structures
|
|
40
|
+
|
|
41
|
+
Each structure ships as its own subpath, so you only bundle what you import.
|
|
42
|
+
|
|
43
|
+
| Import | Structure | Mutable? | Use for |
|
|
44
|
+
| -------------------- | ------------- | -------- | ---------------------------------------------------- |
|
|
45
|
+
| `distillate/bloom` | Classic Bloom | yes | Familiar default, migration from `bloom-filters` |
|
|
46
|
+
| `distillate/blocked` | Blocked Bloom | yes | Streaming inserts, speed-first, cache-friendly |
|
|
47
|
+
| `distillate/fuse` | Binary Fuse | no | Static set built once and queried a lot; least space |
|
|
48
|
+
|
|
49
|
+
### Classic Bloom (`distillate/bloom`)
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { BloomFilter } from "distillate/bloom";
|
|
53
|
+
|
|
54
|
+
const filter = BloomFilter.create(100_000, 0.01); // capacity, target FPR
|
|
55
|
+
filter.add("alice");
|
|
56
|
+
filter.has("alice"); // true
|
|
57
|
+
filter.has("bob"); // false (or a ~1% false positive)
|
|
58
|
+
|
|
59
|
+
const bytes = filter.toBytes();
|
|
60
|
+
const restored = BloomFilter.fromBytes(bytes);
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Also: `union(other)` (merge equal-parameter filters), `bitsPerKey`, and a low-level `new BloomFilter({ m, k, seed })`.
|
|
64
|
+
|
|
65
|
+
### Blocked Bloom (`distillate/blocked`)
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
import { BlockedBloomFilter } from "distillate/blocked";
|
|
69
|
+
|
|
70
|
+
const filter = BlockedBloomFilter.create(100_000, 0.01);
|
|
71
|
+
filter.add("alice");
|
|
72
|
+
filter.has("alice"); // true
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Same surface as Classic Bloom (`add` / `has` / `union` / `toBytes` / `fromBytes` / `bitsPerKey`). Confines every lookup to one cache line, trading ~20-30% more space for cache-friendly throughput.
|
|
76
|
+
|
|
77
|
+
### Binary Fuse (`distillate/fuse`)
|
|
78
|
+
|
|
79
|
+
A **static** filter: built once from the full key set, then immutable. The most space-efficient option (~9 bits/key at ~0.39% FPR for 8-bit; ~19 bits/key at ~1/65536 for 16-bit).
|
|
80
|
+
|
|
81
|
+
```ts
|
|
82
|
+
import { BinaryFuse8, BinaryFuse16 } from "distillate/fuse";
|
|
83
|
+
|
|
84
|
+
const filter = BinaryFuse8.from(["alice", "bob", "carol"]);
|
|
85
|
+
filter.has("alice"); // true
|
|
86
|
+
filter.size; // 3
|
|
87
|
+
filter.bitsPerKey; // ~9
|
|
88
|
+
|
|
89
|
+
// Lower false-positive rate, twice the space:
|
|
90
|
+
const precise = BinaryFuse16.from(["alice", "bob", "carol"]);
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Also: `toBytes` / `fromBytes`. No `add` / `delete`; rebuild `from` the new set to change membership.
|
|
94
|
+
|
|
95
|
+
## Docs
|
|
96
|
+
|
|
97
|
+
Design notes, the structure decision matrix, hashing, and the binary format live in [`docs/`](./docs):
|
|
98
|
+
|
|
99
|
+
- [overview](./docs/overview.md): what and why
|
|
100
|
+
- [structures](./docs/structures.md): decision matrix and the full lineup
|
|
101
|
+
- [architecture](./docs/architecture.md), [hashing](./docs/hashing.md), [serialization](./docs/serialization.md)
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
[MIT](./LICENSE) © Akshay
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_serialize = require("../serialize-C5rwVPvv.cjs");
|
|
3
|
+
//#region src/blocked/blocked.ts
|
|
4
|
+
const TYPE = 2;
|
|
5
|
+
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
6
|
+
/**
|
|
7
|
+
* Split-block probe: derive one block index and 8 single-bit lane masks for
|
|
8
|
+
* `key`. Writes lane word indices into `outWords` and their masks into
|
|
9
|
+
* `outBits` (both length 8, caller-owned so no per-call allocation). A block is
|
|
10
|
+
* 256 bits = 8 contiguous 32-bit lanes; word `block*8 + i` gets one bit set.
|
|
11
|
+
*/
|
|
12
|
+
var BlockedBloomParamMismatchError = class extends Error {
|
|
13
|
+
name = "BlockedBloomParamMismatchError";
|
|
14
|
+
};
|
|
15
|
+
var BlockedBloomFilter = class BlockedBloomFilter {
|
|
16
|
+
#lanes;
|
|
17
|
+
#numBlocks;
|
|
18
|
+
#seed;
|
|
19
|
+
#n;
|
|
20
|
+
#words = /* @__PURE__ */ new Uint32Array(8);
|
|
21
|
+
#bits = /* @__PURE__ */ new Uint32Array(8);
|
|
22
|
+
static #ANCHORS = [
|
|
23
|
+
[2, 10.5],
|
|
24
|
+
[3, 16.9],
|
|
25
|
+
[4, 26.4]
|
|
26
|
+
];
|
|
27
|
+
static create(n, epsilon) {
|
|
28
|
+
const t = Math.log10(1 / epsilon);
|
|
29
|
+
const a = BlockedBloomFilter.#ANCHORS;
|
|
30
|
+
let seg = a.findIndex((p) => t <= p[0]);
|
|
31
|
+
if (seg < 1) seg = seg === -1 ? a.length - 1 : 1;
|
|
32
|
+
const [t0, b0] = a[seg - 1] ?? [0, 0];
|
|
33
|
+
const [t1, b1] = a[seg] ?? [0, 0];
|
|
34
|
+
const bitsPerKey = b0 + (b1 - b0) / (t1 - t0) * (t - t0);
|
|
35
|
+
return new BlockedBloomFilter({
|
|
36
|
+
bitsPerKey: Math.ceil(bitsPerKey),
|
|
37
|
+
capacity: n
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
41
|
+
this.#numBlocks = Math.max(1, Math.ceil(bitsPerKey * capacity / 256));
|
|
42
|
+
this.#lanes = new Uint32Array(this.#numBlocks * 8);
|
|
43
|
+
this.#seed = seed;
|
|
44
|
+
this.#n = capacity;
|
|
45
|
+
}
|
|
46
|
+
get bitsPerKey() {
|
|
47
|
+
return this.#numBlocks * 256 / this.#n;
|
|
48
|
+
}
|
|
49
|
+
static fromBytes(bytes) {
|
|
50
|
+
const { body } = require_serialize.readHeader(bytes);
|
|
51
|
+
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
52
|
+
const numBlocks = dv.getUint32(0, true);
|
|
53
|
+
const seed = dv.getUint32(4, true);
|
|
54
|
+
const n = dv.getUint32(8, true);
|
|
55
|
+
const f = new BlockedBloomFilter({
|
|
56
|
+
bitsPerKey: 256,
|
|
57
|
+
capacity: numBlocks,
|
|
58
|
+
seed
|
|
59
|
+
});
|
|
60
|
+
f.#n = n;
|
|
61
|
+
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
62
|
+
return f;
|
|
63
|
+
}
|
|
64
|
+
toBytes() {
|
|
65
|
+
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
66
|
+
const body = new Uint8Array(12 + lanes.length);
|
|
67
|
+
const dv = new DataView(body.buffer);
|
|
68
|
+
dv.setUint32(0, this.#numBlocks, true);
|
|
69
|
+
dv.setUint32(4, this.#seed, true);
|
|
70
|
+
dv.setUint32(8, this.#n, true);
|
|
71
|
+
body.set(lanes, 12);
|
|
72
|
+
return require_serialize.writeHeader({
|
|
73
|
+
version: 1,
|
|
74
|
+
type: TYPE,
|
|
75
|
+
flags: 0
|
|
76
|
+
}, body);
|
|
77
|
+
}
|
|
78
|
+
union(other) {
|
|
79
|
+
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
80
|
+
const r = new BlockedBloomFilter({
|
|
81
|
+
bitsPerKey: this.bitsPerKey,
|
|
82
|
+
capacity: this.#n,
|
|
83
|
+
seed: this.#seed
|
|
84
|
+
});
|
|
85
|
+
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
86
|
+
return r;
|
|
87
|
+
}
|
|
88
|
+
add(key) {
|
|
89
|
+
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
90
|
+
for (let i = 0; i < 8; i++) {
|
|
91
|
+
const w = this.#words[i] ?? 0;
|
|
92
|
+
this.#lanes[w] = (this.#lanes[w] ?? 0) | (this.#bits[i] ?? 0);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
has(key) {
|
|
96
|
+
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
97
|
+
for (let i = 0; i < 8; i++) {
|
|
98
|
+
const bit = this.#bits[i] ?? 0;
|
|
99
|
+
if (((this.#lanes[this.#words[i] ?? 0] ?? 0) & bit) >>> 0 !== bit) return false;
|
|
100
|
+
}
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
105
|
+
const { h1lo, h1hi, h2lo, h2hi } = require_serialize.hash128(require_serialize.normalize(key), seed);
|
|
106
|
+
const block = require_serialize.reduce((h1lo ^ h1hi) >>> 0, numBlocks);
|
|
107
|
+
const x = (h2lo ^ h2hi) >>> 0;
|
|
108
|
+
const base = block * 8;
|
|
109
|
+
for (let i = 0; i < 8; i++) {
|
|
110
|
+
outWords[i] = base + i;
|
|
111
|
+
outBits[i] = 1 << (Math.imul(x, SALT[i] ?? 0) >>> 27) >>> 0;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
//#endregion
|
|
115
|
+
exports.BlockedBloomFilter = BlockedBloomFilter;
|
|
116
|
+
exports.BlockedBloomParamMismatchError = BlockedBloomParamMismatchError;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
//#region src/blocked/blocked.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* Split-block probe: derive one block index and 8 single-bit lane masks for
|
|
5
|
+
* `key`. Writes lane word indices into `outWords` and their masks into
|
|
6
|
+
* `outBits` (both length 8, caller-owned so no per-call allocation). A block is
|
|
7
|
+
* 256 bits = 8 contiguous 32-bit lanes; word `block*8 + i` gets one bit set.
|
|
8
|
+
*/
|
|
9
|
+
declare class BlockedBloomParamMismatchError extends Error {
|
|
10
|
+
override readonly name = "BlockedBloomParamMismatchError";
|
|
11
|
+
}
|
|
12
|
+
interface BlockedBloomParams {
|
|
13
|
+
bitsPerKey: number;
|
|
14
|
+
capacity: number;
|
|
15
|
+
seed?: number;
|
|
16
|
+
}
|
|
17
|
+
declare class BlockedBloomFilter {
|
|
18
|
+
#private;
|
|
19
|
+
static create(n: number, epsilon: number): BlockedBloomFilter;
|
|
20
|
+
constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
|
|
21
|
+
get bitsPerKey(): number;
|
|
22
|
+
static fromBytes(bytes: Uint8Array): BlockedBloomFilter;
|
|
23
|
+
toBytes(): Uint8Array;
|
|
24
|
+
union(other: BlockedBloomFilter): BlockedBloomFilter;
|
|
25
|
+
add(key: BytesLike): void;
|
|
26
|
+
has(key: BytesLike): boolean;
|
|
27
|
+
}
|
|
28
|
+
//#endregion
|
|
29
|
+
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
//#region src/blocked/blocked.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* Split-block probe: derive one block index and 8 single-bit lane masks for
|
|
5
|
+
* `key`. Writes lane word indices into `outWords` and their masks into
|
|
6
|
+
* `outBits` (both length 8, caller-owned so no per-call allocation). A block is
|
|
7
|
+
* 256 bits = 8 contiguous 32-bit lanes; word `block*8 + i` gets one bit set.
|
|
8
|
+
*/
|
|
9
|
+
declare class BlockedBloomParamMismatchError extends Error {
|
|
10
|
+
override readonly name = "BlockedBloomParamMismatchError";
|
|
11
|
+
}
|
|
12
|
+
interface BlockedBloomParams {
|
|
13
|
+
bitsPerKey: number;
|
|
14
|
+
capacity: number;
|
|
15
|
+
seed?: number;
|
|
16
|
+
}
|
|
17
|
+
declare class BlockedBloomFilter {
|
|
18
|
+
#private;
|
|
19
|
+
static create(n: number, epsilon: number): BlockedBloomFilter;
|
|
20
|
+
constructor({ bitsPerKey, capacity, seed }: BlockedBloomParams);
|
|
21
|
+
get bitsPerKey(): number;
|
|
22
|
+
static fromBytes(bytes: Uint8Array): BlockedBloomFilter;
|
|
23
|
+
toBytes(): Uint8Array;
|
|
24
|
+
union(other: BlockedBloomFilter): BlockedBloomFilter;
|
|
25
|
+
add(key: BytesLike): void;
|
|
26
|
+
has(key: BytesLike): boolean;
|
|
27
|
+
}
|
|
28
|
+
//#endregion
|
|
29
|
+
export { BlockedBloomFilter, BlockedBloomParamMismatchError, type BlockedBloomParams };
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { i as hash128, n as readHeader, o as reduce, r as writeHeader, s as normalize } from "../serialize-E_4WyueA.js";
|
|
2
|
+
//#region src/blocked/blocked.ts
|
|
3
|
+
const TYPE = 2;
|
|
4
|
+
const SALT = Uint32Array.of(1203114875, 1150766481, 2284105051, 2729912477, 1884591559, 770785867, 2667333959, 1550580529);
|
|
5
|
+
/**
|
|
6
|
+
* Split-block probe: derive one block index and 8 single-bit lane masks for
|
|
7
|
+
* `key`. Writes lane word indices into `outWords` and their masks into
|
|
8
|
+
* `outBits` (both length 8, caller-owned so no per-call allocation). A block is
|
|
9
|
+
* 256 bits = 8 contiguous 32-bit lanes; word `block*8 + i` gets one bit set.
|
|
10
|
+
*/
|
|
11
|
+
var BlockedBloomParamMismatchError = class extends Error {
|
|
12
|
+
name = "BlockedBloomParamMismatchError";
|
|
13
|
+
};
|
|
14
|
+
var BlockedBloomFilter = class BlockedBloomFilter {
|
|
15
|
+
#lanes;
|
|
16
|
+
#numBlocks;
|
|
17
|
+
#seed;
|
|
18
|
+
#n;
|
|
19
|
+
#words = /* @__PURE__ */ new Uint32Array(8);
|
|
20
|
+
#bits = /* @__PURE__ */ new Uint32Array(8);
|
|
21
|
+
static #ANCHORS = [
|
|
22
|
+
[2, 10.5],
|
|
23
|
+
[3, 16.9],
|
|
24
|
+
[4, 26.4]
|
|
25
|
+
];
|
|
26
|
+
static create(n, epsilon) {
|
|
27
|
+
const t = Math.log10(1 / epsilon);
|
|
28
|
+
const a = BlockedBloomFilter.#ANCHORS;
|
|
29
|
+
let seg = a.findIndex((p) => t <= p[0]);
|
|
30
|
+
if (seg < 1) seg = seg === -1 ? a.length - 1 : 1;
|
|
31
|
+
const [t0, b0] = a[seg - 1] ?? [0, 0];
|
|
32
|
+
const [t1, b1] = a[seg] ?? [0, 0];
|
|
33
|
+
const bitsPerKey = b0 + (b1 - b0) / (t1 - t0) * (t - t0);
|
|
34
|
+
return new BlockedBloomFilter({
|
|
35
|
+
bitsPerKey: Math.ceil(bitsPerKey),
|
|
36
|
+
capacity: n
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
constructor({ bitsPerKey, capacity, seed = 0 }) {
|
|
40
|
+
this.#numBlocks = Math.max(1, Math.ceil(bitsPerKey * capacity / 256));
|
|
41
|
+
this.#lanes = new Uint32Array(this.#numBlocks * 8);
|
|
42
|
+
this.#seed = seed;
|
|
43
|
+
this.#n = capacity;
|
|
44
|
+
}
|
|
45
|
+
get bitsPerKey() {
|
|
46
|
+
return this.#numBlocks * 256 / this.#n;
|
|
47
|
+
}
|
|
48
|
+
static fromBytes(bytes) {
|
|
49
|
+
const { body } = readHeader(bytes);
|
|
50
|
+
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
51
|
+
const numBlocks = dv.getUint32(0, true);
|
|
52
|
+
const seed = dv.getUint32(4, true);
|
|
53
|
+
const n = dv.getUint32(8, true);
|
|
54
|
+
const f = new BlockedBloomFilter({
|
|
55
|
+
bitsPerKey: 256,
|
|
56
|
+
capacity: numBlocks,
|
|
57
|
+
seed
|
|
58
|
+
});
|
|
59
|
+
f.#n = n;
|
|
60
|
+
new Uint8Array(f.#lanes.buffer).set(body.subarray(12));
|
|
61
|
+
return f;
|
|
62
|
+
}
|
|
63
|
+
toBytes() {
|
|
64
|
+
const lanes = new Uint8Array(this.#lanes.buffer, this.#lanes.byteOffset, this.#lanes.byteLength);
|
|
65
|
+
const body = new Uint8Array(12 + lanes.length);
|
|
66
|
+
const dv = new DataView(body.buffer);
|
|
67
|
+
dv.setUint32(0, this.#numBlocks, true);
|
|
68
|
+
dv.setUint32(4, this.#seed, true);
|
|
69
|
+
dv.setUint32(8, this.#n, true);
|
|
70
|
+
body.set(lanes, 12);
|
|
71
|
+
return writeHeader({
|
|
72
|
+
version: 1,
|
|
73
|
+
type: TYPE,
|
|
74
|
+
flags: 0
|
|
75
|
+
}, body);
|
|
76
|
+
}
|
|
77
|
+
union(other) {
|
|
78
|
+
if (this.#numBlocks !== other.#numBlocks || this.#seed !== other.#seed) throw new BlockedBloomParamMismatchError("cannot union blocked Bloom filters whose parameters do not match");
|
|
79
|
+
const r = new BlockedBloomFilter({
|
|
80
|
+
bitsPerKey: this.bitsPerKey,
|
|
81
|
+
capacity: this.#n,
|
|
82
|
+
seed: this.#seed
|
|
83
|
+
});
|
|
84
|
+
for (let i = 0; i < this.#lanes.length; i++) r.#lanes[i] = (this.#lanes[i] ?? 0) | (other.#lanes[i] ?? 0);
|
|
85
|
+
return r;
|
|
86
|
+
}
|
|
87
|
+
add(key) {
|
|
88
|
+
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
89
|
+
for (let i = 0; i < 8; i++) {
|
|
90
|
+
const w = this.#words[i] ?? 0;
|
|
91
|
+
this.#lanes[w] = (this.#lanes[w] ?? 0) | (this.#bits[i] ?? 0);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
has(key) {
|
|
95
|
+
fillBlock(key, this.#numBlocks, this.#seed, this.#words, this.#bits);
|
|
96
|
+
for (let i = 0; i < 8; i++) {
|
|
97
|
+
const bit = this.#bits[i] ?? 0;
|
|
98
|
+
if (((this.#lanes[this.#words[i] ?? 0] ?? 0) & bit) >>> 0 !== bit) return false;
|
|
99
|
+
}
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
function fillBlock(key, numBlocks, seed, outWords, outBits) {
|
|
104
|
+
const { h1lo, h1hi, h2lo, h2hi } = hash128(normalize(key), seed);
|
|
105
|
+
const block = reduce((h1lo ^ h1hi) >>> 0, numBlocks);
|
|
106
|
+
const x = (h2lo ^ h2hi) >>> 0;
|
|
107
|
+
const base = block * 8;
|
|
108
|
+
for (let i = 0; i < 8; i++) {
|
|
109
|
+
outWords[i] = base + i;
|
|
110
|
+
outBits[i] = 1 << (Math.imul(x, SALT[i] ?? 0) >>> 27) >>> 0;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
//#endregion
|
|
114
|
+
export { BlockedBloomFilter, BlockedBloomParamMismatchError };
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_serialize = require("../serialize-C5rwVPvv.cjs");
|
|
3
|
+
//#region src/core/bitset.ts
|
|
4
|
+
var BitSetRangeError = class extends RangeError {
|
|
5
|
+
name = "BitSetRangeError";
|
|
6
|
+
};
|
|
7
|
+
const MAX_BITS = 2 ** 32;
|
|
8
|
+
var BitSet = class BitSet {
|
|
9
|
+
#bits;
|
|
10
|
+
static fromBytes(bytes) {
|
|
11
|
+
const bs = new BitSet(bytes.length * 8);
|
|
12
|
+
bs.#bits.set(bytes);
|
|
13
|
+
return bs;
|
|
14
|
+
}
|
|
15
|
+
constructor(nbits) {
|
|
16
|
+
if (nbits > MAX_BITS) throw new BitSetRangeError(`BitSet capacity ${String(nbits)} exceeds the 2^32-bit limit`);
|
|
17
|
+
this.#bits = new Uint8Array(Math.ceil(nbits / 8));
|
|
18
|
+
}
|
|
19
|
+
set(i) {
|
|
20
|
+
const idx = i >>> 3;
|
|
21
|
+
this.#bits[idx] = (this.#bits[idx] ?? 0) | 1 << (i & 7);
|
|
22
|
+
}
|
|
23
|
+
get(i) {
|
|
24
|
+
return ((this.#bits[i >>> 3] ?? 0) & 1 << (i & 7)) !== 0;
|
|
25
|
+
}
|
|
26
|
+
get bytes() {
|
|
27
|
+
return this.#bits;
|
|
28
|
+
}
|
|
29
|
+
count() {
|
|
30
|
+
let total = 0;
|
|
31
|
+
for (let byte of this.#bits) while (byte) {
|
|
32
|
+
byte &= byte - 1;
|
|
33
|
+
total++;
|
|
34
|
+
}
|
|
35
|
+
return total;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
//#endregion
|
|
39
|
+
//#region src/core/sizing.ts
|
|
40
|
+
/** Optimal Bloom-filter sizing: `m` bits and `k` hashes for `n` items at target FPR `epsilon`. */
|
|
41
|
+
function optimal(n, epsilon) {
|
|
42
|
+
const m = Math.ceil(-n * Math.log(epsilon) / (Math.LN2 * Math.LN2));
|
|
43
|
+
return {
|
|
44
|
+
m,
|
|
45
|
+
k: Math.max(1, Math.round(m / n * Math.LN2))
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
//#endregion
|
|
49
|
+
//#region src/bloom/bloom.ts
|
|
50
|
+
const TYPE = 1;
|
|
51
|
+
var BloomParamMismatchError = class extends Error {
|
|
52
|
+
name = "BloomParamMismatchError";
|
|
53
|
+
};
|
|
54
|
+
var BloomFilter = class BloomFilter {
|
|
55
|
+
#bits;
|
|
56
|
+
#m;
|
|
57
|
+
#k;
|
|
58
|
+
#seed;
|
|
59
|
+
#scratch;
|
|
60
|
+
#n;
|
|
61
|
+
static create(n, epsilon) {
|
|
62
|
+
const f = new BloomFilter(optimal(n, epsilon));
|
|
63
|
+
f.#n = n;
|
|
64
|
+
return f;
|
|
65
|
+
}
|
|
66
|
+
static fromBytes(bytes) {
|
|
67
|
+
const { body } = require_serialize.readHeader(bytes);
|
|
68
|
+
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
69
|
+
const m = dv.getUint32(0, true);
|
|
70
|
+
const k = body[4] ?? 0;
|
|
71
|
+
const seed = dv.getUint32(5, true);
|
|
72
|
+
const f = new BloomFilter({
|
|
73
|
+
m,
|
|
74
|
+
k,
|
|
75
|
+
seed
|
|
76
|
+
});
|
|
77
|
+
f.#bits.bytes.set(body.subarray(9));
|
|
78
|
+
return f;
|
|
79
|
+
}
|
|
80
|
+
constructor({ m, k, seed = 0 }) {
|
|
81
|
+
this.#bits = new BitSet(m);
|
|
82
|
+
this.#m = m;
|
|
83
|
+
this.#k = k;
|
|
84
|
+
this.#seed = seed;
|
|
85
|
+
this.#scratch = new Uint32Array(k);
|
|
86
|
+
this.#n = Math.round(m * Math.LN2 / k);
|
|
87
|
+
}
|
|
88
|
+
/** Analytic design bits-per-key `m / n`. */
|
|
89
|
+
get bitsPerKey() {
|
|
90
|
+
return this.#m / this.#n;
|
|
91
|
+
}
|
|
92
|
+
toBytes() {
|
|
93
|
+
const payload = this.#bits.bytes;
|
|
94
|
+
const body = new Uint8Array(9 + payload.length);
|
|
95
|
+
const dv = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
96
|
+
dv.setUint32(0, this.#m, true);
|
|
97
|
+
body[4] = this.#k;
|
|
98
|
+
dv.setUint32(5, this.#seed, true);
|
|
99
|
+
body.set(payload, 9);
|
|
100
|
+
return require_serialize.writeHeader({
|
|
101
|
+
version: 1,
|
|
102
|
+
type: TYPE,
|
|
103
|
+
flags: 0
|
|
104
|
+
}, body);
|
|
105
|
+
}
|
|
106
|
+
union(other) {
|
|
107
|
+
if (this.#m !== other.#m || this.#k !== other.#k || this.#seed !== other.#seed) throw new BloomParamMismatchError("cannot union Bloom filters whose parameters do not match");
|
|
108
|
+
const a = this.#bits.bytes;
|
|
109
|
+
const b = other.#bits.bytes;
|
|
110
|
+
const merged = new Uint8Array(a.length);
|
|
111
|
+
for (let i = 0; i < a.length; i++) merged[i] = (a[i] ?? 0) | (b[i] ?? 0);
|
|
112
|
+
const r = new BloomFilter({
|
|
113
|
+
m: this.#m,
|
|
114
|
+
k: this.#k,
|
|
115
|
+
seed: this.#seed
|
|
116
|
+
});
|
|
117
|
+
r.#bits.bytes.set(merged);
|
|
118
|
+
return r;
|
|
119
|
+
}
|
|
120
|
+
add(key) {
|
|
121
|
+
require_serialize.probeInto(key, this.#k, this.#m, this.#seed, this.#scratch);
|
|
122
|
+
for (let i = 0; i < this.#k; i++) this.#bits.set(this.#scratch[i] ?? 0);
|
|
123
|
+
}
|
|
124
|
+
has(key) {
|
|
125
|
+
require_serialize.probeInto(key, this.#k, this.#m, this.#seed, this.#scratch);
|
|
126
|
+
for (let i = 0; i < this.#k; i++) if (!this.#bits.get(this.#scratch[i] ?? 0)) return false;
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
//#endregion
|
|
131
|
+
exports.BloomFilter = BloomFilter;
|
|
132
|
+
exports.BloomParamMismatchError = BloomParamMismatchError;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.cjs";
|
|
2
|
+
//#region src/bloom/bloom.d.ts
|
|
3
|
+
declare class BloomParamMismatchError extends Error {
|
|
4
|
+
override readonly name = "BloomParamMismatchError";
|
|
5
|
+
}
|
|
6
|
+
interface BloomParams {
|
|
7
|
+
m: number;
|
|
8
|
+
k: number;
|
|
9
|
+
seed?: number;
|
|
10
|
+
}
|
|
11
|
+
declare class BloomFilter {
|
|
12
|
+
#private;
|
|
13
|
+
static create(n: number, epsilon: number): BloomFilter;
|
|
14
|
+
static fromBytes(bytes: Uint8Array): BloomFilter;
|
|
15
|
+
constructor({ m, k, seed }: BloomParams);
|
|
16
|
+
/** Analytic design bits-per-key `m / n`. */
|
|
17
|
+
get bitsPerKey(): number;
|
|
18
|
+
toBytes(): Uint8Array;
|
|
19
|
+
union(other: BloomFilter): BloomFilter;
|
|
20
|
+
add(key: BytesLike): void;
|
|
21
|
+
has(key: BytesLike): boolean;
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
export { BloomFilter, BloomParamMismatchError, type BloomParams };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { t as BytesLike } from "../bytes-DCuYtUVS.js";
|
|
2
|
+
//#region src/bloom/bloom.d.ts
|
|
3
|
+
declare class BloomParamMismatchError extends Error {
|
|
4
|
+
override readonly name = "BloomParamMismatchError";
|
|
5
|
+
}
|
|
6
|
+
interface BloomParams {
|
|
7
|
+
m: number;
|
|
8
|
+
k: number;
|
|
9
|
+
seed?: number;
|
|
10
|
+
}
|
|
11
|
+
declare class BloomFilter {
|
|
12
|
+
#private;
|
|
13
|
+
static create(n: number, epsilon: number): BloomFilter;
|
|
14
|
+
static fromBytes(bytes: Uint8Array): BloomFilter;
|
|
15
|
+
constructor({ m, k, seed }: BloomParams);
|
|
16
|
+
/** Analytic design bits-per-key `m / n`. */
|
|
17
|
+
get bitsPerKey(): number;
|
|
18
|
+
toBytes(): Uint8Array;
|
|
19
|
+
union(other: BloomFilter): BloomFilter;
|
|
20
|
+
add(key: BytesLike): void;
|
|
21
|
+
has(key: BytesLike): boolean;
|
|
22
|
+
}
|
|
23
|
+
//#endregion
|
|
24
|
+
export { BloomFilter, BloomParamMismatchError, type BloomParams };
|