@cloudbitmaps/core 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -36
- package/dist/{chunk-BUPDGZRR.js → chunk-5OMC5U7U.js} +87 -40
- package/dist/chunk-5OMC5U7U.js.map +7 -0
- package/dist/core/audit.d.ts +15 -14
- package/dist/core/budget.d.ts +5 -3
- package/dist/core/codec.d.ts +18 -10
- package/dist/core/consistency.d.ts +3 -3
- package/dist/core/cooperative.d.ts +9 -9
- package/dist/core/cost.d.ts +167 -15
- package/dist/core/crbm/format.d.ts +7 -9
- package/dist/core/crbm/reader.d.ts +18 -1
- package/dist/core/crbm/writer.d.ts +2 -2
- package/dist/core/crbm-storage-source.d.ts +150 -60
- package/dist/core/determinism.d.ts +1 -1
- package/dist/core/discover.d.ts +7 -5
- package/dist/core/due-index.d.ts +3 -5
- package/dist/core/engine.d.ts +55 -13
- package/dist/core/erase-id.d.ts +76 -41
- package/dist/core/erasure.d.ts +22 -16
- package/dist/core/errors.d.ts +13 -20
- package/dist/core/generation-gc.d.ts +9 -5
- package/dist/core/keys.d.ts +2 -2
- package/dist/core/load.d.ts +28 -9
- package/dist/core/metrics.d.ts +8 -4
- package/dist/core/name-codec.d.ts +21 -31
- package/dist/core/pinned-storage-source.d.ts +29 -8
- package/dist/core/ports.d.ts +142 -59
- package/dist/core/reader-defaults.d.ts +18 -0
- package/dist/core/registry-scan.d.ts +15 -6
- package/dist/core/retention-sweep.d.ts +26 -18
- package/dist/core/retry.d.ts +8 -4
- package/dist/core/rollback.d.ts +11 -7
- package/dist/core/validate.d.ts +27 -3
- package/dist/driver-kit.d.ts +2 -3
- package/dist/driver-kit.js +13 -52
- package/dist/driver-kit.js.map +3 -3
- package/dist/drivers/_shared/keys.d.ts +4 -5
- package/dist/drivers/_shared/object-registry.d.ts +10 -5
- package/dist/drivers/_shared/registry.d.ts +9 -5
- package/dist/drivers/backends.d.ts +10 -16
- package/dist/drivers/localfs/fs-util.d.ts +3 -2
- package/dist/drivers/localfs/registry.d.ts +4 -4
- package/dist/drivers/memory.d.ts +2 -10
- package/dist/drivers/retry/retrying-chunk-source.d.ts +59 -0
- package/dist/export/index.d.ts +7 -6
- package/dist/index.d.ts +15 -24
- package/dist/index.js +1014 -373
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +1 -1
- package/package.json +1 -1
- package/dist/chunk-BUPDGZRR.js.map +0 -7
- package/dist/drivers/_shared/aws-errors.d.ts +0 -18
- package/dist/drivers/retry/retrying-drivers.d.ts +0 -90
package/dist/core/codec.d.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* `core/` is **codec-agnostic**: `SegmentEngine`, the erasure rewrite, and the `.crbm` read/write helpers only ever
|
|
5
5
|
* construct and combine bitmaps through the {@link CodecInterface} factory + the {@link CodecBitmap} value type
|
|
6
|
-
* defined here — never a concrete implementation. The flagship codec is roaring (
|
|
6
|
+
* defined here — never a concrete implementation. The flagship codec is roaring (in
|
|
7
7
|
* `@cloudbitmaps/roaring`), and another codec would plug in behind the same seam with zero engine or driver
|
|
8
8
|
* changes. That is the seam's value even with one codec shipped: a plain-bitset flavor was measured and
|
|
9
9
|
* **decided against** (above ~6% density a Roaring chunk already *is* an uncompressed bitset), so the seam is
|
|
@@ -21,15 +21,14 @@
|
|
|
21
21
|
* package's own extended value type, not here — the engine never calls them). The one codec-linked concern
|
|
22
22
|
* this seam intentionally leaves in the format layer is the `.crbm` **payload codec id**
|
|
23
23
|
* (`PAYLOAD_CODEC_ROARING_PORTABLE` / `KNOWN_PAYLOAD_CODEC_IDS` in `crbm/format.ts`): it is stamped in the footer
|
|
24
|
-
* and validated on read
|
|
25
|
-
*
|
|
26
|
-
* format
|
|
27
|
-
* registration rather than a major format version. Still deliberately outside this interface: a codec declares
|
|
24
|
+
* and validated on read, as membership of a registry rather than equality with one constant, so a codec-specific
|
|
25
|
+
* name is not frozen into the format at `1.0` and a future codec is a one-line registration rather than a major
|
|
26
|
+
* format version. Still deliberately outside this interface: a codec declares
|
|
28
27
|
* its bytes, the seam does not.
|
|
29
28
|
*/
|
|
30
29
|
/**
|
|
31
30
|
* The value type a codec produces — a mutable set of `u32` with set algebra and portable (de)serialization.
|
|
32
|
-
* This is the shape
|
|
31
|
+
* This is the shape the roaring flavor's bitmap already has; the engine holds these, caches them, and merges tiers with them.
|
|
33
32
|
*/
|
|
34
33
|
export interface CodecBitmap {
|
|
35
34
|
/** Serialize with the codec's **stable, portable** format (never a frozen/unsafe variant). */
|
|
@@ -61,7 +60,10 @@ export interface CodecBitmap {
|
|
|
61
60
|
* cannot answer this in better than O(n) should simply omit it — the engine then skips the check rather
|
|
62
61
|
* than walking every value on the read path, which is the one thing this must never cost.
|
|
63
62
|
*
|
|
64
|
-
* Roaring answers it in O(1) from its container index, so the flagship codec implements it.
|
|
63
|
+
* Roaring answers it in O(1) from its container index, so the flagship codec implements it. The check is only
|
|
64
|
+
* as good as the answer: a codec that implements this must return the true largest value of whatever
|
|
65
|
+
* {@link CodecInterface.safeDeserialize} decoded, which for roaring holds only because that call refuses bytes
|
|
66
|
+
* whose containers or values are out of order.
|
|
65
67
|
*/
|
|
66
68
|
maximum?(): number | undefined;
|
|
67
69
|
/**
|
|
@@ -98,16 +100,22 @@ export interface CodecInterface {
|
|
|
98
100
|
/**
|
|
99
101
|
* Size-cap (`bytes.length <= maxBytes`) then portable-deserialize. Throws `IntegrityError` when the input
|
|
100
102
|
* exceeds the cap or fails to decode — the native decoder is never handed unbounded or unsafe-format input.
|
|
103
|
+
*
|
|
104
|
+
* "Fails to decode" includes bytes that are the right length and the wrong shape. A bitmap this returns must
|
|
105
|
+
* satisfy every invariant its own operations assume (for roaring: containers and values in order, runs
|
|
106
|
+
* disjoint, cardinalities matching the bits), because the engine trusts `has`, `size`, iteration order and
|
|
107
|
+
* {@link CodecBitmap.maximum} on what comes back, and a decoder that only bounds its reads guarantees none of
|
|
108
|
+
* them.
|
|
101
109
|
*/
|
|
102
110
|
safeDeserialize(bytes: Uint8Array, maxBytes: number): CodecBitmap;
|
|
103
111
|
}
|
|
104
112
|
/**
|
|
105
113
|
* Resolve a codec that a **public core entry point** was given, failing fast when it is missing.
|
|
106
114
|
*
|
|
107
|
-
* Why these entry points take `codec?` rather than a required field: `
|
|
108
|
-
* `
|
|
115
|
+
* Why these entry points take `codec?` rather than a required field: `loadSegment` / `eraseIdFromSegment` /
|
|
116
|
+
* `runExport` are call-compatible public API, and core cannot supply a default (the concrete
|
|
109
117
|
* codec lives in a *flavor* package that depends on core — a default here would invert that arrow). A **flavor**
|
|
110
|
-
* package binds the codec for its users (`@cloudbitmaps/roaring`
|
|
118
|
+
* package binds the codec for its users (`@cloudbitmaps/roaring`'s store passes its codec to each of these), so an
|
|
111
119
|
* application never reaches this throw; only someone calling `@cloudbitmaps/core` directly — i.e. a flavor or
|
|
112
120
|
* driver author — can, and for them the typed error names exactly what to pass.
|
|
113
121
|
*/
|
|
@@ -2,15 +2,15 @@
|
|
|
2
2
|
* Fail-safe cross-store disaster-recovery check. The registry (`currentGen`) and the immutable `.crbm`
|
|
3
3
|
* generations can be restored **independently**, so a failover can recover the registry *ahead of* the storage
|
|
4
4
|
* objects — leaving `currentGen` pointing at a generation whose `.crbm` isn't present yet. That is likelier
|
|
5
|
-
* than it sounds even
|
|
5
|
+
* than it sounds even with both in one bucket: a restore scoped to a prefix, or replayed
|
|
6
6
|
* per-object from a version history, recovers the two prefixes at different points. That's a torn restore: reads of the affected segment then throw. This scan
|
|
7
7
|
* detects it up front (run it at startup after a restore) instead of discovering it on the first read.
|
|
8
8
|
*
|
|
9
9
|
* Read-only; bounded fan-out. `destroyed` (crypto-shredded) segments are skipped — their Storage is intentionally
|
|
10
10
|
* gone/unreadable, not a torn restore. A segment whose Storage/registry can't be read this pass is recorded in
|
|
11
11
|
* `errored` (never aborts the scan). Each segment is checked against its **authoritative live pointer** — one
|
|
12
|
-
* strong `registry.get` per segment — never the enumeration snapshot from `registry.list`, which
|
|
13
|
-
* eventually-
|
|
12
|
+
* strong `registry.get` per segment — never the enumeration snapshot from `registry.list`, which a driver may
|
|
13
|
+
* serve eventually-consistently and so lag a recent in-place pointer advance: trusting it would both
|
|
14
14
|
* miss a torn *live* generation and cry torn on a generation the pointer has already advanced past (GC'd during
|
|
15
15
|
* the scan). Residual: a load's publish plus a GC landing in the tiny per-segment get→list gap can still yield a
|
|
16
16
|
* transient false positive — run the scan against a quiesced fleet (the documented restore procedure), or re-run
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
|
|
5
5
|
* where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
|
|
6
|
-
* to occupy. Measured on a 1M-id load
|
|
7
|
-
* not turn at all
|
|
6
|
+
* to occupy. Measured on a 1M-id load of 65,536 chunks onto in-memory storage, on a loaded Apple M3 Pro: **518 ms
|
|
7
|
+
* wall, and 519 ms during which the event loop did not turn at all** (medians of 20 fresh-process runs). Wire that to a request handler and every other in-flight request on the instance — health
|
|
8
8
|
* checks included — waits out the whole load.
|
|
9
9
|
*
|
|
10
10
|
* The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
|
|
@@ -29,10 +29,10 @@
|
|
|
29
29
|
* baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
|
|
30
30
|
*
|
|
31
31
|
* **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
|
|
32
|
-
*
|
|
32
|
+
* 518 ms wall / 519 ms blocked unyielded, 526 ms / 24 ms yielded. (b) This synthetic yield-primitive comparison:
|
|
33
33
|
* 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
|
|
34
|
-
* 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another
|
|
35
|
-
*
|
|
34
|
+
* 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another describes
|
|
35
|
+
* work none of them did, such as a 92 ms operation with 450 ms of starvation inside it.
|
|
36
36
|
*/
|
|
37
37
|
/**
|
|
38
38
|
* The only part of a `Clock` that yielding actually needs.
|
|
@@ -70,9 +70,9 @@ export declare const YIELD_EVERY = 1024;
|
|
|
70
70
|
* The two-line shape is deliberate. `await tick()` would read better and would force a microtask turn on every
|
|
71
71
|
* single iteration — an unconditional cost on the very loop this exists to keep fast.
|
|
72
72
|
*
|
|
73
|
-
* With no `clock` the result always returns `null`: the loop runs
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
73
|
+
* With no `clock` the result always returns `null`: the loop runs unbroken, so adding a yield point to a code
|
|
74
|
+
* path can never change behaviour for a caller who did not opt in. `core/` cannot supply a clock itself — it is
|
|
75
|
+
* timer-free by lint, which is the whole reason scheduling goes through the seam — so `@cloudbitmaps/roaring`
|
|
76
|
+
* pre-binds a real one and flavor users get this with no wiring at all.
|
|
77
77
|
*/
|
|
78
78
|
export declare function yieldEvery(clock: Yielder | undefined, everyN?: number): () => Promise<void> | null;
|
package/dist/core/cost.d.ts
CHANGED
|
@@ -7,38 +7,157 @@ export interface PricingProfile {
|
|
|
7
7
|
/** Object PUT (per million) — what a load pays, per request. */
|
|
8
8
|
readonly putPerMillion: number;
|
|
9
9
|
readonly storagePerGiBMonth: number;
|
|
10
|
+
/**
|
|
11
|
+
* Requests one read costs when it needs the object's size first: a pointer read, and a segment's tail read.
|
|
12
|
+
* Default **1**, S3's, whose suffix-range GET returns the size with the bytes. **2** on GCS and Azure Blob,
|
|
13
|
+
* which read the metadata and then the bytes. A chunk read knows its range, and is one request everywhere.
|
|
14
|
+
*/
|
|
15
|
+
readonly requestsPerSizedRead?: number;
|
|
10
16
|
};
|
|
11
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* The always-on Redis the verdict compares against: exactly one of the two. `sizedToData`, the default's, prices
|
|
19
|
+
* the cheapest cluster that holds the report's stored bytes; `monthlyUSD` is one cluster you name, whatever the
|
|
20
|
+
* data size. A profile carrying both is refused rather than read one way.
|
|
21
|
+
*/
|
|
12
22
|
readonly redis: {
|
|
13
23
|
readonly monthlyUSD: number;
|
|
24
|
+
readonly sizedToData?: never;
|
|
25
|
+
} | {
|
|
26
|
+
readonly sizedToData: RedisSizing;
|
|
27
|
+
readonly monthlyUSD?: never;
|
|
14
28
|
};
|
|
15
29
|
}
|
|
30
|
+
/** One node type a Redis cluster can be built from, as its cloud prices it. */
|
|
31
|
+
export interface RedisNodeType {
|
|
32
|
+
readonly name: string;
|
|
33
|
+
/** Memory, in GiB, as the cloud publishes it for the node type. */
|
|
34
|
+
readonly memoryGiB: number;
|
|
35
|
+
/**
|
|
36
|
+
* SSD, in GiB, on a data-tiering node (ElastiCache's `r6gd`): keys stay in memory, and the values read least
|
|
37
|
+
* recently move to the SSD. Counted in full beside the memory, as AWS counts a data-tiering node's capacity.
|
|
38
|
+
*/
|
|
39
|
+
readonly ssdGiB?: number;
|
|
40
|
+
/** Price per node-hour, in USD. */
|
|
41
|
+
readonly hourlyUSD: number;
|
|
42
|
+
/**
|
|
43
|
+
* The most shards a cluster of this type is priced at. The default catalogue prices its burstable nodes as one
|
|
44
|
+
* shard: a cluster sharded across them is not how data anyone queries is held. Default: no limit.
|
|
45
|
+
*/
|
|
46
|
+
readonly maxShards?: number;
|
|
47
|
+
}
|
|
16
48
|
/**
|
|
17
|
-
*
|
|
18
|
-
*
|
|
49
|
+
* Redis sized to hold the data: enough shards for the bytes, each a primary and its replicas, on whichever node
|
|
50
|
+
* type in `nodeTypes` makes that cheapest. Prices within a millionth of a dollar of the cheapest are a tie, which
|
|
51
|
+
* fewer nodes win, then the lower price, then the name, so the answer never depends on the order of the rows. The data is held at its stored,
|
|
52
|
+
* compressed size, which is a floor on the memory Redis needs for it: a native Redis bitmap is sized by its highest
|
|
53
|
+
* id, not by how many ids it holds, so sparse ids need more.
|
|
54
|
+
*/
|
|
55
|
+
export interface RedisSizing {
|
|
56
|
+
/** Where the prices come from, and when; the report's notes quote it. */
|
|
57
|
+
readonly source: string;
|
|
58
|
+
readonly nodeTypes: readonly RedisNodeType[];
|
|
59
|
+
/** Replicas per shard. AWS's best practice is 2; Multi-AZ needs at least 1. */
|
|
60
|
+
readonly replicasPerShard: number;
|
|
61
|
+
/** The share of each node's memory kept back from data: ElastiCache's `reserved-memory-percent`, 25% by default. */
|
|
62
|
+
readonly reservedMemoryFraction: number;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* **ElastiCache for Redis OSS in us-east-1, on-demand**: node prices from AWS's public price list, version
|
|
66
|
+
* 20260914063714 (published 2026-09-14, effective 2026-09-01,
|
|
67
|
+
* https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonElastiCache/20260914063714/us-east-1/index.json), and the
|
|
68
|
+
* memory AWS publishes for each node type. Every shard is a primary and two replicas, AWS's best practice, with
|
|
69
|
+
* ElastiCache's default 25% of each node's memory reserved, and the burstable `t4g` nodes are priced as one shard.
|
|
70
|
+
*
|
|
71
|
+
* It lists only rows that are the cheapest fit for some size of data: the data-tiering `r6gd` rows above about
|
|
72
|
+
* 30 GiB, and the larger in-memory `r6g` rows, `2xlarge` and up, when those are left out. The current `m7g` and `r7g` nodes are not here,
|
|
73
|
+
* because an `m6g` or `r6g` with the same memory costs less. It is the cheapest cluster of THIS kind; Redis bought
|
|
74
|
+
* another way costs less — one replica a shard a third less, ElastiCache for Valkey 20% less a node, reserved nodes
|
|
75
|
+
* less again — so against those the saving the verdict reports is smaller. Pass their prices to compare with them.
|
|
76
|
+
*/
|
|
77
|
+
export declare const ELASTICACHE_REDIS_US_EAST_1_ONDEMAND: RedisSizing;
|
|
78
|
+
/**
|
|
79
|
+
* One ElastiCache HA cluster, a primary and two replicas of cache.m7g.large — the m7g family's smallest node —
|
|
80
|
+
* at $0.158 an hour each: 3 × 730 h × $0.158 = $346.02, published to the dollar. The per-request crossovers on the
|
|
81
|
+
* benchmarks page are drawn against this one cluster, whatever the data size; pass it as `pricing.redis` to compare
|
|
82
|
+
* with it. It is not the cheapest cluster for any size of data: three cache.m6g.large hold the same memory for less.
|
|
83
|
+
*/
|
|
84
|
+
export declare const ONE_REDIS_HA_CLUSTER: Readonly<{
|
|
85
|
+
monthlyUSD: number;
|
|
86
|
+
}>;
|
|
87
|
+
/**
|
|
88
|
+
* Default profile — **AWS us-east-1, on-demand**, mid-2026, from the fact-checked published pricing rather than
|
|
89
|
+
* copied from a blog post, with Redis sized to the data. Override it for your region, cloud, or committed term.
|
|
19
90
|
*/
|
|
20
91
|
export declare const AWS_US_EAST_1_ONDEMAND: PricingProfile;
|
|
21
|
-
export declare const DEFAULT_PRICING: PricingProfile;
|
|
22
92
|
/** Sustained access pattern. All rates default to 0; unspecified ⇒ that op contributes nothing. */
|
|
23
93
|
export interface Workload {
|
|
24
|
-
/**
|
|
94
|
+
/**
|
|
95
|
+
* Point reads (`has`) per second. Each cache miss is one object GET, once the segment's pointer and index are
|
|
96
|
+
* read.
|
|
97
|
+
*/
|
|
25
98
|
readonly readsPerSec?: number;
|
|
26
99
|
readonly intersectsPerSec?: number;
|
|
27
|
-
/**
|
|
100
|
+
/** Cache hit rate in `[0, 1]` — hits are free; only misses cost. Default 0. */
|
|
28
101
|
readonly cacheHitRate?: number;
|
|
29
|
-
/**
|
|
102
|
+
/**
|
|
103
|
+
* Storage chunks one intersection fetches, summed over its operands: the chunk-skipping survivors. Default 1.
|
|
104
|
+
* Two segments sharing `k` chunks fetch `2k`. The model adds each operand's pointer and index reads itself (see
|
|
105
|
+
* {@link Workload.operandsPerIntersect}), so count chunks only. One more GET per operand whose index outgrows
|
|
106
|
+
* the reader's tail read (256 KiB by default), which then reads the index whole, belongs here too, and so does a
|
|
107
|
+
* pointer re-read by an intersect slow enough to outlive {@link Workload.genTtlMs}.
|
|
108
|
+
*/
|
|
30
109
|
readonly chunksPerIntersect?: number;
|
|
110
|
+
/**
|
|
111
|
+
* Segments each intersection reads, `exclude` operands included; at least 1. Default 2. An intersection is priced
|
|
112
|
+
* **cold**: before its chunks, each operand's pointer is read, then its index, in one read of the object's tail —
|
|
113
|
+
* 2 GETs an operand, so a cold intersect of two segments sharing `k` chunks is `4 + 2k` GETs. `cacheHitRate`
|
|
114
|
+
* does not apply to intersections, so a long-lived reader that answers a repeat from its cache pays less. Other
|
|
115
|
+
* combines read their operands the same way and can be priced here too, with the chunks they fetch.
|
|
116
|
+
*/
|
|
117
|
+
readonly operandsPerIntersect?: number;
|
|
31
118
|
/**
|
|
32
119
|
* Generations published per month across the modeled data — the write side of a loaded store. Default **0**
|
|
33
120
|
* ⇒ loads are not modeled and the report *discloses* the omission rather than silently under-reporting.
|
|
34
121
|
*/
|
|
35
122
|
readonly loadsPerMonth?: number;
|
|
36
123
|
/**
|
|
37
|
-
* PUT-class requests one load issues. Default **1** (a single-object
|
|
38
|
-
* `P + 2` (initiate, the parts, complete) — set it when you know your
|
|
39
|
-
*
|
|
124
|
+
* PUT-class requests one load's object write issues, each priced at the PUT rate. Default **1** (a single-object
|
|
125
|
+
* PUT). A multipart write of `P` parts bills `P + 2` (initiate, the parts, complete) — set it when you know your
|
|
126
|
+
* object sizes. The model adds what `store.load()` does around the write: two listings and the pointer's write,
|
|
127
|
+
* PUT-class on S3, and nine GETs, the pointer read eight times and the current generation's index once. That is
|
|
128
|
+
* a segment with two generations behind it; its first load makes two fewer GETs, and its second one fewer. On
|
|
129
|
+
* S3 at the default prices a single-part `store.load()` is then about $23.60 per million. A segment whose index
|
|
130
|
+
* outgrows the tail read makes one more GET, and a publish that loses a race to another writer reads the
|
|
131
|
+
* pointer again.
|
|
40
132
|
*/
|
|
41
133
|
readonly requestsPerLoad?: number;
|
|
134
|
+
/**
|
|
135
|
+
* Segments each long-lived reader process keeps reading. A reader re-reads a segment's pointer when it reads the
|
|
136
|
+
* segment after {@link Workload.genTtlMs} has passed, so each costs at most one GET per `genTtlMs` in each process
|
|
137
|
+
* that reads it — 1,314,000 GETs a month at the default 2 s — and the whole term at most one GET per point read
|
|
138
|
+
* (`readsPerSec`, across the fleet). Intersections are priced cold and pay their own pointer reads. Default **0**
|
|
139
|
+
* ⇒ the refresh is not modeled, and the report *discloses* the omission when there are reads to refresh for.
|
|
140
|
+
*
|
|
141
|
+
* It assumes each hot segment stays open in the reader's cache (`cache.readerMax`, 1,024 segments by default,
|
|
142
|
+
* and `cache.readerMaxBytes`, 64 MiB of parsed index). A read of a segment the cache evicted opens it again, a
|
|
143
|
+
* pointer read and a tail read, which is not priced here, and neither is the re-open every reader makes after
|
|
144
|
+
* each load, for the new generation's index. Size those caches to keep the hot set open; the report says so when
|
|
145
|
+
* `hotSegments` is more than a store keeps open by default.
|
|
146
|
+
*/
|
|
147
|
+
readonly hotSegments?: number;
|
|
148
|
+
/**
|
|
149
|
+
* Long-lived reader processes, each keeping its own {@link Workload.hotSegments} open and refreshing their
|
|
150
|
+
* pointers on its own: ten processes reading the same hundred segments pay ten times one process's refresh.
|
|
151
|
+
* At least 1. Default 1.
|
|
152
|
+
*/
|
|
153
|
+
readonly readerProcesses?: number;
|
|
154
|
+
/**
|
|
155
|
+
* How long the reader trusts a pointer, in ms: the store's `cache.genTtlMs`. Default 2000, the store's own
|
|
156
|
+
* default; `segment.costReport()` uses the store's. `0` turns the timed refresh off, and the model bills none: a
|
|
157
|
+
* reader then re-reads a pointer only when its cache evicts the segment, a read finds its generation swept, or a
|
|
158
|
+
* write invalidates it, none of which this term prices.
|
|
159
|
+
*/
|
|
160
|
+
readonly genTtlMs?: number;
|
|
42
161
|
}
|
|
43
162
|
/** One (group of) segment(s) for planning. `count` = how many like this (default 1). */
|
|
44
163
|
export interface SegmentSizing {
|
|
@@ -58,16 +177,48 @@ export interface CostReport {
|
|
|
58
177
|
readonly reads: number;
|
|
59
178
|
readonly intersects: number;
|
|
60
179
|
readonly storage: number;
|
|
61
|
-
/**
|
|
180
|
+
/**
|
|
181
|
+
* Loads: each its object's `requestsPerLoad` PUT-class requests plus what `store.load()` adds around them.
|
|
182
|
+
* 0 unless `loadsPerMonth` is set.
|
|
183
|
+
*/
|
|
62
184
|
readonly loads: number;
|
|
185
|
+
/** The pointer refresh of {@link Workload.hotSegments}. 0 unless `hotSegments` is set. */
|
|
186
|
+
readonly pointerRefresh: number;
|
|
63
187
|
};
|
|
64
188
|
readonly total: number;
|
|
65
189
|
};
|
|
66
190
|
/**
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
191
|
+
* The Redis the verdict compares against. Sized, the default, it is the cheapest cluster in
|
|
192
|
+
* `pricing.redis.sizedToData` that holds this report's stored bytes; fixed, it is `pricing.redis.monthlyUSD` as
|
|
193
|
+
* given.
|
|
194
|
+
*
|
|
195
|
+
* **It is not additive across reports.** Each report sizes Redis to its own bytes, so a per-segment report
|
|
196
|
+
* (`segment.costReport()`) compares with a cluster holding that one segment alone, and the baselines of a store's
|
|
197
|
+
* segments do not sum to the store's. To judge a store, price all its segments in one `estimateCost`. To alarm on
|
|
198
|
+
* one, sum `monthlyUSD.total` over its segments and compare the sum with the Redis you would run for it: a
|
|
199
|
+
* per-segment verdict against that whole price fires only when one segment alone costs more than all of it.
|
|
200
|
+
*/
|
|
201
|
+
readonly redisBaseline: {
|
|
202
|
+
readonly basis: 'fixed';
|
|
203
|
+
readonly monthlyUSD: number;
|
|
204
|
+
} | {
|
|
205
|
+
readonly basis: 'sized-to-data';
|
|
206
|
+
readonly monthlyUSD: number;
|
|
207
|
+
/** The node type, the shards, the nodes (every shard's primary and replicas), and whether they tier to SSD. */
|
|
208
|
+
readonly cluster: {
|
|
209
|
+
readonly nodeType: string;
|
|
210
|
+
readonly shards: number;
|
|
211
|
+
readonly nodes: number;
|
|
212
|
+
readonly dataTiering: boolean;
|
|
213
|
+
};
|
|
214
|
+
};
|
|
215
|
+
/**
|
|
216
|
+
* Sustained read rate at which the pay-per-use model's cost passes {@link CostReport.redisBaseline}, **evaluated
|
|
217
|
+
* at this report's `cacheHitRate`** — so a higher cache-hit rate raises it (cache hits are free) — with the other
|
|
218
|
+
* request axes at 0, and with what keeping the data readable costs taken out of the baseline first: storage, and
|
|
219
|
+
* this report's pointer refresh. Loads, the write side, are left out of it. `Infinity` means it never crosses (a
|
|
220
|
+
* 100% cache-hit rate). The published anchor (~329 reads/s) is against {@link ONE_REDIS_HA_CLUSTER}, at
|
|
221
|
+
* `cacheHitRate: 0` with no refresh modeled.
|
|
71
222
|
*/
|
|
72
223
|
readonly redisCrossover: {
|
|
73
224
|
readonly readsPerSec: number;
|
|
@@ -79,6 +230,7 @@ export interface CostReport {
|
|
|
79
230
|
readonly pricingName: string;
|
|
80
231
|
/** True when segment **sizes** were real (grounded `costReport`), false for a pure `estimateCost`. */
|
|
81
232
|
readonly grounded: boolean;
|
|
233
|
+
/** What the model assumed and what it left out, in words. The last says how the Redis was priced. */
|
|
82
234
|
readonly notes: readonly string[];
|
|
83
235
|
};
|
|
84
236
|
}
|
|
@@ -59,12 +59,10 @@ export declare const ELEMENT_WIDTH_32 = 32;
|
|
|
59
59
|
* see the `.crbm` format section of the API reference). This field is what lets one container hold either, the same way ZIP tags each member
|
|
60
60
|
* with a compression method.
|
|
61
61
|
*
|
|
62
|
-
* **
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
* one-line change now and an expensive one after the freeze. The byte layout is untouched: same offset (24),
|
|
67
|
-
* same width (u16), same golden corpus.
|
|
62
|
+
* **A codec id, not a roaring-specific one.** A second codec is genuinely expected — `soaring` is a planned
|
|
63
|
+
* Roaring *variant*, so its serialized bytes are unlikely to be roaring-portable, and it lands *after* the `1.0`
|
|
64
|
+
* format freeze. A field frozen under a codec-specific meaning could not be reinterpreted later without a major
|
|
65
|
+
* format version.
|
|
68
66
|
*
|
|
69
67
|
* **Ids are permanent once published.** Add to {@link KNOWN_PAYLOAD_CODEC_IDS} when a codec ships; never
|
|
70
68
|
* reuse or renumber. Ids are deliberately *not* pre-allocated for codecs that do not exist — a reserved number
|
|
@@ -74,8 +72,8 @@ export declare const PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
|
74
72
|
/**
|
|
75
73
|
* Every payload codec id this reader can decode.
|
|
76
74
|
*
|
|
77
|
-
* The reader validates membership rather than equality with a single constant. That is the whole point of
|
|
78
|
-
*
|
|
75
|
+
* The reader validates membership rather than equality with a single constant. That is the whole point of an
|
|
76
|
+
* id registry: an unknown id is rejected with a typed error naming it, so an old reader meeting a
|
|
79
77
|
* future-codec generation **fails closed** — the correct direction, and the reason a store built on one codec
|
|
80
78
|
* can never silently misread another's bytes as its own. (The homogeneity contract means one store is one
|
|
81
79
|
* codec, so meeting a foreign generation implies misconfiguration, and a loud rejection is exactly what you
|
|
@@ -107,7 +105,7 @@ export declare const MAX_CHUNK_CARDINALITY = 65536;
|
|
|
107
105
|
/** Fixed width of a per-chunk CRC32C field in the index (high-entropy → not varint). */
|
|
108
106
|
export declare const CRC32C_BYTES = 4;
|
|
109
107
|
/**
|
|
110
|
-
* v1 AEAD framing sizes (AES-256-GCM), fixed by the format exactly like the
|
|
108
|
+
* v1 AEAD framing sizes (AES-256-GCM), fixed by the format exactly like the payload codec id — a
|
|
111
109
|
* different cipher is a new format version. An encrypted chunk payload is stored as `nonce ‖ ciphertext ‖ tag`;
|
|
112
110
|
* the encrypted **index**'s nonce/tag live in the footer's reserved `indexNonce`/`indexTag` slots. Matches
|
|
113
111
|
* {@link FOOTER.indexNonce} (12 B) and {@link FOOTER.indexTag} (16 B).
|
|
@@ -37,7 +37,15 @@ export declare class CrbmReader {
|
|
|
37
37
|
readonly servedFromTail: boolean;
|
|
38
38
|
/** Set iff the object is encrypted — used to decrypt each chunk payload in {@link getChunk}. */
|
|
39
39
|
private readonly crypto;
|
|
40
|
+
/** The footer's own CRC, as verified at open: it covers the index's CRC, the chunk count and the generation. */
|
|
41
|
+
private readonly footerCrc;
|
|
40
42
|
private constructor();
|
|
43
|
+
/**
|
|
44
|
+
* Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
|
|
45
|
+
* objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
|
|
46
|
+
* the object it reopens is the one it pinned, since a purged and reloaded name reuses the generation number.
|
|
47
|
+
*/
|
|
48
|
+
get fingerprint(): string;
|
|
41
49
|
/** Total object bytes (from the one-GET tail read) — for grounded storage cost. */
|
|
42
50
|
get sizeBytes(): number;
|
|
43
51
|
/**
|
|
@@ -48,6 +56,15 @@ export declare class CrbmReader {
|
|
|
48
56
|
get retainedIndexBytes(): number;
|
|
49
57
|
/** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
|
|
50
58
|
cardinalities(): Map<number, number>;
|
|
59
|
+
/**
|
|
60
|
+
* Whether the object behind `blob` is the one `fingerprint` names ({@link fingerprint}), from one tail read of a
|
|
61
|
+
* footer's worth, with no key and no index, since the footer is stored in the clear and checked by its own CRC.
|
|
62
|
+
* Another size is another object, whatever its bytes hold: a short one that is no `.crbm` at all is not the one
|
|
63
|
+
* pinned. At the same size the footer's CRC says, and a footer that fails its own checks says nothing about which
|
|
64
|
+
* object is there, so this throws, as an open would. What a pin asks, to tell the object it opened from one that
|
|
65
|
+
* has since been stored under its key, whoever wrote it.
|
|
66
|
+
*/
|
|
67
|
+
static sameObject(blob: BlobReader, fingerprint: string): Promise<boolean>;
|
|
51
68
|
static open(blob: BlobReader, options?: CrbmReaderOptions): Promise<CrbmReader>;
|
|
52
69
|
/** Chunk keys present in this generation, ascending. */
|
|
53
70
|
chunkKeys(): number[];
|
|
@@ -62,7 +79,7 @@ export declare class CrbmReader {
|
|
|
62
79
|
*/
|
|
63
80
|
getChunk(chunkKey: number): Promise<Uint8Array | null>;
|
|
64
81
|
}
|
|
65
|
-
export declare function parseIndex(indexBytes: Uint8Array,
|
|
82
|
+
export declare function parseIndex(indexBytes: Uint8Array, payloadEnd: number, maxPayloadBytes: number, minPayloadBytes?: number): {
|
|
66
83
|
entries: Map<number, CrbmIndexEntry>;
|
|
67
84
|
orderedKeys: number[];
|
|
68
85
|
cardinalitySum: number;
|
|
@@ -6,8 +6,8 @@ export interface CrbmWriterOptions {
|
|
|
6
6
|
/**
|
|
7
7
|
* Which codec produced the chunk payloads. Defaults to {@link PAYLOAD_CODEC_ROARING_PORTABLE}.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* A codec id rather than a roaring-specific one, because `.crbm` is a shared container and a second codec is
|
|
10
|
+
* expected before the format freezes.
|
|
11
11
|
*/
|
|
12
12
|
readonly payloadCodecId?: number;
|
|
13
13
|
readonly elementWidth?: number;
|