@cloudbitmaps/roaring 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/PRIVACY.md +198 -72
- package/README.md +148 -50
- package/dist/bin/export-segments.d.ts +21 -0
- package/dist/bin/export-segments.js +1508 -30
- package/dist/bin/export-segments.js.map +7 -1
- package/dist/codec-bound.d.ts +38 -0
- package/dist/index.d.ts +875 -425
- package/dist/index.js +997 -306
- package/dist/index.js.map +7 -1
- package/dist/moved-options.d.ts +14 -0
- package/dist/portable/decode.d.ts +54 -0
- package/dist/roaring-codec.d.ts +61 -0
- package/dist/system-clock.d.ts +40 -0
- package/dist/testing/conformance.d.ts +34 -0
- package/dist/testing/fuzz-codec.d.ts +9 -0
- package/package.json +11 -116
- package/dist/azure/index.cjs +0 -14
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -9
- package/dist/azure/index.d.ts +0 -9
- package/dist/azure/index.js +0 -3
- package/dist/azure/index.js.map +0 -1
- package/dist/bin/compact-segments.js +0 -134
- package/dist/bin/compact-segments.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -14
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -1
- package/dist/cassandra/index.d.ts +0 -1
- package/dist/cassandra/index.js +0 -3
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-4NVOQJGJ.js +0 -801
- package/dist/chunk-4NVOQJGJ.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -14
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -1
- package/dist/dynamodb/index.d.ts +0 -1
- package/dist/dynamodb/index.js +0 -3
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -14
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -1
- package/dist/gcs/index.d.ts +0 -1
- package/dist/gcs/index.js +0 -3
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -789
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -642
- package/dist/mongodb/index.cjs +0 -14
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -1
- package/dist/mongodb/index.d.ts +0 -1
- package/dist/mongodb/index.js +0 -3
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -14
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -1
- package/dist/mysql/index.d.ts +0 -1
- package/dist/mysql/index.js +0 -3
- package/dist/mysql/index.js.map +0 -1
- package/dist/postgres/index.cjs +0 -14
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -1
- package/dist/postgres/index.d.ts +0 -1
- package/dist/postgres/index.js +0 -3
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -14
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -1
- package/dist/redis/index.d.ts +0 -1
- package/dist/redis/index.js +0 -3
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -14
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -1
- package/dist/s3/index.d.ts +0 -1
- package/dist/s3/index.js +0 -3
- package/dist/s3/index.js.map +0 -1
package/dist/index.d.cts
DELETED
|
@@ -1,642 +0,0 @@
|
|
|
1
|
-
import { bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, CodecBitmap, CodecInterface, BudgetOption, IColdDriver, ColdChunkSource, IWarmDriver, IRegistryDriver, IKeystore, Clock, Rng, RetryPolicy, IMetricsSink, SegmentEngine, SegmentRef, PricingProfile, Workload, Topology, CostReport, IAuditSink, CompactionOptions, CompactionResult, DropResult, ConsistencyReport, ExportSink, ExportOptions, ExportManifest, EstimateInput } from '@cloudbitmaps/core';
|
|
2
|
-
export * from '@cloudbitmaps/core';
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Codec-bound wrappers over the public `@cloudbitmaps/core` entry points that need a bitmap codec.
|
|
6
|
-
*
|
|
7
|
-
* `core` is codec-agnostic and therefore cannot default the codec (the concrete codec lives *here*, in a package
|
|
8
|
-
* that depends on core — a default over there would invert the dependency arrow). But an application using
|
|
9
|
-
* `@cloudbitmaps/roaring` should never have to pass one. So this module re-binds each such entry point with
|
|
10
|
-
* {@link roaringCodec} and the facade re-exports these **explicitly**, which shadows the same names coming from
|
|
11
|
-
* `export * from '@cloudbitmaps/core'` (an explicit export always wins over a star export). Net effect: every
|
|
12
|
-
* call signature is unchanged from before the family split — you can still call
|
|
13
|
-
* `bulkLoadCrbmGeneration(driver, key, ids)` with no options at all.
|
|
14
|
-
*
|
|
15
|
-
* A caller who *wants* a different codec passes it explicitly; the binding only fills an absent one — via
|
|
16
|
-
* `?? roaringCodec` rather than spread order, so an explicit `codec: undefined` still gets the binding
|
|
17
|
-
* (`exactOptionalPropertyTypes` is off, so that call typechecks and must not fall through to the throw).
|
|
18
|
-
*/
|
|
19
|
-
|
|
20
|
-
type BulkLoad = typeof bulkLoadCrbmGeneration$1;
|
|
21
|
-
type CompactSegment = typeof compactSegment$1;
|
|
22
|
-
type RunCompactionCycle = typeof runCompactionCycle$1;
|
|
23
|
-
type RunExport = typeof runExport$1;
|
|
24
|
-
/**
|
|
25
|
-
* {@link coreBulkLoad} with the roaring codec **and a real clock** pre-bound.
|
|
26
|
-
*
|
|
27
|
-
* The clock is what makes a large load **cooperative**: bulk-load yields the event loop periodically instead of
|
|
28
|
-
* stalling the process for its whole duration (measured end-to-end at 450 ms of unbroken blocking for a 1M-id
|
|
29
|
-
* load, now ~19 ms worst case). `core/` cannot
|
|
30
|
-
* default it — it is timer-free by lint, which is precisely why waiting goes through the `Clock` seam — so the
|
|
31
|
-
* flavor package supplies it, exactly as it supplies the codec. A caller who passes their own clock (a virtual
|
|
32
|
-
* one in a simulation, say) keeps it.
|
|
33
|
-
*/
|
|
34
|
-
declare const bulkLoadCrbmGeneration: BulkLoad;
|
|
35
|
-
/** {@link coreCompactSegment} with the roaring codec pre-bound. */
|
|
36
|
-
declare const compactSegment: CompactSegment;
|
|
37
|
-
/** {@link coreRunCompactionCycle} with the roaring codec pre-bound. */
|
|
38
|
-
declare const runCompactionCycle: RunCompactionCycle;
|
|
39
|
-
/** {@link coreRunExport} with the roaring codec pre-bound (only the `'roaring'` format needs it). */
|
|
40
|
-
declare const runExport: RunExport;
|
|
41
|
-
|
|
42
|
-
declare class SafeBitmap implements CodecBitmap {
|
|
43
|
-
private readonly bitmap;
|
|
44
|
-
private constructor();
|
|
45
|
-
static empty(): SafeBitmap;
|
|
46
|
-
static fromValues(values: Iterable<number>): SafeBitmap;
|
|
47
|
-
/**
|
|
48
|
-
* S1: validate size, then deserialize with the **portable** (validated) format.
|
|
49
|
-
* Throws `IntegrityError` if the input exceeds `maxBytes` or fails to decode — the
|
|
50
|
-
* native addon is never handed unbounded or unsafe-format input.
|
|
51
|
-
*/
|
|
52
|
-
static safeDeserialize(bytes: Uint8Array, maxBytes: number): SafeBitmap;
|
|
53
|
-
serialize(): Uint8Array;
|
|
54
|
-
/**
|
|
55
|
-
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
56
|
-
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
57
|
-
*
|
|
58
|
-
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
59
|
-
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
60
|
-
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
61
|
-
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
62
|
-
* whichever encoding is smaller per container.
|
|
63
|
-
*
|
|
64
|
-
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
65
|
-
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
66
|
-
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
67
|
-
* columns; don't carry a figure between them without checking which denominator it had.
|
|
68
|
-
*
|
|
69
|
-
* Called by the engine only when writing an immutable cold generation — see {@link CodecBitmap.optimize} for
|
|
70
|
-
* why the per-operation warm path deliberately does not.
|
|
71
|
-
*/
|
|
72
|
-
optimize(): void;
|
|
73
|
-
/**
|
|
74
|
-
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the container index, so the engine's
|
|
75
|
-
* per-chunk range assertion costs one call per chunk rather than a walk per id.
|
|
76
|
-
*/
|
|
77
|
-
maximum(): number | undefined;
|
|
78
|
-
add(value: number): void;
|
|
79
|
-
addMany(values: Iterable<number>): void;
|
|
80
|
-
remove(value: number): void;
|
|
81
|
-
removeMany(values: Iterable<number>): void;
|
|
82
|
-
has(value: number): boolean;
|
|
83
|
-
get size(): number;
|
|
84
|
-
get isEmpty(): boolean;
|
|
85
|
-
clone(): SafeBitmap;
|
|
86
|
-
/** In-place union: `this = this ∪ other`. */
|
|
87
|
-
orInPlace(other: CodecBitmap): void;
|
|
88
|
-
/** In-place difference: `this = this \ other`. */
|
|
89
|
-
andNotInPlace(other: CodecBitmap): void;
|
|
90
|
-
/** In-place intersection: `this = this ∩ other`. */
|
|
91
|
-
andInPlace(other: CodecBitmap): void;
|
|
92
|
-
/** Ascending iterator over the set values. */
|
|
93
|
-
[Symbol.iterator](): IterableIterator<number>;
|
|
94
|
-
toArray(): number[];
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* The roaring {@link CodecInterface} — the flagship codec, delegating to {@link SafeBitmap}'s statics. This is
|
|
98
|
-
* the default the `CloudRoaring` facade injects into the engine; `core/` itself never hard-references it once a
|
|
99
|
-
* caller supplies a codec. Moves to `@cloudbitmaps/roaring` when the package split lands.
|
|
100
|
-
*/
|
|
101
|
-
declare const roaringCodec: CodecInterface;
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* CloudRoaring — distributed, cloud-native Roaring Bitmaps.
|
|
105
|
-
*
|
|
106
|
-
* The `CloudRoaring` class is the read/write engine over the tiered seams. You wire storage **once**, as a
|
|
107
|
-
* single config object: a Cold driver (`cold`), a Warm driver (`warm`), and optionally a `registry` (the
|
|
108
|
-
* authoritative generation pointer — needed to read encrypted segments and to resolve generations without a
|
|
109
|
-
* cold `list`-scan) and a `keystore` (encryption-at-rest / crypto-shred). Pass a **raw** {@link IColdDriver}
|
|
110
|
-
* as `cold` (e.g. `S3ColdDriver`, `LocalFsColdDriver`, `MemoryColdDriver`) and the store assembles the `.crbm`
|
|
111
|
-
* cold source ({@link CrbmColdChunkSource}) for you — so each driver is named exactly once. Or pass an
|
|
112
|
-
* already-built {@link ColdChunkSource} to control advanced reader options yourself.
|
|
113
|
-
*
|
|
114
|
-
* In-process lifecycle helpers — `compact`, `eraseSubject`, `subjectReport` — reuse the store's own drivers, so
|
|
115
|
-
* you never re-pass them (they need the store built with a raw cold driver + registry). Out-of-process cold
|
|
116
|
-
* writers (the compaction daemon, the seed/bulk-load CLI, `destroySegment`/`eraseNamespace`) wire their own deps
|
|
117
|
-
* against the same drivers — they run in separate processes.
|
|
118
|
-
*
|
|
119
|
-
* **You do not need any of them to start.** Construct a store and call `add`/`addMany`/`has`/`remove` — a segment
|
|
120
|
-
* with no Cold generation at all is fully functional, because a read merges `(cold ∪ warm.adds) \ warm.removes`
|
|
121
|
-
* and an absent Cold tier just makes that merge trivial. `bulkLoadCrbmGeneration` is an **import** path for data
|
|
122
|
-
* you already have elsewhere, not an initialization step, and compaction is a **cost** optimization, never a
|
|
123
|
-
* correctness requirement. See the README and the getting-started guide.
|
|
124
|
-
*/
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Wiring for a {@link CloudRoaring} store. **Only `cold` and `warm` are required** — the minimal call is
|
|
128
|
-
* `new CloudRoaring({ cold, warm })`. Add a `registry` to unlock eject / compaction / encryption and to skip the
|
|
129
|
-
* cold list-scan (recommended for anything beyond a quick look). Everything else is **optional tuning with
|
|
130
|
-
* sensible defaults** — resilience/retries are already on, the hot cache is bounded, metrics are a no-op — so
|
|
131
|
-
* reach for them only when you need to.
|
|
132
|
-
*/
|
|
133
|
-
interface CloudRoaringOptions {
|
|
134
|
-
/**
|
|
135
|
-
* Cold tier. Pass a **raw** {@link IColdDriver} (`S3ColdDriver`, `LocalFsColdDriver`, `MemoryColdDriver`, …)
|
|
136
|
-
* and the store wraps it in a {@link CrbmColdChunkSource} using `registry`/`keystore` below — the common case,
|
|
137
|
-
* so you wire each driver **once**. Or pass an already-built {@link ColdChunkSource} (`MemoryColdChunkSource`,
|
|
138
|
-
* or a `CrbmColdChunkSource` you configured with advanced reader options) to use as-is.
|
|
139
|
-
*/
|
|
140
|
-
readonly cold: IColdDriver | ColdChunkSource;
|
|
141
|
-
/** Warm tier: the per-chunk delta store under OCC (`DynamoDbWarmDriver`, `LocalFsWarmDriver`, `MemoryWarmDriver`). */
|
|
142
|
-
readonly warm: IWarmDriver;
|
|
143
|
-
/**
|
|
144
|
-
* Authoritative registry — the per-segment `currentGen` pointer + wrapped-DEK holder. Applies when `cold` is a
|
|
145
|
-
* **raw driver**: it (a) resolves the current generation with one strong read instead of a cold `list`-scan,
|
|
146
|
-
* and (b) lets the store read **encrypted** segments (that's where wrapped DEKs live). Optional — a
|
|
147
|
-
* registry-less store reads the highest generation by list-scanning Cold (cleartext only). When you pass a
|
|
148
|
-
* pre-built `ColdChunkSource`, that source resolves its own generations, so a top-level `registry` is inert
|
|
149
|
-
* there and rejected as a wiring mistake — configure it on the source instead.
|
|
150
|
-
*/
|
|
151
|
-
readonly registry?: IRegistryDriver;
|
|
152
|
-
/**
|
|
153
|
-
* Keystore for encryption-at-rest / crypto-shred (Phase 4e). Required to read encrypted segments; needs a
|
|
154
|
-
* `registry` (that's where wrapped DEKs are stored). Applied only when `cold` is a raw driver — when you pass a
|
|
155
|
-
* pre-built {@link ColdChunkSource}, configure the keystore on that source instead.
|
|
156
|
-
*/
|
|
157
|
-
readonly keystore?: IKeystore;
|
|
158
|
-
/**
|
|
159
|
-
* Refuse to read a **cleartext** segment — a guard against silently reading data that should be encrypted.
|
|
160
|
-
* Needs a `registry`; applied only when `cold` is a raw driver. Off by default (encryption is opt-in).
|
|
161
|
-
*/
|
|
162
|
-
readonly requireEncryption?: boolean;
|
|
163
|
-
/** Injected for deterministic tests; defaults to a system clock. */
|
|
164
|
-
readonly clock?: Clock;
|
|
165
|
-
/** Injected for deterministic tests; defaults to `Math.random`-backed. Drives backoff jitter. */
|
|
166
|
-
readonly rng?: Rng;
|
|
167
|
-
/** HOT cache ceiling (decoded Cold chunks). */
|
|
168
|
-
readonly cacheMaxChunks?: number;
|
|
169
|
-
/** Optional TTL on cached chunks (ms). */
|
|
170
|
-
readonly cacheTtlMs?: number;
|
|
171
|
-
/**
|
|
172
|
-
* How long (ms) the store trusts a segment's resolved `currentGen` before re-resolving it on the next read
|
|
173
|
-
* (default 2000) — the bound on read staleness after a compaction commits a new generation (gap #4). Applies
|
|
174
|
-
* only when `cold` is a raw driver **and** a `registry` is wired (the cheap `currentGen` read the refresh
|
|
175
|
-
* needs; a registry-less store pins per source lifetime). Lazy — no timer; ≤ one registry read per segment per
|
|
176
|
-
* window, opening a new reader only when the generation actually advanced.
|
|
177
|
-
*/
|
|
178
|
-
readonly coldGenTtlMs?: number;
|
|
179
|
-
/**
|
|
180
|
-
* Ceiling on how many segments' `.crbm` readers (each holding a parsed index) the store keeps open at once
|
|
181
|
-
* (default 1024) — the steady-state memory bound for a long-running server that reads across many segments
|
|
182
|
-
* (gap #1). Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail
|
|
183
|
-
* GET. Applies only when `cold` is a raw driver (a pre-built `ColdChunkSource` manages its own reader cache).
|
|
184
|
-
*/
|
|
185
|
-
readonly coldReaderCacheMax?: number;
|
|
186
|
-
/**
|
|
187
|
-
* Aggregate byte ceiling on the parsed `.crbm` indices the open readers hold (default 64 MiB) — the byte
|
|
188
|
-
* half of the gap #1 memory bound, complementing the `coldReaderCacheMax` *count* bound. A wide/dense
|
|
189
|
-
* segment's parsed index can be several MB, so a count-only bound could let the open readers pin ~GBs and
|
|
190
|
-
* blow a small heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index
|
|
191
|
-
* footprint would exceed the ceiling — whichever of the count/byte bounds binds first. Lower it for
|
|
192
|
-
* memory-tight deployments that read across wide segments. Applies only when `cold` is a raw driver.
|
|
193
|
-
*/
|
|
194
|
-
readonly coldReaderCacheMaxBytes?: number;
|
|
195
|
-
/**
|
|
196
|
-
* Resilience: by default every warm/cold call retries **transient** faults (throttling, 5xx, dropped
|
|
197
|
-
* connections) with bounded, jittered exponential backoff (see {@link DEFAULT_RETRY_POLICY}), and OCC
|
|
198
|
-
* conflicts back off between retries. Pass a {@link RetryPolicy} to tune it, or `false` to disable the
|
|
199
|
-
* transient-retry wrappers entirely (e.g. if your injected client already retries). Deterministic errors
|
|
200
|
-
* (`ValidationError`/`IntegrityError`/`WriteConflictError`/…) are never retried by this layer.
|
|
201
|
-
*/
|
|
202
|
-
readonly retry?: RetryPolicy | false;
|
|
203
|
-
/** Backoff schedule between OCC conflict retries; defaults to a small, tight one. */
|
|
204
|
-
readonly occBackoff?: RetryPolicy;
|
|
205
|
-
/** Observability: called before each transient-retry backoff wait. */
|
|
206
|
-
readonly onRetry?: (info: {
|
|
207
|
-
attempt: number;
|
|
208
|
-
delayMs: number;
|
|
209
|
-
err: unknown;
|
|
210
|
-
}) => void;
|
|
211
|
-
/**
|
|
212
|
-
* Observability sink (Phase 5a): receives typed metric events (cold GET/bytes, warm read/write, cache
|
|
213
|
-
* hit/miss, retries, intersection efficiency, op latency). Defaults to a no-op — emission is skipped
|
|
214
|
-
* entirely when unused (near-zero overhead). Any exception the sink throws is swallowed — metrics can
|
|
215
|
-
* never break a read/write.
|
|
216
|
-
*/
|
|
217
|
-
readonly metrics?: IMetricsSink;
|
|
218
|
-
/**
|
|
219
|
-
* Warm read consistency for the READ paths (`has`/`count`/`iterate`/`intersect`). Default `'strong'`
|
|
220
|
-
* (read-your-writes). `'eventual'` trades read-after-write for **~½ the DynamoDB read cost** (a strong read
|
|
221
|
-
* bills 2× RCU) — a good fit for read-heavy, staleness-tolerant workloads (mirrors the cold tier's bounded
|
|
222
|
-
* eventual reads). The compaction/OCC write path is always strongly consistent regardless. No effect on the
|
|
223
|
-
* in-memory/LocalFs drivers (always strong). (Audit gap #9.)
|
|
224
|
-
*/
|
|
225
|
-
readonly warmReadConsistency?: 'strong' | 'eventual';
|
|
226
|
-
/**
|
|
227
|
-
* Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher). Default **4**
|
|
228
|
-
* (`DEFAULT_WRITE_CONCURRENCY`); set it to `1` for strictly serial writes.
|
|
229
|
-
*
|
|
230
|
-
* Distinct chunks are independent OCC rows, so fanning them out cannot make them conflict with each other.
|
|
231
|
-
* The bound exists for the *backend*: a provisioned-capacity store answers a burst by throttling, and 4 stays
|
|
232
|
-
* comfortably inside what the transient-retry path absorbs (measured 8x below the first observed failure).
|
|
233
|
-
* Raise it if your backend is provisioned for the fan-out — on-demand DynamoDB, or a connection pool sized to
|
|
234
|
-
* match. A pool *smaller* than this needs no action: `pg` and `mysql2` both queue rather than reject, so the
|
|
235
|
-
* flusher simply degrades toward serial.
|
|
236
|
-
*
|
|
237
|
-
* As before, a mid-flush failure can leave a **partial result**; `addMany`/`removeMany` are not atomic. What
|
|
238
|
-
* concurrency changes is only how much of the batch may already have landed when the first error surfaces.
|
|
239
|
-
*/
|
|
240
|
-
readonly writeConcurrency?: number;
|
|
241
|
-
/**
|
|
242
|
-
* Hard ceiling on the warm-delta bytes one segment scan may hold resident. Default **64 MiB**.
|
|
243
|
-
*
|
|
244
|
-
* **A memory bound, deliberately separate from `budget`, and still enforced when `budget: false`.** The
|
|
245
|
-
* budget limits *cost* (billable requests); this limits *memory*. They are different axes, and treating the
|
|
246
|
-
* budget as a memory control is what allowed a segment with thousands of warm chunks to materialise ~12 MB
|
|
247
|
-
* before a `maxRequests: 2` budget could refuse it. It is also the only bound available to `intersect`,
|
|
248
|
-
* whose budget is `common keys × operands` — a product a single wide operand can legitimately exceed in row
|
|
249
|
-
* count while remaining entirely within contract.
|
|
250
|
-
*
|
|
251
|
-
* Raise it for genuinely large segments; it exists to keep a modest container alive, not to second-guess you.
|
|
252
|
-
*/
|
|
253
|
-
readonly maxWarmScanBytes?: number;
|
|
254
|
-
/**
|
|
255
|
-
* Per-op **denial-of-wallet** budget (Decision #3 / invariant T3): the max backend requests a
|
|
256
|
-
* single `count`/`iterate`/`intersect`/`subjectReport`/`eraseSubject` may fan out into before it's refused
|
|
257
|
-
* with {@link BudgetExceededError} — so one runaway op can't drive unbounded RCU/GET cost on a shared backend.
|
|
258
|
-
* **On by default, generous** ({@link DEFAULT_BUDGET}: 1,000,000 requests — a normal op never hits it). Tune
|
|
259
|
-
* with `{ maxRequests }`, override per op (on `intersect`/`subjectReport`/`eraseSubject`), or set `false` to
|
|
260
|
-
* disable. The check is O(1) (before fan-out), so the hot path is untouched; per-request bytes are separately
|
|
261
|
-
* size-capped, so bounding requests transitively bounds bytes.
|
|
262
|
-
*/
|
|
263
|
-
readonly budget?: BudgetOption;
|
|
264
|
-
}
|
|
265
|
-
interface SegmentOptions {
|
|
266
|
-
readonly namespace?: string;
|
|
267
|
-
}
|
|
268
|
-
/** A segment reference in a subject report / erasure ledger. */
|
|
269
|
-
interface SubjectSegmentRef {
|
|
270
|
-
readonly segment: string;
|
|
271
|
-
readonly namespace?: string;
|
|
272
|
-
}
|
|
273
|
-
/** Result of {@link CloudRoaring.subjectReport} — the segments an id is a member of (over registered segments). */
|
|
274
|
-
interface SubjectReport {
|
|
275
|
-
readonly id: number;
|
|
276
|
-
/** The registered segments the id is currently a member of. */
|
|
277
|
-
readonly segments: SubjectSegmentRef[];
|
|
278
|
-
/** How many registered segments were scanned (the completeness denominator). */
|
|
279
|
-
readonly scannedSegments: number;
|
|
280
|
-
}
|
|
281
|
-
/** One segment's entry in an erasure ledger (see {@link EraseSubjectResult}). */
|
|
282
|
-
interface SubjectErasureEntry {
|
|
283
|
-
readonly segment: string;
|
|
284
|
-
readonly namespace?: string;
|
|
285
|
-
/** The id was present and a logical `remove` tombstone was written. */
|
|
286
|
-
readonly removed: boolean;
|
|
287
|
-
/**
|
|
288
|
-
* True iff *this call* committed a new generation with the segment's tombstones applied — the bit is
|
|
289
|
-
* physically gone from Cold (assuming no concurrent re-add of the id; see {@link CloudRoaring.eraseSubject}).
|
|
290
|
-
* `false` is **conservative**: the purge may have been deferred (`note:'leased-by-other'`) or already done by
|
|
291
|
-
* a concurrent daemon (`note:'clean'`); the logical removal holds regardless.
|
|
292
|
-
*/
|
|
293
|
-
readonly physicallyPurged: boolean;
|
|
294
|
-
/** The generation that retired the bit (present when `physicallyPurged`). */
|
|
295
|
-
readonly toGen?: number;
|
|
296
|
-
/** Why physical purge wasn't committed by this call (compaction's `reason`, e.g. `'leased-by-other'`, `'clean'`). */
|
|
297
|
-
readonly note?: string;
|
|
298
|
-
}
|
|
299
|
-
/**
|
|
300
|
-
* The erasure ledger returned by {@link CloudRoaring.eraseSubject} — your proof-of-deletion artifact. It is a
|
|
301
|
-
* return value only (no library-side persistence): persist it / route it to your audit sink as you see fit.
|
|
302
|
-
*/
|
|
303
|
-
interface EraseSubjectResult {
|
|
304
|
-
readonly id: number;
|
|
305
|
-
/** Per-segment records for the segments the id was erased from (absent segments are not listed). */
|
|
306
|
-
readonly erasedFrom: SubjectErasureEntry[];
|
|
307
|
-
/** How many registered segments were scanned. */
|
|
308
|
-
readonly scannedSegments: number;
|
|
309
|
-
}
|
|
310
|
-
declare class CloudRoaring {
|
|
311
|
-
private readonly engine;
|
|
312
|
-
private readonly clock;
|
|
313
|
-
private readonly metrics;
|
|
314
|
-
private readonly coldDriver;
|
|
315
|
-
private readonly warmDriver;
|
|
316
|
-
private readonly registry;
|
|
317
|
-
private readonly keystore;
|
|
318
|
-
private readonly requireEncryption;
|
|
319
|
-
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
320
|
-
private readonly budget;
|
|
321
|
-
constructor(options: CloudRoaringOptions);
|
|
322
|
-
/**
|
|
323
|
-
* Build {@link CompactionDeps} from the store's own drivers, for the in-process lifecycle helpers
|
|
324
|
-
* ({@link compact} / {@link eraseSubject}). Requires the store to have been constructed with a **raw cold
|
|
325
|
-
* driver** (a pre-built `ColdChunkSource` has no underlying `IColdDriver` to compact through) and a `registry`
|
|
326
|
-
* (the authoritative `currentGen` pointer compaction swaps). Out-of-process callers use the `compactSegment`
|
|
327
|
-
* free function with explicit deps.
|
|
328
|
-
*/
|
|
329
|
-
private compactionDeps;
|
|
330
|
-
/** Get a handle to a segment. Validates the name/namespace grammar (finding S2). */
|
|
331
|
-
segment(name: string, options?: SegmentOptions): Segment;
|
|
332
|
-
/**
|
|
333
|
-
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?** (Phase 6b.)
|
|
334
|
-
*
|
|
335
|
-
* Enumerates the **registered** segments (via the store's own `registry`) and does a tier-merging `has(id)` on
|
|
336
|
-
* each — no drivers to re-pass. Complete only over registered segments (register your segments if you claim
|
|
337
|
-
* SAR support). There is deliberately **no `id → segments` reverse index** — that would tax every write for a
|
|
338
|
-
* rare request; this admin scan is `O(registered segments)` and touches no hot path.
|
|
339
|
-
* Requires a `registry` in the store config
|
|
340
|
-
* (throws {@link UnsupportedError} otherwise).
|
|
341
|
-
*/
|
|
342
|
-
subjectReport(id: number, options?: {
|
|
343
|
-
namespace?: string;
|
|
344
|
-
allNamespaces?: boolean;
|
|
345
|
-
concurrency?: number;
|
|
346
|
-
budget?: BudgetOption;
|
|
347
|
-
}): Promise<SubjectReport>;
|
|
348
|
-
/**
|
|
349
|
-
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with a physical-deletion
|
|
350
|
-
* guarantee.** (Phase 6b.)
|
|
351
|
-
*
|
|
352
|
-
* For each **registered** segment the id is a member of: writes a logical `remove` tombstone (immediate) and
|
|
353
|
-
* then **force-compacts that segment now** — folding the tombstone into a fresh immutable generation so the
|
|
354
|
-
* bit is *physically* gone from Cold on return, even for an otherwise-idle/archival segment that organic
|
|
355
|
-
* compaction would never touch (the P13 fix). The returned per-segment record is your **erasure ledger** —
|
|
356
|
-
* persist it / route it to your audit sink as the proof of deletion; the physical-purge proof is
|
|
357
|
-
* compaction's own VERIFY step (a re-`has()` here would be unsound — a pinned cold source can still read the
|
|
358
|
-
* old generation; take a fresh generation view after erasing, exactly as with the compaction daemon).
|
|
359
|
-
*
|
|
360
|
-
* Uses the store's **own** drivers (raw cold + warm + registry), so the old integrator obligation to wire a
|
|
361
|
-
* matching `CompactionDeps` is gone — the `remove()` and the force-compaction provably run over the same tiers.
|
|
362
|
-
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise;
|
|
363
|
-
* a pre-built `ColdChunkSource` store has no `IColdDriver` to compact through — use the `compactSegment` free
|
|
364
|
-
* function there). **One contract remains** (an integrator obligation the library cannot check): **do not
|
|
365
|
-
* concurrently re-add the same id while erasing it.** A normal `add(id)` landing between the `remove` and the
|
|
366
|
-
* compaction clears the tombstone and folds the id back into the new generation
|
|
367
|
-
* (`effective = (cold ∪ adds) \ removes`); `physicallyPurged` attests only that compaction committed a
|
|
368
|
-
* generation with the tombstones applied at pin time — quiesce writes for the subject during erasure.
|
|
369
|
-
*
|
|
370
|
-
* **A `physicallyPurged:false` entry means the logical removal holds but the physical purge didn't run this
|
|
371
|
-
* call** — either a daemon held a live lease (`note:'leased-by-other'`; the daemon finishes it), or the
|
|
372
|
-
* force-compaction hit an isolated fault (`note:'error: …'`; per-segment faults are caught so one segment
|
|
373
|
-
* can't discard the whole ledger). `physicallyPurged:false` is **conservative** (a concurrent daemon may
|
|
374
|
-
* already have purged the bit — `'clean'`). **Recovery:** follow up any `physicallyPurged:false` entry with
|
|
375
|
-
* `store.compact(ref)` (or let the daemon do it) — **re-running `eraseSubject` will NOT re-purge it**, because
|
|
376
|
-
* `has()` now reads false (the tombstone) so the segment is skipped as a non-member. The physical-purge proof
|
|
377
|
-
* is compaction's own VERIFY step (a re-`has()` here would be unsound: a pinned cold source can still read the
|
|
378
|
-
* old generation — take a fresh generation view after erasing, as with the daemon). Admin-only path;
|
|
379
|
-
* `O(registered segments)`, no hot-path cost. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
380
|
-
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
381
|
-
* is the `destroySegment`/`eraseNamespace` free functions.
|
|
382
|
-
*/
|
|
383
|
-
eraseSubject(id: number, options: {
|
|
384
|
-
owner: string;
|
|
385
|
-
namespace?: string;
|
|
386
|
-
allNamespaces?: boolean;
|
|
387
|
-
audit?: IAuditSink;
|
|
388
|
-
concurrency?: number;
|
|
389
|
-
budget?: BudgetOption;
|
|
390
|
-
}): Promise<EraseSubjectResult>;
|
|
391
|
-
/**
|
|
392
|
-
* **Compact one segment now**, in-process, using the store's own drivers — a convenience over the
|
|
393
|
-
* {@link compactSegment} free function. Merges `(cold ∪ adds) \ removes` into a fresh immutable generation,
|
|
394
|
-
* swaps `currentGen`, then version-fenced-purges the archived Warm rows. A no-op (`compacted:false`) when the
|
|
395
|
-
* segment has no dirty rows or a daemon holds a live lease — never throws on the normal contention/lease paths.
|
|
396
|
-
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise);
|
|
397
|
-
* like the daemon, it uses the raw drivers directly (a transient fault surfaces to you rather than retrying
|
|
398
|
-
* under the hood). **Not for the request/hot path** — it reads and rewrites a whole Cold generation (streaming,
|
|
399
|
-
* constant-memory, but full-segment I/O). Use the `compact-segments` daemon (`runCompactionCycle`) for routine
|
|
400
|
-
* background compaction; reach for `store.compact()` for occasional/manual one-shots — a maintenance endpoint,
|
|
401
|
-
* a small daemon-less deployment, or right after a targeted `remove`.
|
|
402
|
-
*/
|
|
403
|
-
compact(ref: SegmentRef, options: CompactionOptions): Promise<CompactionResult>;
|
|
404
|
-
/**
|
|
405
|
-
* **Dispose of a segment — tombstone it, delete its Warm rows, delete its Cold objects.** Irreversible.
|
|
406
|
-
*
|
|
407
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
|
|
408
|
-
* backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
|
|
409
|
-
* removes the storage and works on a cleartext segment. On an encrypted segment it does both.
|
|
410
|
-
*
|
|
411
|
-
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
412
|
-
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
413
|
-
* variable twice.
|
|
414
|
-
*
|
|
415
|
-
* ```ts
|
|
416
|
-
* for (const day of expiredDays) {
|
|
417
|
-
* // A colon is NOT legal in a name — the family goes in the namespace, the date in the segment.
|
|
418
|
-
* const ref = { namespace: 'active-daily', segment: day };
|
|
419
|
-
* await store.dropSegment(ref, { confirmSegment: ref.segment });
|
|
420
|
-
* }
|
|
421
|
-
* ```
|
|
422
|
-
*
|
|
423
|
-
* **Omitting the namespace addresses a different segment and is a silent no-op** — you get
|
|
424
|
-
* `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
|
|
425
|
-
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
426
|
-
*
|
|
427
|
-
* **Works on an accumulator too.** A segment you created by writing to it — never bulk-loaded, never compacted,
|
|
428
|
-
* so it has no registry row and no Cold objects — is retired by deleting its Warm rows alone. That reports
|
|
429
|
-
* `{ dropped: true, reason: 'warm-only' }`: there is no tombstone to write, and deliberately none is written,
|
|
430
|
-
* because a `destroyed` row per retired daily bucket would be registry litter. The data really is gone.
|
|
431
|
-
*
|
|
432
|
-
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
433
|
-
* reclaimed and the drop should be re-run. A compaction that was already in flight when the tombstone landed
|
|
434
|
-
* still finishes staging one more object, so a single sweep can miss it — this call re-sweeps and then reports
|
|
435
|
-
* whatever it still could not remove rather than returning a result that looks like a clean drop.
|
|
436
|
-
*
|
|
437
|
-
* Reads become empty within `coldGenTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
438
|
-
* segment may answer from its cached generation + hot chunks until that window lapses. A reader that never
|
|
439
|
-
* touched it sees empty at once. **That bound needs a clock and `coldGenTtlMs > 0`** — a store built without a
|
|
440
|
-
* clock, or with `coldGenTtlMs: 0` ("pin forever"), holds its resolved snapshot for its own lifetime and can
|
|
441
|
-
* keep answering `true` for a dropped segment indefinitely; restart it.
|
|
442
|
-
*
|
|
443
|
-
* Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
|
|
444
|
-
* because it has to enumerate and delete generations — a pre-built `ColdChunkSource` only reads.
|
|
445
|
-
*/
|
|
446
|
-
dropSegment(ref: SegmentRef, options: {
|
|
447
|
-
confirmSegment: string;
|
|
448
|
-
dryRun?: boolean;
|
|
449
|
-
audit?: IAuditSink;
|
|
450
|
-
}): Promise<DropResult>;
|
|
451
|
-
/**
|
|
452
|
-
* **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
|
|
453
|
-
* segment's `currentGen` actually has its `.crbm` present in Cold — catching a **torn restore** where the
|
|
454
|
-
* registry (`currentGen`) came back ahead of the object store, so a pointer references a generation that
|
|
455
|
-
* isn't there (reads would then throw). Read-only, bounded fan-out; run it at startup after a restore.
|
|
456
|
-
* Returns `{ checked, inconsistent }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to
|
|
457
|
-
* recover (restore the object store, or roll the registry back to a coherent point). Needs the store built
|
|
458
|
-
* with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise). `destroyed`
|
|
459
|
-
* (crypto-shredded) segments are skipped. Pair it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
460
|
-
*/
|
|
461
|
-
checkConsistency(options?: {
|
|
462
|
-
namespace?: string;
|
|
463
|
-
concurrency?: number;
|
|
464
|
-
}): Promise<ConsistencyReport>;
|
|
465
|
-
/**
|
|
466
|
-
* **Export ("eject") every registered segment's current effective set** through the injected `sink`, using
|
|
467
|
-
* only public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's
|
|
468
|
-
* "Your data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
|
|
469
|
-
* (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
|
|
470
|
-
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
471
|
-
* enumeration and the read path provably share one registry; an all-warm segment not yet in the registry can be
|
|
472
|
-
* named via `options.candidates`. Encrypted segments are decrypted transparently **iff** this store was wired
|
|
473
|
-
* with their keystore — the export is therefore **cleartext**; protect it. Crypto-shredded segments are skipped.
|
|
474
|
-
* A segment that can't be read is isolated into the manifest's `failed[]` (the run continues), so "a manifest
|
|
475
|
-
* exists" means the run finished — check `failed`. The `export-segments` CLI wraps this with a filesystem sink.
|
|
476
|
-
*/
|
|
477
|
-
exportSegments(sink: ExportSink, options?: ExportOptions): Promise<ExportManifest>;
|
|
478
|
-
/**
|
|
479
|
-
* Planning cost estimate (Phase 5b) — pure, no instance/data needed: sizing, sales, what-if. For a real,
|
|
480
|
-
* grounded report from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
481
|
-
*/
|
|
482
|
-
static estimateCost(input: EstimateInput): CostReport;
|
|
483
|
-
}
|
|
484
|
-
/**
|
|
485
|
-
* A handle bound to one segment — the read/write ops.
|
|
486
|
-
*
|
|
487
|
-
* **IDs must be integers in `[0, 2^32)`** (dense 32-bit). A non-integer / negative / out-of-range id
|
|
488
|
-
* throws {@link ValidationError}.
|
|
489
|
-
*
|
|
490
|
-
* **`addMany`/`removeMany`/`intersectInto`/`unionInto`/`andNotInto` are not atomic across chunks**: ids are grouped by chunk and
|
|
491
|
-
* applied one chunk at a time, so if a later chunk fails (e.g. {@link WriteConflictError} after retries)
|
|
492
|
-
* earlier chunks are already applied. Within a single chunk the update is atomic.
|
|
493
|
-
*/
|
|
494
|
-
/** Options common to every chunk-aligned combine (`intersect` / `union` / `andNot`). */
|
|
495
|
-
interface BaseCombineOptions {
|
|
496
|
-
/** Max chunk keys resolved concurrently — bounds the Cold footprint. A positive integer. */
|
|
497
|
-
readonly concurrency?: number;
|
|
498
|
-
/** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
|
|
499
|
-
readonly budget?: BudgetOption;
|
|
500
|
-
}
|
|
501
|
-
/**
|
|
502
|
-
* A combine that can also subtract.
|
|
503
|
-
*
|
|
504
|
-
* `exclude` is why suppression does not need an intermediate segment: applied here it folds into the same
|
|
505
|
-
* chunk-aligned pass, and each exclude is read **only at the keys that survived** — so a large global opt-out
|
|
506
|
-
* list costs reads proportional to the audience, not to itself.
|
|
507
|
-
*/
|
|
508
|
-
interface CombineOptions extends BaseCombineOptions {
|
|
509
|
-
/** Segments whose ids are subtracted from the result. */
|
|
510
|
-
readonly exclude?: Segment[];
|
|
511
|
-
}
|
|
512
|
-
/** {@link CombineOptions} plus the write batching used by the `*Into` variants. */
|
|
513
|
-
interface CombineIntoOptions extends CombineOptions {
|
|
514
|
-
/** Ids buffered per `addMany` while materializing into `dest`. */
|
|
515
|
-
readonly batchSize?: number;
|
|
516
|
-
}
|
|
517
|
-
declare class Segment {
|
|
518
|
-
private readonly engine;
|
|
519
|
-
private readonly ref;
|
|
520
|
-
private readonly clock;
|
|
521
|
-
private readonly metrics;
|
|
522
|
-
private readonly metricsOn;
|
|
523
|
-
constructor(engine: SegmentEngine, ref: SegmentRef, clock: Clock, metrics: IMetricsSink);
|
|
524
|
-
/**
|
|
525
|
-
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
526
|
-
* entirely when no sink is wired, so the default path pays nothing.
|
|
527
|
-
*/
|
|
528
|
-
private timed;
|
|
529
|
-
/** Add an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
530
|
-
add(id: number): Promise<void>;
|
|
531
|
-
/**
|
|
532
|
-
* Add many ids, from a **sync or async** iterable. Not atomic across chunks — see the class note.
|
|
533
|
-
*
|
|
534
|
-
* An async source means a database cursor streams straight in — `addMany(athenaCursor())` rather than
|
|
535
|
-
* hand-batching `page → addMany(page)`. Ids are grouped by chunk and each chunk is written **exactly once**,
|
|
536
|
-
* however long the stream: pending ids are held compressed, not buffered-and-flushed, so a long stream costs
|
|
537
|
-
* no more backend writes than a short one. Peak memory is the roaring representation of the ids you pass.
|
|
538
|
-
*
|
|
539
|
-
* **For a very large set, reach for `bulkLoadCrbmGeneration` instead.** `addMany` expresses a *delta* and
|
|
540
|
-
* writes one warm row per touched chunk — around 61,000 of them for a set spread across the whole id space.
|
|
541
|
-
* Bulk-load *replaces* the segment with one immutable Cold object. See the guide's cost comparison; picking
|
|
542
|
-
* the wrong one of these is the most expensive mistake available in this library.
|
|
543
|
-
*/
|
|
544
|
-
addMany(ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
545
|
-
/** Remove an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
546
|
-
remove(id: number): Promise<void>;
|
|
547
|
-
/** Remove many ids, from a **sync or async** iterable. Not atomic across chunks — see {@link addMany}. */
|
|
548
|
-
removeMany(ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
|
|
549
|
-
/**
|
|
550
|
-
* **Claim ids atomically — add them, and get back only the ones that were not already there.**
|
|
551
|
-
*
|
|
552
|
-
* The durable analogue of Redis `SETBIT` returning the prior bit, which is what an exactly-once *"have I
|
|
553
|
-
* already sent to / already processed this id?"* check needs. `has()` then `add()` cannot give you this: two
|
|
554
|
-
* workers can both read absent and both proceed.
|
|
555
|
-
*
|
|
556
|
-
* ```ts
|
|
557
|
-
* const sent = store.segment(day, { namespace: 'sent-daily' });
|
|
558
|
-
* const toSend = await sent.claimMany(candidateIds); // only the ids this worker won
|
|
559
|
-
* for (const id of toSend) enqueue(id); // ...so nobody else will send them
|
|
560
|
-
* ```
|
|
561
|
-
*
|
|
562
|
-
* **Pass a batch, not one id.** A Warm write rewrites a whole 64K-id chunk bitmap, so claiming ids one at a
|
|
563
|
-
* time is the single most expensive way to use this library — measured at ~5,000 writes and ~23 MB for 5,000
|
|
564
|
-
* ids, against 1 write and ~8 KB for the same ids in one call. This method does one OCC read-modify-write per
|
|
565
|
-
* distinct chunk, giving you Redis's *semantics* without Redis's per-id *cost shape*.
|
|
566
|
-
*
|
|
567
|
-
* **Exactly-once holds per id**, which is the guarantee that matters: each id lives in one chunk, a chunk is
|
|
568
|
-
* one OCC row, so exactly one concurrent claimer sees any given id as new. Like `addMany` it is **not** atomic
|
|
569
|
-
* across chunks — a mid-flight failure can leave some chunks claimed — but re-running is safe, because
|
|
570
|
-
* already-claimed ids simply come back as not-new.
|
|
571
|
-
*
|
|
572
|
-
* Order of the returned ids is unspecified.
|
|
573
|
-
*/
|
|
574
|
-
claimMany(ids: Iterable<number>): Promise<number[]>;
|
|
575
|
-
has(id: number): Promise<boolean>;
|
|
576
|
-
count(): Promise<number>;
|
|
577
|
-
iterate(): AsyncIterable<number>;
|
|
578
|
-
/**
|
|
579
|
-
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
580
|
-
* Fetches only the Cold chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
581
|
-
* and is never downloaded), streaming under a bounded in-flight window — so the Cold footprint stays small
|
|
582
|
-
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
583
|
-
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. (Warm state is read
|
|
584
|
-
* up front, so total memory also carries each operand's Warm size — negligible under Topology-A.) Pass
|
|
585
|
-
* `budget` to override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
586
|
-
*/
|
|
587
|
-
/**
|
|
588
|
-
* Map the facade's `Segment` handles in `exclude` down to the plain refs `core` takes. A method rather than
|
|
589
|
-
* a module function because `ref` is class-private — the encapsulation is worth more than the free function.
|
|
590
|
-
*/
|
|
591
|
-
private refsIn;
|
|
592
|
-
intersect(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
|
|
593
|
-
/**
|
|
594
|
-
* Materialize `this ∩ others…` (minus `exclude`) **into** `dest` (added, not replaced). Streaming +
|
|
595
|
-
* bounded-memory, but **not atomic** across chunks — see {@link addMany}.
|
|
596
|
-
*/
|
|
597
|
-
intersectInto(dest: Segment, others: Segment[], options?: CombineIntoOptions): Promise<void>;
|
|
598
|
-
/**
|
|
599
|
-
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
600
|
-
*
|
|
601
|
-
* **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
|
|
602
|
-
* here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
|
|
603
|
-
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
604
|
-
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
605
|
-
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
606
|
-
* segment once (`unionInto`, or a bulk-load rebuild) is the cheaper shape.
|
|
607
|
-
*/
|
|
608
|
-
union(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
|
|
609
|
-
/** Materialize `this ∪ others…` (minus `exclude`) **into** `dest`. **Not atomic** — see {@link addMany}. */
|
|
610
|
-
unionInto(dest: Segment, others: Segment[], options?: CombineIntoOptions): Promise<void>;
|
|
611
|
-
/**
|
|
612
|
-
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
613
|
-
*
|
|
614
|
-
* Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
|
|
615
|
-
* overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
|
|
616
|
-
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
617
|
-
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
618
|
-
*
|
|
619
|
-
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
620
|
-
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
621
|
-
*/
|
|
622
|
-
andNot(excludes: Segment[], options?: BaseCombineOptions): AsyncIterable<number>;
|
|
623
|
-
/** Materialize `this \ (excludes…)` **into** `dest`. **Not atomic** — see {@link addMany}. */
|
|
624
|
-
andNotInto(dest: Segment, excludes: Segment[], options?: BaseCombineOptions & {
|
|
625
|
-
batchSize?: number;
|
|
626
|
-
}): Promise<void>;
|
|
627
|
-
/**
|
|
628
|
-
* Grounded cost report for this segment (Phase 5b): storage cost from its **real** `.crbm` size (exact,
|
|
629
|
-
* no payload reads); request cost from the supplied `workload` rates. A segment with no Cold generation
|
|
630
|
-
* reports zero storage. See {@link CostReport} — it always includes a verdict (incl. the lose-zone).
|
|
631
|
-
*/
|
|
632
|
-
costReport(options?: {
|
|
633
|
-
pricing?: PricingProfile;
|
|
634
|
-
workload?: Workload;
|
|
635
|
-
topology?: Topology;
|
|
636
|
-
}): Promise<CostReport>;
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
/** Package version marker. Kept in sync with package.json at release. */
|
|
640
|
-
declare const VERSION = "0.8.2";
|
|
641
|
-
|
|
642
|
-
export { type BaseCombineOptions, CloudRoaring, type CloudRoaringOptions, type CombineIntoOptions, type CombineOptions, type EraseSubjectResult, SafeBitmap, Segment, type SegmentOptions, type SubjectErasureEntry, type SubjectReport, type SubjectSegmentRef, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };
|
package/dist/mongodb/index.cjs
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var mongodb = require('@cloudbitmaps/core/mongodb');
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
Object.keys(mongodb).forEach(function (k) {
|
|
8
|
-
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
9
|
-
enumerable: true,
|
|
10
|
-
get: function () { return mongodb[k]; }
|
|
11
|
-
});
|
|
12
|
-
});
|
|
13
|
-
//# sourceMappingURL=index.cjs.map
|
|
14
|
-
//# sourceMappingURL=index.cjs.map
|