@cloudbitmaps/roaring 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/PRIVACY.md +198 -72
- package/README.md +121 -57
- package/dist/bin/export-segments.d.ts +21 -0
- package/dist/bin/export-segments.js +1508 -30
- package/dist/bin/export-segments.js.map +7 -1
- package/dist/codec-bound.d.ts +38 -0
- package/dist/index.d.ts +805 -447
- package/dist/index.js +911 -325
- package/dist/index.js.map +7 -1
- package/dist/moved-options.d.ts +14 -0
- package/dist/portable/decode.d.ts +54 -0
- package/dist/roaring-codec.d.ts +61 -0
- package/dist/system-clock.d.ts +40 -0
- package/dist/testing/conformance.d.ts +34 -0
- package/dist/testing/fuzz-codec.d.ts +9 -0
- package/package.json +11 -116
- package/dist/azure/index.cjs +0 -14
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -9
- package/dist/azure/index.d.ts +0 -9
- package/dist/azure/index.js +0 -3
- package/dist/azure/index.js.map +0 -1
- package/dist/bin/compact-segments.js +0 -201
- package/dist/bin/compact-segments.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -14
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -1
- package/dist/cassandra/index.d.ts +0 -1
- package/dist/cassandra/index.js +0 -3
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-D6LIJSGF.js +0 -906
- package/dist/chunk-D6LIJSGF.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -14
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -1
- package/dist/dynamodb/index.d.ts +0 -1
- package/dist/dynamodb/index.js +0 -3
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -14
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -1
- package/dist/gcs/index.d.ts +0 -1
- package/dist/gcs/index.js +0 -3
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -894
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -734
- package/dist/mongodb/index.cjs +0 -14
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -1
- package/dist/mongodb/index.d.ts +0 -1
- package/dist/mongodb/index.js +0 -3
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -14
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -1
- package/dist/mysql/index.d.ts +0 -1
- package/dist/mysql/index.js +0 -3
- package/dist/mysql/index.js.map +0 -1
- package/dist/postgres/index.cjs +0 -14
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -1
- package/dist/postgres/index.d.ts +0 -1
- package/dist/postgres/index.js +0 -3
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -14
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -1
- package/dist/redis/index.d.ts +0 -1
- package/dist/redis/index.js +0 -3
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -14
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -1
- package/dist/s3/index.d.ts +0 -1
- package/dist/s3/index.js +0 -3
- package/dist/s3/index.js.map +0 -1
package/dist/index.cjs
DELETED
|
@@ -1,894 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var core = require('@cloudbitmaps/core');
|
|
4
|
-
var roaring = require('roaring');
|
|
5
|
-
|
|
6
|
-
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
7
|
-
|
|
8
|
-
var roaring__default = /*#__PURE__*/_interopDefault(roaring);
|
|
9
|
-
|
|
10
|
-
// src/index.ts
|
|
11
|
-
var { RoaringBitmap32, SerializationFormat, DeserializationFormat } = roaring__default.default;
|
|
12
|
-
var SafeBitmap = class _SafeBitmap {
|
|
13
|
-
bitmap;
|
|
14
|
-
constructor(bitmap) {
|
|
15
|
-
this.bitmap = bitmap;
|
|
16
|
-
}
|
|
17
|
-
static empty() {
|
|
18
|
-
return new _SafeBitmap(new RoaringBitmap32());
|
|
19
|
-
}
|
|
20
|
-
static fromValues(values) {
|
|
21
|
-
return new _SafeBitmap(new RoaringBitmap32(values));
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* S1: validate size, then deserialize with the **portable** (validated) format.
|
|
25
|
-
* Throws `IntegrityError` if the input exceeds `maxBytes` or fails to decode — the
|
|
26
|
-
* native addon is never handed unbounded or unsafe-format input.
|
|
27
|
-
*/
|
|
28
|
-
static safeDeserialize(bytes, maxBytes) {
|
|
29
|
-
if (bytes.length > maxBytes) {
|
|
30
|
-
throw new core.IntegrityError(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
31
|
-
}
|
|
32
|
-
try {
|
|
33
|
-
return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
|
|
34
|
-
} catch (err) {
|
|
35
|
-
throw new core.IntegrityError(`failed to deserialize bitmap: ${err.message}`);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
serialize() {
|
|
39
|
-
return this.bitmap.serialize(SerializationFormat.portable);
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
43
|
-
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
44
|
-
*
|
|
45
|
-
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
46
|
-
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
47
|
-
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
48
|
-
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
49
|
-
* whichever encoding is smaller per container.
|
|
50
|
-
*
|
|
51
|
-
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
52
|
-
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
53
|
-
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
54
|
-
* columns; don't carry a figure between them without checking which denominator it had.
|
|
55
|
-
*
|
|
56
|
-
* Called by the engine only when writing an immutable cold generation — see {@link CodecBitmap.optimize} for
|
|
57
|
-
* why the per-operation warm path deliberately does not.
|
|
58
|
-
*/
|
|
59
|
-
optimize() {
|
|
60
|
-
this.bitmap.runOptimize();
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the container index, so the engine's
|
|
64
|
-
* per-chunk range assertion costs one call per chunk rather than a walk per id.
|
|
65
|
-
*/
|
|
66
|
-
maximum() {
|
|
67
|
-
return this.bitmap.isEmpty ? void 0 : this.bitmap.maximum();
|
|
68
|
-
}
|
|
69
|
-
add(value) {
|
|
70
|
-
this.bitmap.add(value);
|
|
71
|
-
}
|
|
72
|
-
addMany(values) {
|
|
73
|
-
this.bitmap.addMany(values);
|
|
74
|
-
}
|
|
75
|
-
remove(value) {
|
|
76
|
-
this.bitmap.remove(value);
|
|
77
|
-
}
|
|
78
|
-
removeMany(values) {
|
|
79
|
-
this.bitmap.removeMany(values);
|
|
80
|
-
}
|
|
81
|
-
has(value) {
|
|
82
|
-
return this.bitmap.has(value);
|
|
83
|
-
}
|
|
84
|
-
get size() {
|
|
85
|
-
return this.bitmap.size;
|
|
86
|
-
}
|
|
87
|
-
get isEmpty() {
|
|
88
|
-
return this.bitmap.isEmpty;
|
|
89
|
-
}
|
|
90
|
-
clone() {
|
|
91
|
-
return new _SafeBitmap(this.bitmap.clone());
|
|
92
|
-
}
|
|
93
|
-
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
94
|
-
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
95
|
-
// `bitmap` (accessible on same-class instances) for the native op.
|
|
96
|
-
/** In-place union: `this = this ∪ other`. */
|
|
97
|
-
orInPlace(other) {
|
|
98
|
-
this.bitmap.orInPlace(other.bitmap);
|
|
99
|
-
}
|
|
100
|
-
/** In-place difference: `this = this \ other`. */
|
|
101
|
-
andNotInPlace(other) {
|
|
102
|
-
this.bitmap.andNotInPlace(other.bitmap);
|
|
103
|
-
}
|
|
104
|
-
/** In-place intersection: `this = this ∩ other`. */
|
|
105
|
-
andInPlace(other) {
|
|
106
|
-
this.bitmap.andInPlace(other.bitmap);
|
|
107
|
-
}
|
|
108
|
-
/** Ascending iterator over the set values. */
|
|
109
|
-
[Symbol.iterator]() {
|
|
110
|
-
return this.bitmap[Symbol.iterator]();
|
|
111
|
-
}
|
|
112
|
-
toArray() {
|
|
113
|
-
return this.bitmap.toArray();
|
|
114
|
-
}
|
|
115
|
-
};
|
|
116
|
-
var roaringCodec = {
|
|
117
|
-
empty: () => SafeBitmap.empty(),
|
|
118
|
-
fromValues: (values) => SafeBitmap.fromValues(values),
|
|
119
|
-
safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
|
|
120
|
-
};
|
|
121
|
-
|
|
122
|
-
// src/system-clock.ts
|
|
123
|
-
var SystemClock = class {
|
|
124
|
-
now() {
|
|
125
|
-
return Date.now();
|
|
126
|
-
}
|
|
127
|
-
sleep(ms) {
|
|
128
|
-
if (ms <= 0) return Promise.resolve();
|
|
129
|
-
return new Promise((resolve) => {
|
|
130
|
-
setTimeout(resolve, ms);
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
|
|
135
|
-
*
|
|
136
|
-
* A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
|
|
137
|
-
* co-resident HTTP server keeps answering. Measured on the **synthetic yield-primitive benchmark** — a
|
|
138
|
-
* 61,035-iteration loop yielding every 1,024, isolating the primitive rather than timing a real load (the
|
|
139
|
-
* end-to-end figures are 450 ms → 19 ms; see `cooperative.ts`, which lists all three experiments):
|
|
140
|
-
*
|
|
141
|
-
* ```text
|
|
142
|
-
* no yield 568 ms wall 568.0 ms worst event-loop gap
|
|
143
|
-
* sleep(0) 555 ms wall 555.2 ms ← a microtask. Yields NOTHING.
|
|
144
|
-
* setImmediate 569 ms wall 13.8 ms ← free, and 41x less starvation
|
|
145
|
-
* setTimeout(1) 625 ms wall 15.0 ms ← +10% wall for no extra relief
|
|
146
|
-
* ```
|
|
147
|
-
*
|
|
148
|
-
* `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
|
|
149
|
-
* before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
|
|
150
|
-
* callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
|
|
151
|
-
* server wants — inbound requests are serviced before the loop resumes chewing.
|
|
152
|
-
*
|
|
153
|
-
* A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
|
|
154
|
-
* "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
|
|
155
|
-
*/
|
|
156
|
-
yieldNow() {
|
|
157
|
-
return new Promise((resolve) => {
|
|
158
|
-
setImmediate(resolve);
|
|
159
|
-
});
|
|
160
|
-
}
|
|
161
|
-
};
|
|
162
|
-
var bulkLoadCrbmGeneration = (driver, key, ids, options = {}) => core.bulkLoadCrbmGeneration(driver, key, ids, {
|
|
163
|
-
...options,
|
|
164
|
-
codec: options.codec ?? roaringCodec,
|
|
165
|
-
clock: options.clock ?? new SystemClock()
|
|
166
|
-
});
|
|
167
|
-
var compactSegment = (ref, deps, options) => core.compactSegment(ref, { ...deps, codec: deps.codec ?? roaringCodec }, options);
|
|
168
|
-
var runCompactionCycle = (deps, options) => core.runCompactionCycle({ ...deps, codec: deps.codec ?? roaringCodec }, options);
|
|
169
|
-
var runExport = (reader, registry, sink, options = {}) => core.runExport(reader, registry, sink, { ...options, codec: options.codec ?? roaringCodec });
|
|
170
|
-
|
|
171
|
-
// src/index.ts
|
|
172
|
-
var SystemRng = class {
|
|
173
|
-
next() {
|
|
174
|
-
return Math.random();
|
|
175
|
-
}
|
|
176
|
-
};
|
|
177
|
-
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
178
|
-
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
179
|
-
function validateConcurrency(concurrency) {
|
|
180
|
-
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
181
|
-
throw new core.ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
function requireScope(options, op) {
|
|
185
|
-
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
186
|
-
throw new core.ValidationError(
|
|
187
|
-
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
188
|
-
);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
function resolveColdSource(options, clock) {
|
|
192
|
-
const cold = options.cold;
|
|
193
|
-
if (cold === null || typeof cold !== "object") {
|
|
194
|
-
throw new core.ValidationError("`cold` must be an IColdDriver or a ColdChunkSource");
|
|
195
|
-
}
|
|
196
|
-
const hasGetChunk = typeof cold.getChunk === "function";
|
|
197
|
-
const hasPutImmutable = typeof cold.putImmutable === "function";
|
|
198
|
-
if (hasGetChunk && hasPutImmutable) {
|
|
199
|
-
throw new core.ValidationError(
|
|
200
|
-
"`cold` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass an IColdDriver or a ColdChunkSource, not a hybrid"
|
|
201
|
-
);
|
|
202
|
-
}
|
|
203
|
-
if (!hasGetChunk && !hasPutImmutable) {
|
|
204
|
-
throw new core.ValidationError("`cold` must be an IColdDriver or a ColdChunkSource");
|
|
205
|
-
}
|
|
206
|
-
if (hasGetChunk) {
|
|
207
|
-
if (options.registry !== void 0 || options.keystore !== void 0 || options.requireEncryption === true) {
|
|
208
|
-
throw new core.ValidationError(
|
|
209
|
-
"registry/keystore/requireEncryption apply only when `cold` is a raw IColdDriver; configure them on the ColdChunkSource you passed instead"
|
|
210
|
-
);
|
|
211
|
-
}
|
|
212
|
-
return { source: cold, driver: void 0 };
|
|
213
|
-
}
|
|
214
|
-
const driver = cold;
|
|
215
|
-
return {
|
|
216
|
-
source: new core.CrbmColdChunkSource(driver, {
|
|
217
|
-
registry: options.registry,
|
|
218
|
-
keystore: options.keystore,
|
|
219
|
-
requireEncryption: options.requireEncryption,
|
|
220
|
-
clock,
|
|
221
|
-
currentGenTtlMs: options.coldGenTtlMs,
|
|
222
|
-
maxOpenSegments: options.coldReaderCacheMax,
|
|
223
|
-
maxOpenIndexBytes: options.coldReaderCacheMaxBytes
|
|
224
|
-
}),
|
|
225
|
-
driver
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
var CloudRoaring = class {
|
|
229
|
-
engine;
|
|
230
|
-
clock;
|
|
231
|
-
metrics;
|
|
232
|
-
// The store's own drivers, kept so the in-process lifecycle helpers (`compact`/`eraseSubject`/`subjectReport`)
|
|
233
|
-
// reuse them instead of making you re-pass a CompactionDeps. `coldDriver` is set only when `cold` was a raw
|
|
234
|
-
// IColdDriver (a pre-built ColdChunkSource has no underlying driver to compact through).
|
|
235
|
-
coldDriver;
|
|
236
|
-
warmDriver;
|
|
237
|
-
registry;
|
|
238
|
-
keystore;
|
|
239
|
-
requireEncryption;
|
|
240
|
-
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
241
|
-
budget;
|
|
242
|
-
constructor(options) {
|
|
243
|
-
const clock = options.clock ?? new SystemClock();
|
|
244
|
-
const rng = options.rng ?? new SystemRng();
|
|
245
|
-
const metrics = core.safeMetrics(options.metrics ?? core.NOOP_METRICS);
|
|
246
|
-
const cache = new core.BoundedLru({
|
|
247
|
-
maxEntries: options.cacheMaxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
248
|
-
ttlMs: options.cacheTtlMs,
|
|
249
|
-
clock
|
|
250
|
-
});
|
|
251
|
-
const resolved = resolveColdSource(options, clock);
|
|
252
|
-
let cold = resolved.source;
|
|
253
|
-
let warm = options.warm;
|
|
254
|
-
if (options.retry !== false) {
|
|
255
|
-
const retryOpts = {
|
|
256
|
-
clock,
|
|
257
|
-
rng,
|
|
258
|
-
policy: options.retry,
|
|
259
|
-
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
260
|
-
onRetry: (info) => {
|
|
261
|
-
metrics.onEvent({
|
|
262
|
-
kind: "retry",
|
|
263
|
-
reason: "transient",
|
|
264
|
-
attempt: info.attempt,
|
|
265
|
-
delayMs: info.delayMs
|
|
266
|
-
});
|
|
267
|
-
options.onRetry?.(info);
|
|
268
|
-
}
|
|
269
|
-
};
|
|
270
|
-
warm = new core.RetryingWarmDriver(options.warm, retryOpts);
|
|
271
|
-
cold = new core.RetryingColdChunkSource(cold, retryOpts);
|
|
272
|
-
}
|
|
273
|
-
if (options.writeConcurrency !== void 0 && (!Number.isInteger(options.writeConcurrency) || options.writeConcurrency < 1)) {
|
|
274
|
-
throw new core.ValidationError(
|
|
275
|
-
`writeConcurrency must be a positive integer; got ${options.writeConcurrency}`
|
|
276
|
-
);
|
|
277
|
-
}
|
|
278
|
-
this.budget = core.resolveBudget(options.budget, core.DEFAULT_BUDGET);
|
|
279
|
-
const deps = {
|
|
280
|
-
warm,
|
|
281
|
-
cold,
|
|
282
|
-
cache,
|
|
283
|
-
codec: roaringCodec,
|
|
284
|
-
// the facade injects the flagship codec; core stays codec-agnostic
|
|
285
|
-
clock,
|
|
286
|
-
rng,
|
|
287
|
-
occBackoff: options.occBackoff,
|
|
288
|
-
metrics,
|
|
289
|
-
warmReadConsistency: options.warmReadConsistency,
|
|
290
|
-
writeConcurrency: options.writeConcurrency,
|
|
291
|
-
maxWarmScanBytes: options.maxWarmScanBytes,
|
|
292
|
-
budget: this.budget
|
|
293
|
-
};
|
|
294
|
-
this.engine = new core.SegmentEngine(deps);
|
|
295
|
-
this.clock = clock;
|
|
296
|
-
this.metrics = metrics;
|
|
297
|
-
this.coldDriver = resolved.driver;
|
|
298
|
-
this.warmDriver = options.warm;
|
|
299
|
-
this.registry = options.registry;
|
|
300
|
-
this.keystore = options.keystore;
|
|
301
|
-
this.requireEncryption = options.requireEncryption ?? false;
|
|
302
|
-
}
|
|
303
|
-
/**
|
|
304
|
-
* Build {@link CompactionDeps} from the store's own drivers, for the in-process lifecycle helpers
|
|
305
|
-
* ({@link compact} / {@link eraseSubject}). Requires the store to have been constructed with a **raw cold
|
|
306
|
-
* driver** (a pre-built `ColdChunkSource` has no underlying `IColdDriver` to compact through) and a `registry`
|
|
307
|
-
* (the authoritative `currentGen` pointer compaction swaps). Out-of-process callers use the `compactSegment`
|
|
308
|
-
* free function with explicit deps.
|
|
309
|
-
*/
|
|
310
|
-
compactionDeps() {
|
|
311
|
-
if (this.coldDriver === void 0) {
|
|
312
|
-
throw new core.UnsupportedError(
|
|
313
|
-
"compact/dropSegment/retireExpired/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
|
|
314
|
-
);
|
|
315
|
-
}
|
|
316
|
-
if (this.registry === void 0) {
|
|
317
|
-
throw new core.UnsupportedError(
|
|
318
|
-
"compact/dropSegment/retireExpired/eraseSubject/checkConsistency need a `registry` in the store config"
|
|
319
|
-
);
|
|
320
|
-
}
|
|
321
|
-
return {
|
|
322
|
-
cold: this.coldDriver,
|
|
323
|
-
warm: this.warmDriver,
|
|
324
|
-
registry: this.registry,
|
|
325
|
-
clock: this.clock,
|
|
326
|
-
codec: roaringCodec,
|
|
327
|
-
// facade injects the flagship codec
|
|
328
|
-
keystore: this.keystore,
|
|
329
|
-
requireEncryption: this.requireEncryption,
|
|
330
|
-
metrics: this.metrics
|
|
331
|
-
// safe-wrapped sink → store.compact/eraseSubject emit `compaction` events (gap #2)
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
/** Get a handle to a segment. Validates the name/namespace grammar (finding S2). */
|
|
335
|
-
segment(name, options) {
|
|
336
|
-
const ref = { segment: name, namespace: options?.namespace };
|
|
337
|
-
core.validateSegmentRef(ref);
|
|
338
|
-
return new Segment(this.engine, ref, this.clock, this.metrics);
|
|
339
|
-
}
|
|
340
|
-
/**
|
|
341
|
-
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?** (Phase 6b.)
|
|
342
|
-
*
|
|
343
|
-
* Enumerates the **registered** segments (via the store's own `registry`) and does a tier-merging `has(id)` on
|
|
344
|
-
* each — no drivers to re-pass. Complete only over registered segments (register your segments if you claim
|
|
345
|
-
* SAR support). There is deliberately **no `id → segments` reverse index** — that would tax every write for a
|
|
346
|
-
* rare request; this admin scan is `O(registered segments)` and touches no hot path.
|
|
347
|
-
* Requires a `registry` in the store config
|
|
348
|
-
* (throws {@link UnsupportedError} otherwise).
|
|
349
|
-
*/
|
|
350
|
-
async subjectReport(id, options = {}) {
|
|
351
|
-
const registry = this.requireRegistry("subjectReport");
|
|
352
|
-
requireScope(options, "subjectReport");
|
|
353
|
-
validateConcurrency(options.concurrency);
|
|
354
|
-
const budget = core.resolvePerOpBudget(options.budget, this.budget);
|
|
355
|
-
core.splitId(id);
|
|
356
|
-
const recs = await core.collectWithinBudget(
|
|
357
|
-
registry.list(options.namespace),
|
|
358
|
-
budget,
|
|
359
|
-
"subjectReport"
|
|
360
|
-
);
|
|
361
|
-
const membership = await core.mapWithConcurrency(
|
|
362
|
-
recs,
|
|
363
|
-
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
364
|
-
async (rec) => {
|
|
365
|
-
if (rec.status === "destroyed") return null;
|
|
366
|
-
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
367
|
-
return await this.engine.has(ref, id, { consistent: true }) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
368
|
-
}
|
|
369
|
-
);
|
|
370
|
-
const segments = membership.filter((m) => m !== null);
|
|
371
|
-
return { id, segments, scannedSegments: recs.length };
|
|
372
|
-
}
|
|
373
|
-
/**
|
|
374
|
-
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with a physical-deletion
|
|
375
|
-
* guarantee.** (Phase 6b.)
|
|
376
|
-
*
|
|
377
|
-
* For each **registered** segment the id is a member of: writes a logical `remove` tombstone (immediate) and
|
|
378
|
-
* then **force-compacts that segment now** — folding the tombstone into a fresh immutable generation so the
|
|
379
|
-
* bit is *physically* gone from Cold on return, even for an otherwise-idle/archival segment that organic
|
|
380
|
-
* compaction would never touch (the P13 fix). The returned per-segment record is your **erasure ledger** —
|
|
381
|
-
* persist it / route it to your audit sink as the proof of deletion; the physical-purge proof is
|
|
382
|
-
* compaction's own VERIFY step (a re-`has()` here would be unsound — a pinned cold source can still read the
|
|
383
|
-
* old generation; take a fresh generation view after erasing, exactly as with the compaction daemon).
|
|
384
|
-
*
|
|
385
|
-
* Uses the store's **own** drivers (raw cold + warm + registry), so the old integrator obligation to wire a
|
|
386
|
-
* matching `CompactionDeps` is gone — the `remove()` and the force-compaction provably run over the same tiers.
|
|
387
|
-
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise;
|
|
388
|
-
* a pre-built `ColdChunkSource` store has no `IColdDriver` to compact through — use the `compactSegment` free
|
|
389
|
-
* function there). **One contract remains** (an integrator obligation the library cannot check): **do not
|
|
390
|
-
* concurrently re-add the same id while erasing it.** A normal `add(id)` landing between the `remove` and the
|
|
391
|
-
* compaction clears the tombstone and folds the id back into the new generation
|
|
392
|
-
* (`effective = (cold ∪ adds) \ removes`); `physicallyPurged` attests only that compaction committed a
|
|
393
|
-
* generation with the tombstones applied at pin time — quiesce writes for the subject during erasure.
|
|
394
|
-
*
|
|
395
|
-
* **A `physicallyPurged:false` entry means the logical removal holds but the physical purge didn't run this
|
|
396
|
-
* call** — either a daemon held a live lease (`note:'leased-by-other'`; the daemon finishes it), or the
|
|
397
|
-
* force-compaction hit an isolated fault (`note:'error: …'`; per-segment faults are caught so one segment
|
|
398
|
-
* can't discard the whole ledger). `physicallyPurged:false` is **conservative** (a concurrent daemon may
|
|
399
|
-
* already have purged the bit — `'clean'`). **Recovery:** follow up any `physicallyPurged:false` entry with
|
|
400
|
-
* `store.compact(ref)` (or let the daemon do it) — **re-running `eraseSubject` will NOT re-purge it**, because
|
|
401
|
-
* `has()` now reads false (the tombstone) so the segment is skipped as a non-member. The physical-purge proof
|
|
402
|
-
* is compaction's own VERIFY step (a re-`has()` here would be unsound: a pinned cold source can still read the
|
|
403
|
-
* old generation — take a fresh generation view after erasing, as with the daemon). Admin-only path;
|
|
404
|
-
* `O(registered segments)`, no hot-path cost. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
405
|
-
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
406
|
-
* is the `destroySegment`/`eraseNamespace` free functions.
|
|
407
|
-
*/
|
|
408
|
-
async eraseSubject(id, options) {
|
|
409
|
-
const compaction = this.compactionDeps();
|
|
410
|
-
core.validateCompactionOptions({ owner: options.owner });
|
|
411
|
-
requireScope(options, "eraseSubject");
|
|
412
|
-
validateConcurrency(options.concurrency);
|
|
413
|
-
const budget = core.resolvePerOpBudget(options.budget, this.budget);
|
|
414
|
-
core.splitId(id);
|
|
415
|
-
const recs = await core.collectWithinBudget(
|
|
416
|
-
compaction.registry.list(options.namespace),
|
|
417
|
-
budget,
|
|
418
|
-
"eraseSubject"
|
|
419
|
-
);
|
|
420
|
-
const entries = await core.mapWithConcurrency(
|
|
421
|
-
recs,
|
|
422
|
-
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
423
|
-
async (rec) => {
|
|
424
|
-
if (rec.status === "destroyed") return null;
|
|
425
|
-
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
426
|
-
let removed = false;
|
|
427
|
-
try {
|
|
428
|
-
if (!await this.engine.has(ref, id, { consistent: true })) return null;
|
|
429
|
-
await this.engine.remove(ref, id);
|
|
430
|
-
removed = true;
|
|
431
|
-
const result = await core.compactSegment(ref, compaction, {
|
|
432
|
-
owner: options.owner,
|
|
433
|
-
audit: options.audit
|
|
434
|
-
});
|
|
435
|
-
return {
|
|
436
|
-
segment: rec.segment,
|
|
437
|
-
namespace: rec.namespace,
|
|
438
|
-
removed: true,
|
|
439
|
-
physicallyPurged: result.compacted,
|
|
440
|
-
toGen: result.toGen,
|
|
441
|
-
note: result.compacted ? void 0 : result.reason
|
|
442
|
-
};
|
|
443
|
-
} catch (err) {
|
|
444
|
-
return {
|
|
445
|
-
segment: rec.segment,
|
|
446
|
-
namespace: rec.namespace,
|
|
447
|
-
removed,
|
|
448
|
-
physicallyPurged: false,
|
|
449
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
);
|
|
454
|
-
const erasedFrom = entries.filter((e) => e !== null);
|
|
455
|
-
return { id, erasedFrom, scannedSegments: recs.length };
|
|
456
|
-
}
|
|
457
|
-
/**
|
|
458
|
-
* **Compact one segment now**, in-process, using the store's own drivers — a convenience over the
|
|
459
|
-
* {@link compactSegment} free function. Merges `(cold ∪ adds) \ removes` into a fresh immutable generation,
|
|
460
|
-
* swaps `currentGen`, then version-fenced-purges the archived Warm rows. A no-op (`compacted:false`) when the
|
|
461
|
-
* segment has no dirty rows or a daemon holds a live lease — never throws on the normal contention/lease paths.
|
|
462
|
-
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise);
|
|
463
|
-
* like the daemon, it uses the raw drivers directly (a transient fault surfaces to you rather than retrying
|
|
464
|
-
* under the hood). **Not for the request/hot path** — it reads and rewrites a whole Cold generation (streaming,
|
|
465
|
-
* constant-memory, but full-segment I/O). Use the `compact-segments` daemon (`runCompactionCycle`) for routine
|
|
466
|
-
* background compaction; reach for `store.compact()` for occasional/manual one-shots — a maintenance endpoint,
|
|
467
|
-
* a small daemon-less deployment, or right after a targeted `remove`.
|
|
468
|
-
*/
|
|
469
|
-
async compact(ref, options) {
|
|
470
|
-
core.validateSegmentRef(ref);
|
|
471
|
-
const deps = this.compactionDeps();
|
|
472
|
-
const result = await core.compactSegment(ref, deps, options);
|
|
473
|
-
if (result.compacted) {
|
|
474
|
-
await core.gcOrphanGenerations(ref, { cold: deps.cold, registry: deps.registry }).catch(
|
|
475
|
-
() => void 0
|
|
476
|
-
);
|
|
477
|
-
}
|
|
478
|
-
return result;
|
|
479
|
-
}
|
|
480
|
-
/**
|
|
481
|
-
* **Dispose of a segment — tombstone it, delete its Warm rows, delete its Cold objects.** Irreversible.
|
|
482
|
-
*
|
|
483
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
|
|
484
|
-
* backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
|
|
485
|
-
* removes the storage and works on a cleartext segment. On an encrypted segment it does both.
|
|
486
|
-
*
|
|
487
|
-
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
488
|
-
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
489
|
-
* variable twice.
|
|
490
|
-
*
|
|
491
|
-
* ```ts
|
|
492
|
-
* for (const day of expiredDays) {
|
|
493
|
-
* // A colon is NOT legal in a name — the family goes in the namespace, the date in the segment.
|
|
494
|
-
* const ref = { namespace: 'active-daily', segment: day };
|
|
495
|
-
* await store.dropSegment(ref, { confirmSegment: ref.segment });
|
|
496
|
-
* }
|
|
497
|
-
* ```
|
|
498
|
-
*
|
|
499
|
-
* **Omitting the namespace addresses a different segment and is a silent no-op** — you get
|
|
500
|
-
* `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
|
|
501
|
-
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
502
|
-
*
|
|
503
|
-
* **Works on an accumulator too.** A segment you created by writing to it — never bulk-loaded, never compacted,
|
|
504
|
-
* so it has no registry row and no Cold objects — is retired by deleting its Warm rows alone. That reports
|
|
505
|
-
* `{ dropped: true, reason: 'warm-only' }`: there is no tombstone to write, and deliberately none is written,
|
|
506
|
-
* because a `destroyed` row per retired daily bucket would be registry litter. The data really is gone.
|
|
507
|
-
*
|
|
508
|
-
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
509
|
-
* reclaimed and the drop should be re-run. A compaction that was already in flight when the tombstone landed
|
|
510
|
-
* still finishes staging one more object, so a single sweep can miss it — this call re-sweeps and then reports
|
|
511
|
-
* whatever it still could not remove rather than returning a result that looks like a clean drop.
|
|
512
|
-
*
|
|
513
|
-
* Reads become empty within `coldGenTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
514
|
-
* segment may answer from its cached generation + hot chunks until that window lapses. A reader that never
|
|
515
|
-
* touched it sees empty at once. **That bound needs a clock and `coldGenTtlMs > 0`** — a store built without a
|
|
516
|
-
* clock, or with `coldGenTtlMs: 0` ("pin forever"), holds its resolved snapshot for its own lifetime and can
|
|
517
|
-
* keep answering `true` for a dropped segment indefinitely; restart it.
|
|
518
|
-
*
|
|
519
|
-
* Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
|
|
520
|
-
* because it has to enumerate and delete generations — a pre-built `ColdChunkSource` only reads.
|
|
521
|
-
*/
|
|
522
|
-
async dropSegment(ref, options) {
|
|
523
|
-
core.validateSegmentRef(ref);
|
|
524
|
-
const deps = this.compactionDeps();
|
|
525
|
-
return core.dropSegment(ref, { registry: deps.registry, warm: deps.warm, cold: deps.cold }, options);
|
|
526
|
-
}
|
|
527
|
-
/**
|
|
528
|
-
* **Record when this segment becomes eligible for retirement.** One registry write; nothing is deleted here,
|
|
529
|
-
* and nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
530
|
-
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
531
|
-
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
532
|
-
*
|
|
533
|
-
* `expiresAt` is an **absolute epoch-ms you compute**, not a duration the library derives. A relative TTL would
|
|
534
|
-
* have to be anchored to something the library knows — `updatedAt`, or the current generation — and compaction
|
|
535
|
-
* rewrites both, so "expire 30 days after the last write" would keep a busy daily bucket alive forever
|
|
536
|
-
* precisely because the daemon is working.
|
|
537
|
-
*
|
|
538
|
-
* ```ts
|
|
539
|
-
* const DAY = 86_400_000;
|
|
540
|
-
* const ref = { namespace: 'active-daily', segment: '2026-08-05' };
|
|
541
|
-
* await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY });
|
|
542
|
-
* ```
|
|
543
|
-
*
|
|
544
|
-
* **Works on an accumulator**, which is the whole point: a segment you created by writing to it has no
|
|
545
|
-
* registry row, so it is invisible to `registry.list()` and therefore to every fleet-wide operation, including
|
|
546
|
-
* the sweep. This call mints the row (`createdRow: true` in the result) with **no Cold generation**, so it
|
|
547
|
-
* becomes enumerable while every read still resolves exactly as before.
|
|
548
|
-
*
|
|
549
|
-
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
550
|
-
* buckets is normal. A value below `MIN_EXPIRES_AT_MS` (2001-09-09) is rejected: it is almost certainly epoch
|
|
551
|
-
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a `registry`
|
|
552
|
-
* in the store config (throws {@link UnsupportedError} otherwise), and refuses a crypto-shredded segment.
|
|
553
|
-
*/
|
|
554
|
-
async setRetention(ref, policy) {
|
|
555
|
-
core.validateSegmentRef(ref);
|
|
556
|
-
return core.setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
|
|
557
|
-
}
|
|
558
|
-
/**
|
|
559
|
-
* **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
|
|
560
|
-
* tombstone). Returns the string `'invalid'` for a row whose `expiresAt` is present but unusable — a
|
|
561
|
-
* hand-edited row, or one from a restore — so a malformed policy is visible rather than silently reading as
|
|
562
|
-
* "never expires" on a segment someone believes is expiring.
|
|
563
|
-
*/
|
|
564
|
-
async getRetention(ref) {
|
|
565
|
-
core.validateSegmentRef(ref);
|
|
566
|
-
return core.getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
567
|
-
}
|
|
568
|
-
/**
|
|
569
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether a policy was actually removed (`false`
|
|
570
|
-
* when there was none). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
571
|
-
* to the setter is how a typo becomes a deletion.
|
|
572
|
-
*/
|
|
573
|
-
async clearRetention(ref) {
|
|
574
|
-
core.validateSegmentRef(ref);
|
|
575
|
-
return core.clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
|
|
576
|
-
}
|
|
577
|
-
/**
|
|
578
|
-
* **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
|
|
579
|
-
* the policies `setRetention` records — and it is a **call, not a daemon**. Nothing schedules it; you run it from
|
|
580
|
-
* whatever heartbeat your deployment already has (an EventBridge rule, a Kubernetes CronJob, a queue consumer,
|
|
581
|
-
* your compaction worker's loop). A library that started a timer would behave differently in a Lambda, an edge
|
|
582
|
-
* isolate and a long-lived server, which is worse than not having one.
|
|
583
|
-
*
|
|
584
|
-
* Each retirement goes through `dropSegment`, so the Warm → registry → Cold ordering, the re-sweep for a
|
|
585
|
-
* generation staged by an in-flight compaction, and the `generationsRemaining` report all come from one
|
|
586
|
-
* implementation rather than two.
|
|
587
|
-
*
|
|
588
|
-
* ```ts
|
|
589
|
-
* // In your scheduled handler. Start with a preview in a new deployment.
|
|
590
|
-
* const preview = await store.retireExpired({ namespace: 'active-daily', dryRun: true });
|
|
591
|
-
* console.log(`would retire ${preview.retired} of ${preview.scanned} (limited: ${preview.limited})`);
|
|
592
|
-
*
|
|
593
|
-
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
594
|
-
* for (const e of swept.entries) {
|
|
595
|
-
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`); // invalid-policy / limit / failed: …
|
|
596
|
-
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
597
|
-
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
598
|
-
* }
|
|
599
|
-
* }
|
|
600
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more are still eligible
|
|
601
|
-
* ```
|
|
602
|
-
*
|
|
603
|
-
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
604
|
-
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
605
|
-
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
606
|
-
* `limit` (default 100) cut the pass short and more are eligible. That cap is charged on **attempts**, not
|
|
607
|
-
* successes, which is what makes it a real bound: `dropSegment` deletes Warm and writes the tombstone before
|
|
608
|
-
* sweeping Cold, so a fault in the Cold phase is a segment that is already retired. Retirements are sequential
|
|
609
|
-
* (~8 round trips each), so `limit` is a wall-clock knob too, and `retired` counts deletions only — a dry run
|
|
610
|
-
* reports `wouldRetire` instead, so a dashboard summing `retired` can never show a phantom deletion.
|
|
611
|
-
*
|
|
612
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
613
|
-
* and only once that segment's Warm rows and Cold generations are provably gone — collecting a straggler
|
|
614
|
-
* generation itself first, since nothing else ever would for a tombstoned segment. Attribution is a **positive
|
|
615
|
-
* marker the sweep stamps on its own retirements**, not an inference from "destroyed + an expired policy": a
|
|
616
|
-
* crypto-shred leaves `retention` untouched, so setting a policy and then honouring a right-to-erasure request
|
|
617
|
-
* mid-window produces exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the
|
|
618
|
-
* name. Pass `purgeTombstones: false` to keep every tombstone.
|
|
619
|
-
*
|
|
620
|
-
* Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
|
|
621
|
-
* because retiring a segment deletes its Cold objects. `now` defaults to the store's clock.
|
|
622
|
-
*/
|
|
623
|
-
async retireExpired(options = {}) {
|
|
624
|
-
const deps = this.compactionDeps();
|
|
625
|
-
return core.retireExpired(
|
|
626
|
-
{ registry: deps.registry, warm: deps.warm, cold: deps.cold },
|
|
627
|
-
{ ...options, now: options.now ?? this.clock.now() }
|
|
628
|
-
);
|
|
629
|
-
}
|
|
630
|
-
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
631
|
-
requireRegistry(op) {
|
|
632
|
-
if (this.registry === void 0) {
|
|
633
|
-
throw new core.UnsupportedError(`${op} needs a \`registry\` in the store config`);
|
|
634
|
-
}
|
|
635
|
-
return this.registry;
|
|
636
|
-
}
|
|
637
|
-
/**
|
|
638
|
-
* **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
|
|
639
|
-
* segment's `currentGen` actually has its `.crbm` present in Cold — catching a **torn restore** where the
|
|
640
|
-
* registry (`currentGen`) came back ahead of the object store, so a pointer references a generation that
|
|
641
|
-
* isn't there (reads would then throw). Read-only, bounded fan-out; run it at startup after a restore.
|
|
642
|
-
* Returns `{ checked, inconsistent }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to
|
|
643
|
-
* recover (restore the object store, or roll the registry back to a coherent point). Needs the store built
|
|
644
|
-
* with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise). `destroyed`
|
|
645
|
-
* (crypto-shredded) segments are skipped. Pair it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
646
|
-
*/
|
|
647
|
-
async checkConsistency(options = {}) {
|
|
648
|
-
const deps = this.compactionDeps();
|
|
649
|
-
return core.runConsistencyCheck({ cold: deps.cold, registry: deps.registry }, options);
|
|
650
|
-
}
|
|
651
|
-
/**
|
|
652
|
-
* **Export ("eject") every registered segment's current effective set** through the injected `sink`, using
|
|
653
|
-
* only public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's
|
|
654
|
-
* "Your data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
|
|
655
|
-
* (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
|
|
656
|
-
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
657
|
-
* enumeration and the read path provably share one registry; an all-warm segment not yet in the registry can be
|
|
658
|
-
* named via `options.candidates`. Encrypted segments are decrypted transparently **iff** this store was wired
|
|
659
|
-
* with their keystore — the export is therefore **cleartext**; protect it. Crypto-shredded segments are skipped.
|
|
660
|
-
* A segment that can't be read is isolated into the manifest's `failed[]` (the run continues), so "a manifest
|
|
661
|
-
* exists" means the run finished — check `failed`. The `export-segments` CLI wraps this with a filesystem sink.
|
|
662
|
-
*/
|
|
663
|
-
async exportSegments(sink, options = {}) {
|
|
664
|
-
const registry = this.requireRegistry("exportSegments");
|
|
665
|
-
return core.runExport(this, registry, sink, {
|
|
666
|
-
...options,
|
|
667
|
-
codec: options.codec ?? roaringCodec
|
|
668
|
-
});
|
|
669
|
-
}
|
|
670
|
-
/**
|
|
671
|
-
* Planning cost estimate (Phase 5b) — pure, no instance/data needed: sizing, sales, what-if. For a real,
|
|
672
|
-
* grounded report from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
673
|
-
*/
|
|
674
|
-
static estimateCost(input) {
|
|
675
|
-
return core.estimateCost(input);
|
|
676
|
-
}
|
|
677
|
-
};
|
|
678
|
-
var Segment = class {
|
|
679
|
-
constructor(engine, ref, clock, metrics) {
|
|
680
|
-
this.engine = engine;
|
|
681
|
-
this.ref = ref;
|
|
682
|
-
this.clock = clock;
|
|
683
|
-
this.metrics = metrics;
|
|
684
|
-
this.metricsOn = metrics !== core.NOOP_METRICS;
|
|
685
|
-
}
|
|
686
|
-
engine;
|
|
687
|
-
ref;
|
|
688
|
-
clock;
|
|
689
|
-
metrics;
|
|
690
|
-
metricsOn;
|
|
691
|
-
/**
|
|
692
|
-
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
693
|
-
* entirely when no sink is wired, so the default path pays nothing.
|
|
694
|
-
*/
|
|
695
|
-
async timed(name, fn) {
|
|
696
|
-
if (!this.metricsOn) return fn();
|
|
697
|
-
const startedAt = this.clock.now();
|
|
698
|
-
try {
|
|
699
|
-
return await fn();
|
|
700
|
-
} finally {
|
|
701
|
-
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
/** Add an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
705
|
-
add(id) {
|
|
706
|
-
return this.timed("add", () => this.engine.add(this.ref, id));
|
|
707
|
-
}
|
|
708
|
-
/**
|
|
709
|
-
* Add many ids, from a **sync or async** iterable. Not atomic across chunks — see the class note.
|
|
710
|
-
*
|
|
711
|
-
* An async source means a database cursor streams straight in — `addMany(athenaCursor())` rather than
|
|
712
|
-
* hand-batching `page → addMany(page)`. Ids are grouped by chunk and each chunk is written **exactly once**,
|
|
713
|
-
* however long the stream: pending ids are held compressed, not buffered-and-flushed, so a long stream costs
|
|
714
|
-
* no more backend writes than a short one. Peak memory is the roaring representation of the ids you pass.
|
|
715
|
-
*
|
|
716
|
-
* **For a very large set, reach for `bulkLoadCrbmGeneration` instead.** `addMany` expresses a *delta* and
|
|
717
|
-
* writes one warm row per touched chunk — around 61,000 of them for a set spread across the whole id space.
|
|
718
|
-
* Bulk-load *replaces* the segment with one immutable Cold object. See the guide's cost comparison; picking
|
|
719
|
-
* the wrong one of these is the most expensive mistake available in this library.
|
|
720
|
-
*/
|
|
721
|
-
addMany(ids) {
|
|
722
|
-
return this.timed("addMany", () => this.engine.addMany(this.ref, ids));
|
|
723
|
-
}
|
|
724
|
-
/** Remove an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
725
|
-
remove(id) {
|
|
726
|
-
return this.timed("remove", () => this.engine.remove(this.ref, id));
|
|
727
|
-
}
|
|
728
|
-
/** Remove many ids, from a **sync or async** iterable. Not atomic across chunks — see {@link addMany}. */
|
|
729
|
-
removeMany(ids) {
|
|
730
|
-
return this.timed("removeMany", () => this.engine.removeMany(this.ref, ids));
|
|
731
|
-
}
|
|
732
|
-
/**
|
|
733
|
-
* **Claim ids atomically — add them, and get back only the ones that were not already there.**
|
|
734
|
-
*
|
|
735
|
-
* The durable analogue of Redis `SETBIT` returning the prior bit, which is what an exactly-once *"have I
|
|
736
|
-
* already sent to / already processed this id?"* check needs. `has()` then `add()` cannot give you this: two
|
|
737
|
-
* workers can both read absent and both proceed.
|
|
738
|
-
*
|
|
739
|
-
* ```ts
|
|
740
|
-
* const sent = store.segment(day, { namespace: 'sent-daily' });
|
|
741
|
-
* const toSend = await sent.claimMany(candidateIds); // only the ids this worker won
|
|
742
|
-
* for (const id of toSend) enqueue(id); // ...so nobody else will send them
|
|
743
|
-
* ```
|
|
744
|
-
*
|
|
745
|
-
* **Pass a batch, not one id.** A Warm write rewrites a whole 64K-id chunk bitmap, so claiming ids one at a
|
|
746
|
-
* time is the single most expensive way to use this library — measured at ~5,000 writes and ~23 MB for 5,000
|
|
747
|
-
* ids, against 1 write and ~8 KB for the same ids in one call. This method does one OCC read-modify-write per
|
|
748
|
-
* distinct chunk, giving you Redis's *semantics* without Redis's per-id *cost shape*.
|
|
749
|
-
*
|
|
750
|
-
* **Exactly-once holds per id**, which is the guarantee that matters: each id lives in one chunk, a chunk is
|
|
751
|
-
* one OCC row, so exactly one concurrent claimer sees any given id as new. Like `addMany` it is **not** atomic
|
|
752
|
-
* across chunks — a mid-flight failure can leave some chunks claimed — but re-running is safe, because
|
|
753
|
-
* already-claimed ids simply come back as not-new.
|
|
754
|
-
*
|
|
755
|
-
* Order of the returned ids is unspecified.
|
|
756
|
-
*/
|
|
757
|
-
claimMany(ids) {
|
|
758
|
-
return this.timed("claimMany", () => this.engine.claimMany(this.ref, ids));
|
|
759
|
-
}
|
|
760
|
-
has(id) {
|
|
761
|
-
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
762
|
-
}
|
|
763
|
-
count() {
|
|
764
|
-
return this.timed("count", () => this.engine.count(this.ref));
|
|
765
|
-
}
|
|
766
|
-
iterate() {
|
|
767
|
-
return this.engine.iterate(this.ref);
|
|
768
|
-
}
|
|
769
|
-
/**
|
|
770
|
-
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
771
|
-
* Fetches only the Cold chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
772
|
-
* and is never downloaded), streaming under a bounded in-flight window — so the Cold footprint stays small
|
|
773
|
-
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
774
|
-
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. (Warm state is read
|
|
775
|
-
* up front, so total memory also carries each operand's Warm size — negligible under Topology-A.) Pass
|
|
776
|
-
* `budget` to override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
777
|
-
*/
|
|
778
|
-
/**
|
|
779
|
-
* Map the facade's `Segment` handles in `exclude` down to the plain refs `core` takes. A method rather than
|
|
780
|
-
* a module function because `ref` is class-private — the encapsulation is worth more than the free function.
|
|
781
|
-
*/
|
|
782
|
-
refsIn(options) {
|
|
783
|
-
if (options === void 0) return void 0;
|
|
784
|
-
const { exclude, ...rest } = options;
|
|
785
|
-
return exclude === void 0 ? rest : { ...rest, exclude: exclude.map((o) => o.ref) };
|
|
786
|
-
}
|
|
787
|
-
intersect(others, options) {
|
|
788
|
-
return this.engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
|
|
789
|
-
}
|
|
790
|
-
/**
|
|
791
|
-
* Materialize `this ∩ others…` (minus `exclude`) **into** `dest` (added, not replaced). Streaming +
|
|
792
|
-
* bounded-memory, but **not atomic** across chunks — see {@link addMany}.
|
|
793
|
-
*/
|
|
794
|
-
intersectInto(dest, others, options) {
|
|
795
|
-
return this.timed(
|
|
796
|
-
"intersectInto",
|
|
797
|
-
() => this.engine.intersectInto(
|
|
798
|
-
dest.ref,
|
|
799
|
-
[this.ref, ...others.map((o) => o.ref)],
|
|
800
|
-
this.refsIn(options)
|
|
801
|
-
)
|
|
802
|
-
);
|
|
803
|
-
}
|
|
804
|
-
/**
|
|
805
|
-
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
806
|
-
*
|
|
807
|
-
* **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
|
|
808
|
-
* here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
|
|
809
|
-
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
810
|
-
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
811
|
-
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
812
|
-
* segment once (`unionInto`, or a bulk-load rebuild) is the cheaper shape.
|
|
813
|
-
*/
|
|
814
|
-
union(others, options) {
|
|
815
|
-
return this.engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
|
|
816
|
-
}
|
|
817
|
-
/** Materialize `this ∪ others…` (minus `exclude`) **into** `dest`. **Not atomic** — see {@link addMany}. */
|
|
818
|
-
unionInto(dest, others, options) {
|
|
819
|
-
return this.timed(
|
|
820
|
-
"unionInto",
|
|
821
|
-
() => this.engine.unionInto(
|
|
822
|
-
dest.ref,
|
|
823
|
-
[this.ref, ...others.map((o) => o.ref)],
|
|
824
|
-
this.refsIn(options)
|
|
825
|
-
)
|
|
826
|
-
);
|
|
827
|
-
}
|
|
828
|
-
/**
|
|
829
|
-
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
830
|
-
*
|
|
831
|
-
* Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
|
|
832
|
-
* overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
|
|
833
|
-
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
834
|
-
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
835
|
-
*
|
|
836
|
-
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
837
|
-
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
838
|
-
*/
|
|
839
|
-
andNot(excludes, options) {
|
|
840
|
-
return this.engine.andNot(
|
|
841
|
-
this.ref,
|
|
842
|
-
excludes.map((o) => o.ref),
|
|
843
|
-
options
|
|
844
|
-
);
|
|
845
|
-
}
|
|
846
|
-
/** Materialize `this \ (excludes…)` **into** `dest`. **Not atomic** — see {@link addMany}. */
|
|
847
|
-
andNotInto(dest, excludes, options) {
|
|
848
|
-
return this.timed(
|
|
849
|
-
"andNotInto",
|
|
850
|
-
() => this.engine.andNotInto(
|
|
851
|
-
dest.ref,
|
|
852
|
-
this.ref,
|
|
853
|
-
excludes.map((o) => o.ref),
|
|
854
|
-
options
|
|
855
|
-
)
|
|
856
|
-
);
|
|
857
|
-
}
|
|
858
|
-
/**
|
|
859
|
-
* Grounded cost report for this segment (Phase 5b): storage cost from its **real** `.crbm` size (exact,
|
|
860
|
-
* no payload reads); request cost from the supplied `workload` rates. A segment with no Cold generation
|
|
861
|
-
* reports zero storage. See {@link CostReport} — it always includes a verdict (incl. the lose-zone).
|
|
862
|
-
*/
|
|
863
|
-
async costReport(options) {
|
|
864
|
-
const canMeasure = this.engine.supportsColdSize;
|
|
865
|
-
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
866
|
-
return core.groundedReport({
|
|
867
|
-
coldBytes: size?.sizeBytes ?? 0,
|
|
868
|
-
grounded: canMeasure,
|
|
869
|
-
workload: options?.workload,
|
|
870
|
-
topology: options?.topology,
|
|
871
|
-
pricing: options?.pricing,
|
|
872
|
-
extraNotes: canMeasure ? void 0 : ["cold source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
873
|
-
});
|
|
874
|
-
}
|
|
875
|
-
};
|
|
876
|
-
var VERSION = "0.9.0";
|
|
877
|
-
|
|
878
|
-
exports.CloudRoaring = CloudRoaring;
|
|
879
|
-
exports.SafeBitmap = SafeBitmap;
|
|
880
|
-
exports.Segment = Segment;
|
|
881
|
-
exports.VERSION = VERSION;
|
|
882
|
-
exports.bulkLoadCrbmGeneration = bulkLoadCrbmGeneration;
|
|
883
|
-
exports.compactSegment = compactSegment;
|
|
884
|
-
exports.roaringCodec = roaringCodec;
|
|
885
|
-
exports.runCompactionCycle = runCompactionCycle;
|
|
886
|
-
exports.runExport = runExport;
|
|
887
|
-
Object.keys(core).forEach(function (k) {
|
|
888
|
-
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
889
|
-
enumerable: true,
|
|
890
|
-
get: function () { return core[k]; }
|
|
891
|
-
});
|
|
892
|
-
});
|
|
893
|
-
//# sourceMappingURL=index.cjs.map
|
|
894
|
-
//# sourceMappingURL=index.cjs.map
|