@cloudbitmaps/roaring 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/NOTICE +3 -3
  2. package/PRIVACY.md +268 -76
  3. package/README.md +85 -95
  4. package/dist/bin/export-segments.d.ts +21 -0
  5. package/dist/bin/export-segments.js +1972 -30
  6. package/dist/bin/export-segments.js.map +7 -1
  7. package/dist/index.d.ts +969 -458
  8. package/dist/index.js +1461 -354
  9. package/dist/index.js.map +7 -1
  10. package/dist/open-charging-storage.d.ts +34 -0
  11. package/dist/option-keys.d.ts +19 -0
  12. package/dist/portable/decode.d.ts +53 -0
  13. package/dist/portable/layout.d.ts +32 -0
  14. package/dist/reserved-namespace.d.ts +3 -0
  15. package/dist/roaring-codec.d.ts +68 -0
  16. package/dist/system-clock.d.ts +39 -0
  17. package/dist/testing/conformance.d.ts +54 -0
  18. package/dist/testing/fuzz-codec.d.ts +23 -0
  19. package/package.json +11 -117
  20. package/dist/azure/index.cjs +0 -14
  21. package/dist/azure/index.cjs.map +0 -1
  22. package/dist/azure/index.d.cts +0 -9
  23. package/dist/azure/index.d.ts +0 -9
  24. package/dist/azure/index.js +0 -3
  25. package/dist/azure/index.js.map +0 -1
  26. package/dist/bin/compact-segments.js +0 -201
  27. package/dist/bin/compact-segments.js.map +0 -1
  28. package/dist/cassandra/index.cjs +0 -14
  29. package/dist/cassandra/index.cjs.map +0 -1
  30. package/dist/cassandra/index.d.cts +0 -1
  31. package/dist/cassandra/index.d.ts +0 -1
  32. package/dist/cassandra/index.js +0 -3
  33. package/dist/cassandra/index.js.map +0 -1
  34. package/dist/chunk-D6LIJSGF.js +0 -906
  35. package/dist/chunk-D6LIJSGF.js.map +0 -1
  36. package/dist/dynamodb/index.cjs +0 -14
  37. package/dist/dynamodb/index.cjs.map +0 -1
  38. package/dist/dynamodb/index.d.cts +0 -1
  39. package/dist/dynamodb/index.d.ts +0 -1
  40. package/dist/dynamodb/index.js +0 -3
  41. package/dist/dynamodb/index.js.map +0 -1
  42. package/dist/gcs/index.cjs +0 -14
  43. package/dist/gcs/index.cjs.map +0 -1
  44. package/dist/gcs/index.d.cts +0 -1
  45. package/dist/gcs/index.d.ts +0 -1
  46. package/dist/gcs/index.js +0 -3
  47. package/dist/gcs/index.js.map +0 -1
  48. package/dist/index.cjs +0 -894
  49. package/dist/index.cjs.map +0 -1
  50. package/dist/index.d.cts +0 -734
  51. package/dist/mongodb/index.cjs +0 -14
  52. package/dist/mongodb/index.cjs.map +0 -1
  53. package/dist/mongodb/index.d.cts +0 -1
  54. package/dist/mongodb/index.d.ts +0 -1
  55. package/dist/mongodb/index.js +0 -3
  56. package/dist/mongodb/index.js.map +0 -1
  57. package/dist/mysql/index.cjs +0 -14
  58. package/dist/mysql/index.cjs.map +0 -1
  59. package/dist/mysql/index.d.cts +0 -1
  60. package/dist/mysql/index.d.ts +0 -1
  61. package/dist/mysql/index.js +0 -3
  62. package/dist/mysql/index.js.map +0 -1
  63. package/dist/postgres/index.cjs +0 -14
  64. package/dist/postgres/index.cjs.map +0 -1
  65. package/dist/postgres/index.d.cts +0 -1
  66. package/dist/postgres/index.d.ts +0 -1
  67. package/dist/postgres/index.js +0 -3
  68. package/dist/postgres/index.js.map +0 -1
  69. package/dist/redis/index.cjs +0 -14
  70. package/dist/redis/index.cjs.map +0 -1
  71. package/dist/redis/index.d.cts +0 -1
  72. package/dist/redis/index.d.ts +0 -1
  73. package/dist/redis/index.js +0 -3
  74. package/dist/redis/index.js.map +0 -1
  75. package/dist/s3/index.cjs +0 -14
  76. package/dist/s3/index.cjs.map +0 -1
  77. package/dist/s3/index.d.cts +0 -1
  78. package/dist/s3/index.d.ts +0 -1
  79. package/dist/s3/index.js +0 -3
  80. package/dist/s3/index.js.map +0 -1
@@ -1,21 +1,1957 @@
1
1
  #!/usr/bin/env node
2
- import { src_exports, CloudRoaring } from '../chunk-D6LIJSGF.js';
3
- import { randomUUID } from 'crypto';
4
- import { mkdir, writeFile, rename, rm, open } from 'fs/promises';
5
- import { join } from 'path';
6
- import { pathToFileURL } from 'url';
7
- import { validateSegmentRef } from '@cloudbitmaps/core';
8
2
 
9
- function parseSegments(raw) {
10
- const entries = (raw ?? "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
11
- if (entries.length === 0) return void 0;
12
- return entries.map((entry) => {
13
- const slash = entry.indexOf("/");
14
- const ref = slash === -1 ? { segment: entry } : { namespace: entry.slice(0, slash), segment: entry.slice(slash + 1) };
15
- validateSegmentRef(ref);
16
- return ref;
17
- });
3
+ // src/bin/export-segments.ts
4
+ import { randomUUID } from "node:crypto";
5
+ import { realpathSync } from "node:fs";
6
+ import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";
7
+ import { join } from "node:path";
8
+ import { pathToFileURL } from "node:url";
9
+ import { encodeNameForPath, namespacePathPart } from "@cloudbitmaps/core/driver-kit";
10
+
11
+ // src/index.ts
12
+ import {
13
+ BoundedLru,
14
+ PinnedStorageChunkSource,
15
+ segmentKey,
16
+ CrbmStorageChunkSource,
17
+ DEFAULT_BUDGET,
18
+ DEFAULT_RETRY_POLICY,
19
+ withRetry,
20
+ isStorageBackend,
21
+ NOOP_METRICS,
22
+ RetryingStorageChunkSource,
23
+ SegmentEngine,
24
+ UnsupportedError,
25
+ ValidationError as ValidationError2,
26
+ WriteConflictError,
27
+ MIN_EXPIRES_AT_MS,
28
+ checkBudget,
29
+ collectWithinBudget,
30
+ excludingReservedRows,
31
+ dropSegment,
32
+ eraseIdFromSegment,
33
+ estimateCost,
34
+ groundedReport,
35
+ loadSegment,
36
+ mapWithConcurrency,
37
+ resolveBudget,
38
+ resolvePerOpBudget,
39
+ retireExpired,
40
+ runConsistencyCheck,
41
+ runExport,
42
+ setSegmentRetention,
43
+ clearSegmentRetention,
44
+ getSegmentRetention,
45
+ safeMetrics,
46
+ splitId
47
+ } from "@cloudbitmaps/core";
48
+ import { validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
49
+
50
+ // src/open-charging-storage.ts
51
+ var OpenChargingStorage = class {
52
+ /**
53
+ * @param inner the driver the erasure would otherwise use
54
+ * @param current the generation the segment's row named when it was listed, or `null` when it named none
55
+ * @param charge called once per charged open; it throws to refuse
56
+ */
57
+ constructor(inner, current, charge) {
58
+ this.inner = inner;
59
+ this.charge = charge;
60
+ if (current !== null) this.free.add(current);
61
+ }
62
+ inner;
63
+ charge;
64
+ free = /* @__PURE__ */ new Set();
65
+ capabilities() {
66
+ return this.inner.capabilities();
67
+ }
68
+ async putImmutable(key, write) {
69
+ const written = await this.inner.putImmutable(key, write);
70
+ this.free.add(key.generation);
71
+ return written;
72
+ }
73
+ getRange(key, offset, length) {
74
+ return this.inner.getRange(key, offset, length);
75
+ }
76
+ /** One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. */
77
+ async getTail(key, maxBytes) {
78
+ if (!this.free.has(key.generation)) this.charge();
79
+ return this.inner.getTail(key, maxBytes);
80
+ }
81
+ delete(key) {
82
+ return this.inner.delete(key);
83
+ }
84
+ list(ref) {
85
+ return this.inner.list(ref);
86
+ }
87
+ };
88
+
89
+ // src/index.ts
90
+ import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
91
+ import { listSegments, segmentExists } from "@cloudbitmaps/core";
92
+
93
+ // src/reserved-namespace.ts
94
+ import { ValidationError } from "@cloudbitmaps/core";
95
+ var RESERVED_NAMESPACE_PREFIX = "cbm.due.";
96
+ function refuseReservedNamespace(namespace) {
97
+ if (namespace !== void 0 && namespace.startsWith(RESERVED_NAMESPACE_PREFIX)) {
98
+ throw new ValidationError(
99
+ `namespace "${namespace}" is reserved: names starting with "${RESERVED_NAMESPACE_PREFIX}" hold the library's own bookkeeping rows, which every fleet-wide scan skips, so a segment there would be invisible to an erasure, a consistency check, an export and a retention sweep. Choose another namespace.`
100
+ );
101
+ }
18
102
  }
103
+
104
+ // src/roaring-codec.ts
105
+ import roaring from "roaring";
106
+ import { IntegrityError as IntegrityError2 } from "@cloudbitmaps/core";
107
+
108
+ // src/portable/layout.ts
109
+ import { IntegrityError } from "@cloudbitmaps/core";
110
+ var SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
111
+ var SERIAL_COOKIE = 12347;
112
+ var NO_OFFSET_THRESHOLD = 4;
113
+ var ARRAY_MAX_CARDINALITY = 4096;
114
+ var BITMAP_CONTAINER_BYTES = 8192;
115
+ var MAX_LOW = 65535;
116
+ var ARRAY = 0;
117
+ var BITMAP = 1;
118
+ var RUN = 2;
119
+ function overrun(bytes, offset, length, what) {
120
+ return new IntegrityError(
121
+ `portable roaring: ${what} needs bytes [${offset}, ${offset + length}) of a ${bytes.byteLength}-byte buffer`
122
+ );
123
+ }
124
+ function checkPortableLayout(bytes) {
125
+ walk(bytes, null);
126
+ }
127
+ function walk(bytes, out) {
128
+ const length = bytes.byteLength;
129
+ if (length === 0) return;
130
+ const view = new DataView(bytes.buffer, bytes.byteOffset, length);
131
+ if (length < 4) throw overrun(bytes, 0, 4, "the cookie");
132
+ const cookie = view.getUint32(0, true);
133
+ let count;
134
+ let runFlagsAt = -1;
135
+ let pos;
136
+ if (cookie === SERIAL_COOKIE_NO_RUNCONTAINER) {
137
+ if (length < 8) throw overrun(bytes, 4, 4, "the container count");
138
+ count = view.getUint32(4, true);
139
+ pos = 8;
140
+ } else if ((cookie & 65535) === SERIAL_COOKIE) {
141
+ count = (cookie >>> 16) + 1;
142
+ runFlagsAt = 4;
143
+ const flagBytes = count + 7 >>> 3;
144
+ if (runFlagsAt + flagBytes > length) {
145
+ throw overrun(bytes, runFlagsAt, flagBytes, "the run-container bitmap");
146
+ }
147
+ pos = runFlagsAt + flagBytes;
148
+ } else {
149
+ throw new IntegrityError(
150
+ `portable roaring: unrecognized cookie 0x${cookie.toString(16)}, not a portable-format bitmap`
151
+ );
152
+ }
153
+ const descriptiveAt = pos;
154
+ if (pos + count * 4 > length) throw overrun(bytes, pos, count * 4, "the descriptive header");
155
+ pos += count * 4;
156
+ let offsetsAt = -1;
157
+ if (runFlagsAt < 0 || count >= NO_OFFSET_THRESHOLD) {
158
+ if (pos + count * 4 > length) throw overrun(bytes, pos, count * 4, "the offset header");
159
+ offsetsAt = pos;
160
+ pos += count * 4;
161
+ }
162
+ let previousKey = -1;
163
+ for (let i = 0; i < count; i++) {
164
+ const key = view.getUint16(descriptiveAt + i * 4, true);
165
+ if (key <= previousKey) {
166
+ throw new IntegrityError(
167
+ `portable roaring: container keys are not strictly ascending (${previousKey} then ${key} at index ${i})`
168
+ );
169
+ }
170
+ previousKey = key;
171
+ const cardinality = view.getUint16(descriptiveAt + i * 4 + 2, true) + 1;
172
+ const isRun = runFlagsAt >= 0 && (bytes[runFlagsAt + (i >>> 3)] & 1 << (i & 7)) !== 0;
173
+ const kind = isRun ? RUN : cardinality > ARRAY_MAX_CARDINALITY ? BITMAP : ARRAY;
174
+ if (offsetsAt >= 0) {
175
+ const stated = view.getUint32(offsetsAt + i * 4, true);
176
+ if (stated !== pos) {
177
+ throw new IntegrityError(
178
+ `portable roaring: the offset header puts container ${i} at byte ${stated}, but it starts at ${pos}`
179
+ );
180
+ }
181
+ }
182
+ let size;
183
+ if (kind === ARRAY) {
184
+ size = cardinality * 2;
185
+ if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (array)`);
186
+ checkArray(view, pos, cardinality, i);
187
+ } else if (kind === BITMAP) {
188
+ size = BITMAP_CONTAINER_BYTES;
189
+ if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (bitmap)`);
190
+ const bits = popcount(view, pos);
191
+ if (bits !== cardinality) {
192
+ throw new IntegrityError(
193
+ `portable roaring: container ${i} (bitmap) holds ${bits} values, but its header says ${cardinality}`
194
+ );
195
+ }
196
+ } else {
197
+ if (pos + 2 > length) throw overrun(bytes, pos, 2, `container ${i} (run count)`);
198
+ const runs = view.getUint16(pos, true);
199
+ size = 2 + runs * 4;
200
+ if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (run)`);
201
+ const covered = checkRuns(view, pos + 2, runs, i);
202
+ if (covered !== cardinality) {
203
+ throw new IntegrityError(
204
+ `portable roaring: container ${i} (run) covers ${covered} values, but its header says ${cardinality}`
205
+ );
206
+ }
207
+ }
208
+ out?.push({ key, cardinality, kind, offset: pos });
209
+ pos += size;
210
+ }
211
+ }
212
+ function checkArray(view, at, cardinality, index) {
213
+ let previous = -1;
214
+ let p = at;
215
+ const end = at + cardinality * 2;
216
+ for (; p + 4 <= end; p += 4) {
217
+ const pair = view.getUint32(p, true);
218
+ const first = pair & 65535;
219
+ const second = pair >>> 16;
220
+ if (first <= previous || second <= first) throw unordered(index, (p - at) / 2);
221
+ previous = second;
222
+ }
223
+ if (p < end && view.getUint16(p, true) <= previous) throw unordered(index, (p - at) / 2);
224
+ }
225
+ function unordered(index, element) {
226
+ return new IntegrityError(
227
+ `portable roaring: container ${index} (array) is not strictly ascending at or after element ${element}`
228
+ );
229
+ }
230
+ function checkRuns(view, at, runs, index) {
231
+ let covered = 0;
232
+ let lowest = 0;
233
+ for (let r = 0; r < runs; r++) {
234
+ const run = view.getUint32(at + r * 4, true);
235
+ const start = run & 65535;
236
+ const last = start + (run >>> 16);
237
+ if (start < lowest) {
238
+ throw new IntegrityError(
239
+ `portable roaring: container ${index} (run) has run ${r} out of order, overlapping or touching the one before`
240
+ );
241
+ }
242
+ if (last > MAX_LOW) {
243
+ throw new IntegrityError(
244
+ `portable roaring: container ${index} (run) has run ${r} running past the end of the container`
245
+ );
246
+ }
247
+ covered += last - start + 1;
248
+ lowest = last + 2;
249
+ }
250
+ return covered;
251
+ }
252
+ function popcount(view, at) {
253
+ let bits = 0;
254
+ for (let p = at, end = at + BITMAP_CONTAINER_BYTES; p < end; p += 4) {
255
+ let w = view.getUint32(p, true);
256
+ w -= w >>> 1 & 1431655765;
257
+ w = (w & 858993459) + (w >>> 2 & 858993459);
258
+ bits += Math.imul(w + (w >>> 4) & 252645135, 16843009) >>> 24;
259
+ }
260
+ return bits;
261
+ }
262
+
263
+ // src/roaring-codec.ts
264
+ var { RoaringBitmap32, SerializationFormat, DeserializationFormat } = roaring;
265
+ var SafeBitmap = class _SafeBitmap {
266
+ bitmap;
267
+ constructor(bitmap) {
268
+ this.bitmap = bitmap;
269
+ }
270
+ static empty() {
271
+ return new _SafeBitmap(new RoaringBitmap32());
272
+ }
273
+ static fromValues(values) {
274
+ return new _SafeBitmap(new RoaringBitmap32(values));
275
+ }
276
+ /**
277
+ * Cap the size, check the structure, then deserialize with the **portable** format.
278
+ * Throws `IntegrityError` if the input exceeds `maxBytes`, is not a well-formed portable bitmap, or fails to
279
+ * decode — the native addon is never handed unbounded, malformed or unsafe-format input.
280
+ *
281
+ * The structural check is what makes the decoded bitmap's answers mean anything. The native deserializer
282
+ * bounds its reads and checks nothing else, so bytes of the right length and the wrong shape (containers or
283
+ * values out of order, overlapping runs, a cardinality that disagrees with the bits) decode into a bitmap
284
+ * whose `maximum()`, `has()` and `size` are wrong, and some of those shapes crash the process when used.
285
+ */
286
+ static safeDeserialize(bytes, maxBytes) {
287
+ if (bytes.length > maxBytes) {
288
+ throw new IntegrityError2(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
289
+ }
290
+ checkPortableLayout(bytes);
291
+ try {
292
+ return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
293
+ } catch (err) {
294
+ throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
295
+ }
296
+ }
297
+ serialize() {
298
+ return this.bitmap.serialize(SerializationFormat.portable);
299
+ }
300
+ /**
301
+ * Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
302
+ * the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
303
+ *
304
+ * Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
305
+ * `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
306
+ * at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
307
+ * (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
308
+ * whichever encoding is smaller per container.
309
+ *
310
+ * Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
311
+ * (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
312
+ * a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
313
+ * columns; don't carry a figure between them without checking which denominator it had.
314
+ *
315
+ * Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
316
+ * path deliberately does not.
317
+ */
318
+ optimize() {
319
+ this.bitmap.runOptimize();
320
+ }
321
+ /**
322
+ * Largest value, or `undefined` when empty. O(1) — roaring reads it off the last container, so the engine's
323
+ * per-chunk range assertion costs one call per chunk rather than a walk per id. The last container holds the
324
+ * largest value only because {@link SafeBitmap.safeDeserialize} refuses bytes whose containers or values are
325
+ * out of order.
326
+ */
327
+ maximum() {
328
+ return this.bitmap.isEmpty ? void 0 : this.bitmap.maximum();
329
+ }
330
+ add(value) {
331
+ this.bitmap.add(value);
332
+ }
333
+ addMany(values) {
334
+ this.bitmap.addMany(values);
335
+ }
336
+ remove(value) {
337
+ this.bitmap.remove(value);
338
+ }
339
+ removeMany(values) {
340
+ this.bitmap.removeMany(values);
341
+ }
342
+ has(value) {
343
+ return this.bitmap.has(value);
344
+ }
345
+ get size() {
346
+ return this.bitmap.size;
347
+ }
348
+ get isEmpty() {
349
+ return this.bitmap.isEmpty;
350
+ }
351
+ clone() {
352
+ return new _SafeBitmap(this.bitmap.clone());
353
+ }
354
+ // The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
355
+ // single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
356
+ // `bitmap` (accessible on same-class instances) for the native op.
357
+ /** In-place union: `this = this ∪ other`. */
358
+ orInPlace(other) {
359
+ this.bitmap.orInPlace(other.bitmap);
360
+ }
361
+ /** In-place difference: `this = this \ other`. */
362
+ andNotInPlace(other) {
363
+ this.bitmap.andNotInPlace(other.bitmap);
364
+ }
365
+ /** In-place intersection: `this = this ∩ other`. */
366
+ andInPlace(other) {
367
+ this.bitmap.andInPlace(other.bitmap);
368
+ }
369
+ /** Ascending iterator over the set values. */
370
+ [Symbol.iterator]() {
371
+ return this.bitmap[Symbol.iterator]();
372
+ }
373
+ toArray() {
374
+ return this.bitmap.toArray();
375
+ }
376
+ };
377
+ var roaringCodec = {
378
+ empty: () => SafeBitmap.empty(),
379
+ fromValues: (values) => SafeBitmap.fromValues(values),
380
+ safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
381
+ };
382
+
383
+ // src/system-clock.ts
384
+ var SystemClock = class {
385
+ now() {
386
+ return Date.now();
387
+ }
388
+ sleep(ms) {
389
+ if (ms <= 0) return Promise.resolve();
390
+ return new Promise((resolve) => {
391
+ setTimeout(resolve, ms);
392
+ });
393
+ }
394
+ /**
395
+ * `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
396
+ *
397
+ * A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
398
+ * co-resident HTTP server keeps answering. Measured on the **synthetic yield-primitive benchmark** — a
399
+ * 61,035-iteration loop yielding every 1,024, isolating the primitive rather than timing a real load (the
400
+ * end-to-end figures are 519 ms → 24 ms; see `cooperative.ts`, which lists all three experiments):
401
+ *
402
+ * ```text
403
+ * no yield 568 ms wall 568.0 ms worst event-loop gap
404
+ * sleep(0) 555 ms wall 555.2 ms ← a microtask. Yields NOTHING.
405
+ * setImmediate 569 ms wall 13.8 ms ← free, and 41x less starvation
406
+ * setTimeout(1) 625 ms wall 15.0 ms ← +10% wall for no extra relief
407
+ * ```
408
+ *
409
+ * `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
410
+ * before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
411
+ * callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
412
+ * server wants — inbound requests are serviced before the loop resumes chewing.
413
+ *
414
+ * A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
415
+ * "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
416
+ */
417
+ yieldNow() {
418
+ return new Promise((resolve) => {
419
+ setImmediate(resolve);
420
+ });
421
+ }
422
+ };
423
+
424
+ // src/option-keys.ts
425
+ var OPTION_KEYS = {
426
+ top: ["storage", "cache", "encryption", "retry", "metrics", "budget", "seams"],
427
+ cache: ["maxChunks", "ttlMs", "genTtlMs", "readerMax", "readerMaxBytes"],
428
+ encryption: ["keystore", "required"],
429
+ retry: ["maxAttempts", "baseDelayMs", "maxDelayMs", "backoffFactor", "jitter", "onRetry"],
430
+ budget: ["maxRequests"],
431
+ seams: ["clock", "rng"]
432
+ };
433
+
434
+ // src/index.ts
435
+ import {
436
+ MemoryStorage,
437
+ LocalFsStorage,
438
+ CrbmStorageChunkSource as CrbmStorageChunkSource2,
439
+ destroySegment,
440
+ eraseNamespace,
441
+ excludingReservedRows as excludingReservedRows2,
442
+ readRetentionPolicy,
443
+ MIN_EXPIRES_AT_MS as MIN_EXPIRES_AT_MS2,
444
+ InProcessKeystore,
445
+ NodeAead,
446
+ aadFor,
447
+ CloudRoaringError,
448
+ ValidationError as ValidationError3,
449
+ WriteConflictError as WriteConflictError2,
450
+ IntegrityError as IntegrityError3,
451
+ NotFoundError,
452
+ UnsupportedError as UnsupportedError2,
453
+ CapabilityError,
454
+ TransientError,
455
+ KeyUnavailableError,
456
+ BudgetExceededError,
457
+ isCloudRoaringError,
458
+ isWriteConflictError,
459
+ isTransientError,
460
+ isNotFoundError,
461
+ isIntegrityError,
462
+ isValidationError,
463
+ CrbmReader,
464
+ BufferReader,
465
+ CountingMetricsSink,
466
+ RecordingAuditSink,
467
+ DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2,
468
+ AWS_US_EAST_1_ONDEMAND,
469
+ ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
470
+ ONE_REDIS_HA_CLUSTER
471
+ } from "@cloudbitmaps/core";
472
+ var SystemRng = class {
473
+ /** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
474
+ next() {
475
+ return Math.random();
476
+ }
477
+ };
478
+ var DEFAULT_CACHE_MAX_CHUNKS = 1024;
479
+ var DEFAULT_ADMIN_CONCURRENCY = 8;
480
+ function validateConcurrency(concurrency) {
481
+ if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
482
+ throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
483
+ }
484
+ }
485
+ function requireScope(options, op) {
486
+ refuseReservedNamespace(options.namespace);
487
+ if (options.namespace === void 0 && options.allNamespaces !== true) {
488
+ throw new ValidationError2(
489
+ `${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
490
+ );
491
+ }
492
+ }
493
+ function resolveStorageSource(options, clock) {
494
+ const storage = options.storage;
495
+ if (storage === null || typeof storage !== "object") {
496
+ throw new ValidationError2(
497
+ "`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
498
+ );
499
+ }
500
+ const asBackend = storage;
501
+ const isObject = (v) => v !== null && typeof v === "object";
502
+ const hasGetChunk = typeof storage.getChunk === "function";
503
+ const hasPutImmutable = typeof storage.putImmutable === "function";
504
+ const isBackend = isStorageBackend(storage);
505
+ if (isBackend && (hasGetChunk || hasPutImmutable)) {
506
+ throw new ValidationError2(
507
+ `\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
508
+ );
509
+ }
510
+ const hasStorageHalf = isObject(asBackend.storage);
511
+ const hasRegistryHalf = isObject(asBackend.registry);
512
+ if (!isBackend && hasPutImmutable && hasRegistryHalf) {
513
+ throw new ValidationError2(
514
+ "`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
515
+ );
516
+ }
517
+ if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
518
+ const storageOk = typeof asBackend.storage.putImmutable === "function";
519
+ const registryOk = typeof asBackend.registry.compareAndSwap === "function";
520
+ if (!storageOk || !registryOk) {
521
+ const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
522
+ throw new ValidationError2(
523
+ "`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
524
+ );
525
+ }
526
+ throw new ValidationError2(
527
+ "`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
528
+ );
529
+ }
530
+ if (isBackend) {
531
+ const backend = storage;
532
+ return {
533
+ source: new CrbmStorageChunkSource(backend.storage, {
534
+ registry: backend.registry,
535
+ keystore: options.encryption?.keystore,
536
+ requireEncryption: options.encryption?.required,
537
+ clock,
538
+ currentGenTtlMs: options.cache?.genTtlMs,
539
+ maxOpenSegments: options.cache?.readerMax,
540
+ maxOpenIndexBytes: options.cache?.readerMaxBytes
541
+ }),
542
+ driver: backend.storage,
543
+ registry: backend.registry
544
+ };
545
+ }
546
+ if (hasGetChunk && hasPutImmutable) {
547
+ throw new ValidationError2(
548
+ "`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
549
+ );
550
+ }
551
+ if (!hasGetChunk && !hasPutImmutable) {
552
+ throw new ValidationError2(
553
+ "`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
554
+ );
555
+ }
556
+ if (hasGetChunk) {
557
+ if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
558
+ throw new ValidationError2(
559
+ "keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
560
+ );
561
+ }
562
+ return { source: storage, driver: void 0, registry: void 0 };
563
+ }
564
+ const driver = storage;
565
+ return {
566
+ source: new CrbmStorageChunkSource(driver, {
567
+ keystore: options.encryption?.keystore,
568
+ requireEncryption: options.encryption?.required,
569
+ clock,
570
+ currentGenTtlMs: options.cache?.genTtlMs,
571
+ maxOpenSegments: options.cache?.readerMax,
572
+ maxOpenIndexBytes: options.cache?.readerMaxBytes
573
+ }),
574
+ driver,
575
+ registry: void 0
576
+ };
577
+ }
578
+ var NOT_A_GROUP = {
579
+ Null: "null",
580
+ Array: "an array",
581
+ Map: "a Map",
582
+ Set: "a Set",
583
+ Boolean: "a boxed boolean",
584
+ Number: "a boxed number",
585
+ String: "a boxed string"
586
+ };
587
+ var CloudRoaring = class _CloudRoaring {
588
+ engine;
589
+ cache;
590
+ /** The store's retry options, for a pinned engine to read through as the store's own does; undefined when off. */
591
+ retryOptions;
592
+ crbmSource;
593
+ clock;
594
+ metrics;
595
+ // The store's own drivers, kept so the lifecycle helpers and the `*Into` verbs reuse them instead of making
596
+ // you re-pass deps. `storageDriver` is set only when `storage` was a raw IStorageDriver (a pre-built StorageChunkSource has
597
+ // no underlying driver to write through).
598
+ storageDriver;
599
+ registry;
600
+ keystore;
601
+ requireEncryption;
602
+ /** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
603
+ budget;
604
+ /**
605
+ * Refuse any option key the store does not take, at the top level or inside a group, naming each one.
606
+ *
607
+ * An option the store ignored would do nothing and look as if it had — a typo'd `cache.maxChunk`, a key from
608
+ * another library's config spread into this one — so every key is checked against {@link OPTION_KEYS}.
609
+ */
610
+ static rejectUnknownOptions(options) {
611
+ if (options === null || options === void 0 || typeof options !== "object") {
612
+ throw new ValidationError2(
613
+ "CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
614
+ );
615
+ }
616
+ const bag = options;
617
+ const unknown = [];
618
+ for (const key of Object.keys(bag)) {
619
+ if (!OPTION_KEYS.top.includes(key)) unknown.push({ group: null, key });
620
+ }
621
+ const groups = Object.keys(OPTION_KEYS).filter((g) => g !== "top");
622
+ for (const group of groups) {
623
+ const value = bag[group];
624
+ if (value === void 0) continue;
625
+ const offOk = group === "retry" || group === "budget";
626
+ if (value === false && offOk) continue;
627
+ const tag = Object.prototype.toString.call(value).slice(8, -1);
628
+ if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
629
+ const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
630
+ throw new ValidationError2(
631
+ `CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
632
+ );
633
+ }
634
+ const known = OPTION_KEYS[group];
635
+ for (const key of Object.keys(value)) if (!known.includes(key)) unknown.push({ group, key });
636
+ }
637
+ if (unknown.length === 0) return;
638
+ const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
639
+ const named = [...new Set(unknown.map((u) => u.group))];
640
+ throw new ValidationError2(
641
+ `CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
642
+ (g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
643
+ ).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
644
+ );
645
+ }
646
+ constructor(options) {
647
+ _CloudRoaring.rejectUnknownOptions(options);
648
+ const clock = options.seams?.clock ?? new SystemClock();
649
+ const rng = options.seams?.rng ?? new SystemRng();
650
+ const metrics = safeMetrics(options.metrics ?? NOOP_METRICS);
651
+ const cache = new BoundedLru({
652
+ maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
653
+ ttlMs: options.cache?.ttlMs,
654
+ clock
655
+ });
656
+ const resolved = resolveStorageSource(options, clock);
657
+ let storage = resolved.source;
658
+ if (options.retry !== false) {
659
+ const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
660
+ const policy = {
661
+ maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
662
+ baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
663
+ maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
664
+ backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
665
+ jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
666
+ };
667
+ const retryOpts = {
668
+ clock,
669
+ rng,
670
+ policy,
671
+ // Bridge transient-fault retries into the metrics stream, then call the user's own hook.
672
+ onRetry: (info) => {
673
+ metrics.onEvent({
674
+ kind: "retry",
675
+ reason: "transient",
676
+ attempt: info.attempt,
677
+ delayMs: info.delayMs
678
+ });
679
+ userOnRetry?.(info);
680
+ }
681
+ };
682
+ storage = new RetryingStorageChunkSource(storage, retryOpts);
683
+ this.retryOptions = retryOpts;
684
+ }
685
+ this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
686
+ const deps = {
687
+ storage,
688
+ cache,
689
+ codec: roaringCodec,
690
+ // the facade injects the flagship codec; core stays codec-agnostic
691
+ clock,
692
+ metrics,
693
+ budget: this.budget
694
+ };
695
+ this.engine = new SegmentEngine(deps);
696
+ this.cache = cache;
697
+ this.crbmSource = resolved.source instanceof CrbmStorageChunkSource ? resolved.source : void 0;
698
+ this.clock = clock;
699
+ this.metrics = metrics;
700
+ this.storageDriver = resolved.driver;
701
+ this.registry = resolved.registry;
702
+ this.keystore = options.encryption?.keystore;
703
+ this.requireEncryption = options.encryption?.required ?? false;
704
+ }
705
+ /**
706
+ * The write-side deps, from the store's own drivers, for the lifecycle helpers and the `*Into` verbs. Requires
707
+ * the store to have been constructed with a **backend**, which supplies both halves: an `IStorageDriver` to
708
+ * write generations through (a pre-built `StorageChunkSource` has none) and the registry holding the pointer
709
+ * every write publishes.
710
+ */
711
+ lifecycleDeps(op) {
712
+ if (this.storageDriver === void 0) {
713
+ throw new UnsupportedError(
714
+ `${op} needs the store built with a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A pre-built StorageChunkSource is read-only: it has no IStorageDriver underneath to write generations through. Build the store on one of those backend classes to write.`
715
+ );
716
+ }
717
+ if (this.registry === void 0) {
718
+ throw new UnsupportedError(
719
+ `${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
720
+ );
721
+ }
722
+ return {
723
+ storage: this.storageDriver,
724
+ registry: this.registry,
725
+ clock: this.clock,
726
+ codec: roaringCodec,
727
+ // facade injects the flagship codec
728
+ keystore: this.keystore,
729
+ requireEncryption: this.requireEncryption
730
+ };
731
+ }
732
+ /** Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). */
733
+ segment(name, options) {
734
+ const ref = { segment: name, namespace: options?.namespace };
735
+ validateSegmentRef(ref);
736
+ refuseReservedNamespace(ref.namespace);
737
+ const expiresAt = options?.expiresAt;
738
+ if (expiresAt !== void 0) {
739
+ if (!Number.isFinite(expiresAt) || !Number.isInteger(expiresAt) || expiresAt < MIN_EXPIRES_AT_MS) {
740
+ throw new ValidationError2(
741
+ `segment: expiresAt must be an integer epoch-MILLISECONDS >= ${MIN_EXPIRES_AT_MS}; got ${expiresAt} (a value in seconds lands in 1970 and would make the segment read as permanently empty)`
742
+ );
743
+ }
744
+ }
745
+ return makeSegment({
746
+ engine: this.engine,
747
+ ref,
748
+ clock: this.clock,
749
+ metrics: this.metrics,
750
+ materialize: (dest, ids, op, options2) => this.materialize(dest, ids, op, options2),
751
+ pinned: (r, e) => this.pinSegment(r, e),
752
+ combineEngine: (handles) => this.engineForCombine(handles),
753
+ expiresAt
754
+ });
755
+ }
756
+ /**
757
+ * Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
758
+ * materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
759
+ * publish on the row's token; a guarded load (the default, since the empty refusal reads the current generation)
760
+ * also fences on the pointer it judged (`expectFrom`), and one that found no row fences on that absence instead.
761
+ * Only an unguarded load (`allowEmpty: true` and no `guard.minRetained`) onto a segment with no row publishes bare
762
+ * forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
763
+ * within `cache.genTtlMs`).
764
+ *
765
+ * This routes through `loadSegment` rather than writing the generation itself, and that is the whole point
766
+ * of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
767
+ * everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
768
+ * guard runs while the old generation is still authoritative, the publish is fenced as above, and a refused
769
+ * object is reclaimed only when a re-read finds the row unchanged (the same token) or gone (hard invariant 1:
770
+ * deleting it after a purge-and-recreate would put a live row over a missing generation).
771
+ *
772
+ * That last check narrows the window rather than closing it: the row read and the delete are two round
773
+ * trips, and `IStorageDriver` has no conditional delete to make them one. the collection a load runs carries
774
+ * the same residual. The failure it leaves is an orphan object, which costs storage until
775
+ * something collects it — deliberately the cheaper side of the trade.
776
+ *
777
+ * Written and published in one step, with no guard, an empty combine — a typo'd operand, an `exclude` that
778
+ * swallowed everything, an operand that had not loaded yet — would silently replace `dest` with an empty
779
+ * generation: the same failure `load()`'s guard exists to prevent, on the same data, reachable without passing
780
+ * any option at all.
781
+ *
782
+ * **A lost race throws.** `loadSegment` reports one as `reason: 'superseded'`; the `*Into` verbs throw
783
+ * {@link WriteConflictError} for it, so a caller can `catch (WriteConflictError)`. That one refusal is
784
+ * translated into the throw, and `MaterializeResult.reason` never carries it.
785
+ *
786
+ * **A `WriteConflictError` does not by itself mean nothing was published**, and that is worth knowing
787
+ * before you write the retry. `'superseded'` covers five different causes — the write-once PUT collided,
788
+ * the pointer moved, a row appeared where the load found none, the row's token changed, the row was purged —
789
+ * and only the first three are the "somebody beat us" the name suggests. A token can also change on a write
790
+ * that is not a supersession at all, such as a `setRetention` on the destination. On top of that, the collection pass that runs AFTER a
791
+ * successful publish can raise the same error. So: treat it as "re-read the destination and decide",
792
+ * never as "the write did not happen".
793
+ *
794
+ * **And by default it collects nothing**, unlike `load()`. See the `keep` default below.
795
+ */
796
+ async materialize(dest, ids, op, options) {
797
+ const deps = this.lifecycleDeps(op);
798
+ let result;
799
+ try {
800
+ result = await loadSegment(dest, ids, deps, {
801
+ ...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
802
+ ...options?.guard === void 0 ? {} : { guard: options.guard },
803
+ // COLLECT NOTHING by default, which `loadSegment` does not — it keeps a grace window of 1 and deletes
804
+ // the rest. By default a materialisation collects nothing: the guide states "**It deletes nothing**,
805
+ // unlike `load()`. The destination's previous generations stay in the bucket until you collect them", and
806
+ // the ownership table
807
+ // puts that call on the operator. Inheriting `load()`'s collection would silently delete the
808
+ // generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
809
+ // as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
810
+ keep: options?.keep ?? KEEP_EVERY_GENERATION,
811
+ ...options?.audit === void 0 ? {} : { audit: options.audit }
812
+ });
813
+ } finally {
814
+ this.engine.invalidate(dest);
815
+ }
816
+ const reason = result.reason;
817
+ if (reason === "superseded") {
818
+ const wrote = result.size > 0 ? `generation ${result.generation} was written and did not become current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
819
+ throw new WriteConflictError(
820
+ `${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so it never became current: ${wrote}. The pointer may have moved, the row may have been rewritten (a retention policy does this) or purged. Re-read the destination and re-run.`
821
+ );
822
+ }
823
+ return {
824
+ generation: result.generation,
825
+ published: result.published,
826
+ ...reason === void 0 ? {} : { reason },
827
+ cardinality: result.cardinality,
828
+ cardinalityBefore: result.cardinalityBefore,
829
+ chunkCount: result.chunkCount,
830
+ size: result.size,
831
+ collected: result.collected
832
+ };
833
+ }
834
+ /**
835
+ * **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?**
836
+ *
837
+ * Enumerates the **registered** segments (via the store's own `registry`) and does a `has(id)` on each — no
838
+ * drivers to re-pass. Complete only over registered segments (every loaded segment has a row, so build the
839
+ * store on the backend the loads used). There is deliberately **no `id → segments` reverse index** — that would
840
+ * tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
841
+ * a storage backend (throws {@link UnsupportedError} otherwise).
842
+ *
843
+ * **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
844
+ * compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
845
+ * read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
846
+ * The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
847
+ * generation 0 while this store held its old generation 0 is told apart too.
848
+ */
849
+ async subjectReport(id, options = {}) {
850
+ const registry = this.requireRegistry("subjectReport");
851
+ requireScope(options, "subjectReport");
852
+ validateConcurrency(options.concurrency);
853
+ const budget = resolvePerOpBudget(options.budget, this.budget);
854
+ splitId(id);
855
+ const recs = await collectWithinBudget(
856
+ // A subject cannot be in a due-index row, and charging this request's budget for them would refuse a
857
+ // GDPR Art. 15 report for a reason unrelated to the subject.
858
+ excludingReservedRows(registry.list(options.namespace)),
859
+ budget,
860
+ "subjectReport"
861
+ );
862
+ const membership = await mapWithConcurrency(
863
+ recs,
864
+ options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
865
+ async (rec) => {
866
+ if (rec.status === "destroyed") return null;
867
+ const ref = { segment: rec.segment, namespace: rec.namespace };
868
+ if (this.crbmSource !== void 0) {
869
+ const held = await this.crbmSource.currentVersion(ref);
870
+ const listed = rec.currentGen === null ? null : `${rec.currentGen}:${String(rec.token)}`;
871
+ if (held !== listed) this.engine.invalidate(ref);
872
+ }
873
+ return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
874
+ }
875
+ );
876
+ const segments = membership.filter((m) => m !== null);
877
+ return { id, segments, scannedSegments: recs.length };
878
+ }
879
+ /**
880
+ * **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
881
+ * from the bucket on return.**
882
+ *
883
+ * For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
884
+ * without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
885
+ * generation it was derived from**, and collects the generation that held the bit. A segment whose current
886
+ * generation does not hold the id is searched anyway, every generation in its bucket: a retained superseded one
887
+ * can still hold it, and so can one above the pointer after a {@link CloudRoaring.rollback}, which the rollback
888
+ * could make current again. Each holder is deleted — below the pointer by a `keep: 0` collection, above it one
889
+ * by one, re-proved against the row first — and the generations above the pointer that never held the id stay
890
+ * as rollback targets. An entry says `erased: true` only once the call has listed the bucket and read what is
891
+ * left: **no generation of the segment holds the id**. The returned per-segment record is your **erasure ledger** —
892
+ * persist it / route it to your audit sink as the proof of deletion (a `segment.rewrite` audit event is also
893
+ * emitted per rewrite when you pass `audit`).
894
+ *
895
+ * Uses the backend's **own** two halves, so the membership check and the rewrite provably run
896
+ * over the same generation. Requires the store built with a **backend** (throws
897
+ * {@link UnsupportedError} otherwise; a pre-built `StorageChunkSource` store has no `IStorageDriver` to write
898
+ * through; build the store on a backend instead).
899
+ *
900
+ * **One contract remains** (an integrator obligation the library cannot check): **do not load the segment
901
+ * while erasing from it.** A load that lands after the rewrite carries whatever its source held, and the
902
+ * library cannot know that source was meant to exclude the id. Quiesce loads of the affected segments for the
903
+ * duration, or fix the source first and load after. A writer that lands *during* the rewrite is caught: the
904
+ * rewrite's publish is refused **by the fence** — the publish lands its compare-and-swap only
905
+ * while the pointer is still on the generation the rewrite streamed — and the entry says `note: 'superseded'`,
906
+ * so re-run. Forward-only alone would NOT refuse it: the rewrite's number goes above everything in the bucket,
907
+ * so the rewrite would out-rank the newer generation and then collect it.
908
+ *
909
+ * A racing **erasure** is caught before that, and reported the same way. It collects with `keep: 0`, taking
910
+ * every generation below its new pointer — the one this rewrite is streaming, and the object this rewrite
911
+ * just wrote — so the loser can find its own inputs deleted mid-flight. That surfaces as an outcome rather than
912
+ * an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
913
+ * that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger, as a fresh
914
+ * call would leave it out.
915
+ *
916
+ * **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
917
+ * another writer (a load, another erasure, or a rollback) moved the pointer mid-call, so **this call** did not
918
+ * erase the id. Re-run: it erases the id if it is still there, and lists nothing for the segment if a racing
919
+ * erasure of the same id already removed it. Do not read `'superseded'` as "the id is still present" —
920
+ * read it as "not done by this call, and the re-run settles it".
921
+ * `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
922
+ * on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
923
+ * it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
924
+ * absent from every read and what remains is an object in the bucket that still contains the bit. The same
925
+ * note, after a publish, can mean a rollback moved the pointer back onto a generation that still holds the id
926
+ * while the collection ran — then the id is served again, and the entry says so rather than `erased: true`. A
927
+ * re-run searches every generation in the bucket, so it rewrites or collects what is left and usually reports
928
+ * `erased: true`; {@link SubjectErasureEntry.note} says what else a re-run can report. Re-running is otherwise safe
929
+ * and idempotent: a segment the id is no longer in is not listed. Admin-only path;
930
+ * `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
931
+ * the id is searched generation by generation, one open for each other generation in its bucket, and with the
932
+ * keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
933
+ * `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
934
+ * so a fleet with long histories can exhaust it where the segment count alone would not. A segment that does is
935
+ * reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
936
+ * rewrite is the last check for a generation a concurrent writer left behind), and the
937
+ * rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
938
+ * co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
939
+ * is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
940
+ */
941
+ async eraseSubject(id, options = {}) {
942
+ const deps = this.lifecycleDeps("eraseSubject");
943
+ requireScope(options, "eraseSubject");
944
+ validateConcurrency(options.concurrency);
945
+ const budget = resolvePerOpBudget(options.budget, this.budget);
946
+ splitId(id);
947
+ const recs = await collectWithinBudget(
948
+ excludingReservedRows(deps.registry.list(options.namespace)),
949
+ budget,
950
+ "eraseSubject"
951
+ );
952
+ let opens = 0;
953
+ const charge = () => {
954
+ opens += 1;
955
+ checkBudget(budget, recs.length + opens, "eraseSubject");
956
+ };
957
+ const entries = await mapWithConcurrency(
958
+ recs,
959
+ options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
960
+ async (rec) => {
961
+ if (rec.status === "destroyed") return null;
962
+ const ref = { segment: rec.segment, namespace: rec.namespace };
963
+ try {
964
+ const result = await eraseIdFromSegment(
965
+ ref,
966
+ id,
967
+ budget === null ? deps : { ...deps, storage: new OpenChargingStorage(deps.storage, rec.currentGen, charge) },
968
+ { audit: options.audit }
969
+ );
970
+ if (result.reason === "not-member" || result.reason === "absent") return null;
971
+ if (result.reason === "no-generation") return null;
972
+ if (result.reason === "destroyed") return null;
973
+ return {
974
+ segment: rec.segment,
975
+ namespace: rec.namespace,
976
+ erased: result.erased,
977
+ fromGeneration: result.fromGeneration,
978
+ generation: result.generation,
979
+ note: result.erased ? void 0 : result.reason
980
+ };
981
+ } catch (err) {
982
+ return {
983
+ segment: rec.segment,
984
+ namespace: rec.namespace,
985
+ erased: false,
986
+ note: `error: ${err instanceof Error ? err.message : String(err)}`
987
+ };
988
+ } finally {
989
+ try {
990
+ this.engine.invalidate(ref);
991
+ } catch {
992
+ }
993
+ }
994
+ }
995
+ );
996
+ const erasedFrom = entries.filter((e) => e !== null);
997
+ return { id, erasedFrom, scannedSegments: recs.length };
998
+ }
999
+ /**
1000
+ * Every generation still in the bucket for this segment, ascending, with the current one marked.
1001
+ *
1002
+ * What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
1003
+ * the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
1004
+ * choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
1005
+ *
1006
+ * Needs a backend.
1007
+ */
1008
+ async generations(ref) {
1009
+ validateSegmentRef(ref);
1010
+ return listGenerations(ref, this.lifecycleDeps("generations"));
1011
+ }
1012
+ /**
1013
+ * Whether a read of this segment would find anything — the "do I already have this?" question, as one
1014
+ * registry point read.
1015
+ *
1016
+ * There is no `create` in this library: {@link CloudRoaring.segment} is a validated address and does no I/O,
1017
+ * so naming a segment can never collide with an existing one. A segment starts existing when something is
1018
+ * first loaded into it, and this is how you ask whether that has happened.
1019
+ *
1020
+ * **Not the same as `count() > 0`.** A segment loaded with no ids exists and counts zero. Distinguishing
1021
+ * "never loaded" from "loaded, and genuinely empty" is the thing `count()` cannot do and the reason this
1022
+ * exists. It is `false` for a segment whose row was minted ahead of its first load (by `setRetention`) and
1023
+ * for a `destroyed` tombstone, because a read answers empty in both cases.
1024
+ *
1025
+ * Two states answer `true` where a read still gives you nothing: a torn restore (a live pointer whose object
1026
+ * was deleted) makes reads *throw* rather than answer empty — `checkConsistency` is the call for that — and
1027
+ * a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
1028
+ *
1029
+ * Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
1030
+ * `load`'s `guard`, or `expectFrom`/`expectToken` on a publish. Needs a storage backend.
1031
+ *
1032
+ * ```ts
1033
+ * if (!(await store.exists({ segment: 'users' }))) {
1034
+ * await store.load({ segment: 'users' }, idsFromUpstream);
1035
+ * }
1036
+ * ```
1037
+ */
1038
+ async exists(ref) {
1039
+ validateSegmentRef(ref);
1040
+ return segmentExists(ref, this.requireRegistry("exists"));
1041
+ }
1042
+ /**
1043
+ * Every segment the registry knows about, streamed — optionally scoped to one namespace.
1044
+ *
1045
+ * The registry is already the list of your segments, which is why you should not keep a second one beside it:
1046
+ * a hand-maintained list is a source of truth that drifts from this one the first time a load fails halfway.
1047
+ *
1048
+ * **An admin/discovery call, not a request-path one.** This is the registry's own enumeration — a paged
1049
+ * LIST over the `registry/` prefix — so its cost grows with the size of the fleet rather than with what you
1050
+ * are looking for.
1051
+ *
1052
+ * Scoping to a namespace narrows the LIST prefix, so it really is the difference between reading one tenant
1053
+ * and reading all of them.
1054
+ *
1055
+ * It streams, and stopping the iteration stops the scan. It reads the registry directly and is not retried: a
1056
+ * transient fault part-way through ends the loop with that error, and calling `segments()` again scans from the
1057
+ * start.
1058
+ *
1059
+ * Yields `destroyed` tombstones and rows with `currentGen: null`, because a filtered enumeration that looks
1060
+ * complete is worse than an honest one — filter on `status`/`currentGen` yourself, or ask
1061
+ * {@link CloudRoaring.exists} the narrower question. Needs a storage backend.
1062
+ *
1063
+ * ```ts
1064
+ * for await (const s of store.segments({ namespace: 'active-daily' })) {
1065
+ * console.log(s.segment, s.currentGen, s.status);
1066
+ * }
1067
+ * ```
1068
+ */
1069
+ segments(options = {}) {
1070
+ if (options.namespace !== void 0) {
1071
+ validateSegmentRef({ segment: "x", namespace: options.namespace });
1072
+ refuseReservedNamespace(options.namespace);
1073
+ }
1074
+ return listSegments(this.requireRegistry("segments"), options);
1075
+ }
1076
+ /**
1077
+ * Move this segment's pointer **back** to a generation still in the bucket — the one write in the library that
1078
+ * is not forward-only.
1079
+ *
1080
+ * Forward-only is right for a writer: a load whose ids came from upstream loses nothing by being out-raced, and
1081
+ * letting it regress would let a slow loader silently undo a fast one. It is wrong for an operator who has
1082
+ * looked at the segment, decided the current generation is wrong, and knows which one they want. So this is
1083
+ * reachable only by asking for it by name — no sweep, retry or reconciliation calls it — and it is audited
1084
+ * (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
1085
+ * one cannot.
1086
+ *
1087
+ * It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
1088
+ * `NotFoundError` naming what *is* available, and a crypto-shredded segment throws
1089
+ * {@link ValidationError} because every generation of it is unreadable. Rolling to the generation already
1090
+ * current is a no-op that reports itself.
1091
+ *
1092
+ * It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
1093
+ * and are then *above* `currentGen`, where collection never looks. They remain until a load numbers above them
1094
+ * (collection then keeps the newest `keep` of what is below its pointer), {@link CloudRoaring.dropSegment}
1095
+ * deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and only those holding
1096
+ * the id when the current generation does not. An operator who has just undone a bad load should not have the
1097
+ * evidence collected out from under them, while a rollback target that still holds erased data would make the
1098
+ * erasure undoable.
1099
+ *
1100
+ * A target **above** the pointer needs `{ allowForward: true }`, and is refused with {@link ValidationError}
1101
+ * without it: that is also where objects live that were never published, such as a load that wrote its object
1102
+ * and died before the publish. Undoing an earlier rollback is what the opt-in is for.
1103
+ *
1104
+ * Needs a backend.
1105
+ */
1106
+ async rollback(ref, toGeneration, options = {}) {
1107
+ validateSegmentRef(ref);
1108
+ const deps = this.lifecycleDeps("rollback");
1109
+ try {
1110
+ return await rollbackSegment(ref, toGeneration, deps, options);
1111
+ } finally {
1112
+ this.engine.invalidate(ref);
1113
+ }
1114
+ }
1115
+ /**
1116
+ * **Replace this segment's contents** with `ids`, as one new immutable generation, and make it current.
1117
+ *
1118
+ * The whole write path in one call: take the next generation number, write the object, check the result is
1119
+ * plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
1120
+ * the one that gets left out is the last, so segments quietly accumulate superseded generations nobody pays
1121
+ * attention to and everybody pays for.
1122
+ *
1123
+ * ```ts
1124
+ * const r = await store.load({ segment: 'audience:active' }, idsFromWarehouse, {
1125
+ * guard: { minRetained: 0.5 }, // refuse a load that would drop more than half the segment
1126
+ * });
1127
+ * if (!r.published) console.warn(`load refused: ${r.reason}`);
1128
+ * ```
1129
+ *
1130
+ * **A load REPLACES.** Whatever the stream contains is what the segment contains afterwards, so an upstream
1131
+ * query that returns fewer rows than usual is a shrink nobody asked for and an empty one is a wipe — both
1132
+ * reported as a successful write, because at the storage layer they are one. That is what `guard` and the
1133
+ * default empty refusal are for, and why they run **between** the write and the publish: the only moment where
1134
+ * the new content is known and the old one is still authoritative.
1135
+ *
1136
+ * **Branch on `published`.** A refusal is a normal outcome, not an exception — `published: false` with a
1137
+ * `reason`, in the same shape as a success — so a discarded result is a load that silently did nothing, and
1138
+ * with the empty guard on by default this verb refuses more readily than any other. Note the sibling `*Into`
1139
+ * verbs *throw* on the same superseded condition rather than reporting it; this one follows
1140
+ * `eraseIdFromSegment` instead, because three of its four refusals are expected guard outcomes rather than
1141
+ * faults.
1142
+ *
1143
+ * `'superseded'` means another writer got there first: another load took the same generation number, so this
1144
+ * one wrote nothing (`size: 0`), or the segment's registry row changed while the load was writing — another load
1145
+ * published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
1146
+ *
1147
+ * A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
1148
+ * collection deliberately never looks — but only while the segment's registry row is unchanged or gone. Once
1149
+ * another write has changed the row, the generation number it holds may name another incarnation's live object,
1150
+ * so it leaves the orphan rather than risk deleting live data. The orphan is an ordinary generation once a later
1151
+ * one is current above it, and collection counts it within `keep`.
1152
+ *
1153
+ * What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
1154
+ * segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
1155
+ * that will not open when a guard reads its size (`IntegrityError`); a driver failure; and a collection pass
1156
+ * that could not prove the segment was still the same one (`WriteConflictError`). The last can be raised
1157
+ * **after** the publish already landed, so a throw does not by itself mean the load did not take effect —
1158
+ * re-read the pointer rather than assuming.
1159
+ *
1160
+ * Needs a backend (throws {@link UnsupportedError} otherwise).
1161
+ */
1162
+ async load(ref, ids, options = {}) {
1163
+ validateSegmentRef(ref);
1164
+ const deps = this.lifecycleDeps("load");
1165
+ try {
1166
+ return await loadSegment(ref, ids, deps, options);
1167
+ } finally {
1168
+ this.engine.invalidate(ref);
1169
+ }
1170
+ }
1171
+ /**
1172
+ * **Dispose of a segment — tombstone it, then delete its storage objects.** Irreversible.
1173
+ *
1174
+ * The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
1175
+ * backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
1176
+ * removes the storage and works on a cleartext segment. On an encrypted segment it does both.
1177
+ *
1178
+ * Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
1179
+ * matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
1180
+ * variable twice.
1181
+ *
1182
+ * ```ts
1183
+ * for (const day of expiredDays) {
1184
+ * // The family in the namespace, the date in the segment — `registry.list(namespace)` then enumerates
1185
+ * // exactly this family's buckets, which a flat `active:2026-08-01` name cannot do without string-matching.
1186
+ * const ref = { namespace: 'active-daily', segment: day };
1187
+ * await store.dropSegment(ref, { confirmSegment: ref.segment });
1188
+ * }
1189
+ * ```
1190
+ *
1191
+ * **Omitting the namespace addresses a different segment and is a silent no-op** — you get
1192
+ * `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
1193
+ * forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
1194
+ *
1195
+ * **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
1196
+ * reclaimed and the drop should be re-run. A load that was already writing when the tombstone landed still
1197
+ * finishes its object, so a single sweep can miss it — this call re-sweeps and then reports whatever it still
1198
+ * could not remove rather than returning a result that looks like a clean drop.
1199
+ *
1200
+ * Reads become empty within `cache.genTtlMs` (default 2 s), not instantly: a store that had already read this
1201
+ * segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
1202
+ * cannot be read because of a transient fault, until a retry 500 ms apart reaches it again. A reader that never
1203
+ * touched it sees empty at once. **That bound needs a registry and `cache.genTtlMs > 0`.** A store with no registry
1204
+ * (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a storage source built with no clock, has no timed
1205
+ * refresh. It notices the drop only when a read has to fetch
1206
+ * from a deleted generation or its reader cache evicts the segment, and until then a chunk it has cached
1207
+ * answers without reaching storage, so it can keep answering `true` for a dropped segment indefinitely. Tell
1208
+ * it with {@link invalidate}, or restart it.
1209
+ *
1210
+ * Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
1211
+ * because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
1212
+ */
1213
+ async dropSegment(ref, options) {
1214
+ validateSegmentRef(ref);
1215
+ const deps = this.lifecycleDeps("dropSegment");
1216
+ try {
1217
+ return await dropSegment(ref, { registry: deps.registry, storage: deps.storage }, options);
1218
+ } finally {
1219
+ if (options.dryRun !== true) this.engine.invalidate(ref);
1220
+ }
1221
+ }
1222
+ /**
1223
+ * **Record when this segment becomes eligible for retirement.** One registry write; nothing is deleted here,
1224
+ * and nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
1225
+ * an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
1226
+ * no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
1227
+ *
1228
+ * `expiresAt` is an **absolute epoch-ms you compute**, not a duration the library derives. A relative TTL would
1229
+ * have to be anchored to something the library knows — `updatedAt`, or the current generation — and every load
1230
+ * rewrites both, so "expire 30 days after the last write" would keep a busy daily bucket alive forever
1231
+ * precisely because it is being reloaded.
1232
+ *
1233
+ * ```ts
1234
+ * const DAY = 86_400_000;
1235
+ * const ref = { namespace: 'active-daily', segment: '2026-08-05' };
1236
+ * await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY });
1237
+ * ```
1238
+ *
1239
+ * **Works before the first load**: a segment with no registry row yet gets one (`createdRow: true` in the
1240
+ * result) with **no Storage generation**, so the policy is recorded ahead of the data and the segment is already
1241
+ * enumerable by the sweep; the first load then publishes onto that row.
1242
+ *
1243
+ * A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
1244
+ * buckets is normal. A value below `MIN_EXPIRES_AT_MS` (2001-09-09) is rejected: it is almost certainly epoch
1245
+ * **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
1246
+ * backend (throws {@link UnsupportedError} otherwise), and refuses a crypto-shredded segment.
1247
+ */
1248
+ async setRetention(ref, policy) {
1249
+ validateSegmentRef(ref);
1250
+ return setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
1251
+ }
1252
+ /**
1253
+ * **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
1254
+ * tombstone). Returns the string `'invalid'` for a row whose `expiresAt` is present but unusable — a
1255
+ * hand-edited row, or one from a restore — so a malformed policy is visible rather than silently reading as
1256
+ * "never expires" on a segment someone believes is expiring.
1257
+ */
1258
+ async getRetention(ref) {
1259
+ validateSegmentRef(ref);
1260
+ return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
1261
+ }
1262
+ /**
1263
+ * **Cancel a segment's expiry** so no sweep retires it. Returns whether a policy was actually removed (`false`
1264
+ * when there was none). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
1265
+ * to the setter is how a typo becomes a deletion.
1266
+ */
1267
+ async clearRetention(ref) {
1268
+ validateSegmentRef(ref);
1269
+ return clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
1270
+ }
1271
+ /**
1272
+ * **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
1273
+ * the policies `setRetention` records — and it is a **call, not a daemon**. Nothing schedules it; you run it from
1274
+ * whatever heartbeat your deployment already has (an EventBridge rule, a Kubernetes CronJob, a queue consumer,
1275
+ * the job that runs your loads). A library that started a timer would behave differently in a Lambda, an edge
1276
+ * isolate and a long-lived server, which is worse than not having one.
1277
+ *
1278
+ * Each retirement goes through `dropSegment`, so the registry → Storage ordering, the re-sweep for an object a
1279
+ * load was still writing, and the `generationsRemaining` report all come from one implementation rather than
1280
+ * two.
1281
+ *
1282
+ * ```ts
1283
+ * // In your scheduled handler. Start with a preview in a new deployment.
1284
+ * const preview = await store.retireExpired({ namespace: 'active-daily', dryRun: true });
1285
+ * console.log(`would retire ${preview.wouldRetire} of ${preview.scanned} (limited: ${preview.limited})`);
1286
+ *
1287
+ * const swept = await store.retireExpired({ namespace: 'active-daily' });
1288
+ * for (const e of swept.entries) {
1289
+ * if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`); // invalid-policy / limit / failed: …
1290
+ * if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
1291
+ * console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
1292
+ * }
1293
+ * }
1294
+ * if (swept.limited) scheduleAnotherPassSoon(); // more are still eligible
1295
+ * ```
1296
+ *
1297
+ * **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
1298
+ * sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
1299
+ * argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
1300
+ * `limit` (default 100) cut the pass short and more are eligible. That cap is charged on **attempts**, not
1301
+ * successes, which is what makes it a real bound: `dropSegment` writes the tombstone before sweeping Storage, so a
1302
+ * fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
1303
+ * wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
1304
+ * dashboard summing `retired` can never show a phantom deletion.
1305
+ *
1306
+ * It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
1307
+ * and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
1308
+ * first, since nothing else ever would for a tombstoned segment. Attribution is a **positive marker the sweep
1309
+ * stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
1310
+ * `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
1311
+ * exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
1312
+ * segment that held nothing has its row deleted in the pass that retires it, since that row would only fence the name.
1313
+ * Pass `purgeTombstones: false` to keep every tombstone, that row included.
1314
+ *
1315
+ * Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
1316
+ * because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
1317
+ */
1318
+ async retireExpired(options = {}) {
1319
+ const deps = this.lifecycleDeps("retireExpired");
1320
+ const result = await retireExpired(
1321
+ { registry: deps.registry, storage: deps.storage },
1322
+ { ...options, now: options.now ?? this.clock.now() }
1323
+ );
1324
+ if (options.dryRun !== true) {
1325
+ for (const entry of result.entries) {
1326
+ this.engine.invalidate({ segment: entry.segment, namespace: entry.namespace });
1327
+ }
1328
+ }
1329
+ return result;
1330
+ }
1331
+ /**
1332
+ * **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
1333
+ * through a path this store did not perform itself.
1334
+ *
1335
+ * A store keeps two layers of derived state — a resolved snapshot per segment (an open `.crbm` reader, plus
1336
+ * the DEK it unwrapped) and decoded chunks keyed by generation. Both are built for one event: **a publish
1337
+ * that advances `currentGen`**, which the snapshot TTL notices and the generation-keyed cache misses on.
1338
+ * Neither notices an event that *destroys* what they were derived from.
1339
+ *
1340
+ * The store's own writes handle themselves: `load`, `rollback`, `eraseSubject`, `dropSegment` and `retireExpired`
1341
+ * here, and a segment's `*Into` verbs, invalidate what they touch. This method is for the cases they cannot see:
1342
+ *
1343
+ * - **`destroySegment` / `eraseNamespace`**, which are free functions over raw drivers rather than methods
1344
+ * here, so a crypto-shred performed beside this store invalidates nothing in it. It keeps its open reader and
1345
+ * the unwrapped DEK, and decrypts with them, chunks it had never fetched before the shred included, until its
1346
+ * snapshot re-resolves: within `cache.genTtlMs` when the store has a registry and a `genTtlMs` above 0, and
1347
+ * otherwise only when its caches let the segment go or a read has to fetch from a generation a sweep collected.
1348
+ * A pin answers with no timed bound, until it is told or both caches let it go.
1349
+ * - **Another process.** Erasing on one box invalidates nothing on the others; each store bounds its own
1350
+ * staleness by `cache.genTtlMs`, and a store with no registry (a bare `IStorageDriver`), with
1351
+ * `cache.genTtlMs: 0`, or on a storage source built with no clock has no bound at all: it converges only when a read happens to miss its caches. If a compliance deadline depends on every
1352
+ * reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
1353
+ *
1354
+ * It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
1355
+ * object back, the pin reads it again.
1356
+ *
1357
+ * Synchronous, best-effort, and safe to call for a segment this store has never read.
1358
+ *
1359
+ * ```ts
1360
+ * await destroySegment(ref, { registry: backend.registry }, { confirmSegment: ref.segment });
1361
+ * store.invalidate(ref); // this process
1362
+ * await bus.publish('cloudbitmaps.invalidate', ref); // and every other one
1363
+ * ```
1364
+ */
1365
+ invalidate(ref) {
1366
+ validateSegmentRef(ref);
1367
+ refuseReservedNamespace(ref.namespace);
1368
+ this.engine.invalidate(ref);
1369
+ }
1370
+ /**
1371
+ * Build a handle held at the generation `ref` resolves to right now. See {@link Segment.pin}.
1372
+ *
1373
+ * The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
1374
+ * pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
1375
+ * those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
1376
+ */
1377
+ async pinSegment(ref, expiresAt) {
1378
+ const crbm = this.crbmSource;
1379
+ if (crbm === void 0) {
1380
+ throw new UnsupportedError(
1381
+ "pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
1382
+ );
1383
+ }
1384
+ const at = await this.withRetries(() => crbm.pinGeneration(ref));
1385
+ const pinnedAt = {
1386
+ generation: at?.generation ?? null,
1387
+ version: at?.version ?? null,
1388
+ fingerprint: at?.fingerprint ?? null
1389
+ };
1390
+ const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
1391
+ return makeSegment({
1392
+ engine: this.engineWithPins(pins),
1393
+ ref,
1394
+ clock: this.clock,
1395
+ metrics: this.metrics,
1396
+ materialize: (dest, ids, op, options) => this.materialize(dest, ids, op, options),
1397
+ pinned: (r, e) => this.pinSegment(r, e),
1398
+ combineEngine: (handles) => this.engineForCombine(handles),
1399
+ expiresAt,
1400
+ pinnedAt
1401
+ });
1402
+ }
1403
+ /**
1404
+ * An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
1405
+ * retries, so a transient fault on a pinned read is retried as it would be on a live one.
1406
+ *
1407
+ * It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment.
1408
+ */
1409
+ engineWithPins(pins) {
1410
+ const crbm = this.crbmSource;
1411
+ if (crbm === void 0) {
1412
+ throw new UnsupportedError(
1413
+ "pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
1414
+ );
1415
+ }
1416
+ const pinned = new PinnedStorageChunkSource(crbm, pins);
1417
+ return new SegmentEngine({
1418
+ storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource(pinned, this.retryOptions),
1419
+ cache: this.cache,
1420
+ codec: roaringCodec,
1421
+ clock: this.clock,
1422
+ metrics: this.metrics,
1423
+ budget: this.budget
1424
+ });
1425
+ }
1426
+ /**
1427
+ * The engine a combine should run on, given the handles involved.
1428
+ *
1429
+ * A pinned handle passed as an **operand** must still be read at its pin. Routing by "which handle the call
1430
+ * was made on" reads it live instead — `snap.intersect([other])` honours the pin while
1431
+ * `other.intersect([snap])` silently does not, and the two are the same question. So the combine collects
1432
+ * every pin in play and runs on an engine that honours all of them. With no pins anywhere this is the
1433
+ * store's own engine and costs nothing.
1434
+ *
1435
+ * One engine reads a segment at one generation, so a combine that holds the same segment at two — two pins of
1436
+ * it, or a pin and a live handle — is refused. Keeping one of them would answer the question for the wrong
1437
+ * instant, and say nothing: diffing two snapshots of a segment returned no ids at all.
1438
+ */
1439
+ engineForCombine(handles) {
1440
+ if (!handles.some((h) => h.pinnedAt !== void 0)) return void 0;
1441
+ const pins = /* @__PURE__ */ new Map();
1442
+ const held = /* @__PURE__ */ new Map();
1443
+ for (const h of handles) {
1444
+ const key = h.key();
1445
+ const at = h.pinnedAt;
1446
+ if (held.has(key)) {
1447
+ const before = held.get(key);
1448
+ if (!samePin(before, at)) {
1449
+ throw new ValidationError2(
1450
+ `the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`intersectInto(dest, [])\``
1451
+ );
1452
+ }
1453
+ } else {
1454
+ held.set(key, at);
1455
+ }
1456
+ if (at !== void 0) pins.set(key, at);
1457
+ }
1458
+ return this.engineWithPins(pins);
1459
+ }
1460
+ /** `op` under the store's retries, as the store's own reads run; bare when the store has retries off. */
1461
+ withRetries(op) {
1462
+ const r = this.retryOptions;
1463
+ return r === void 0 ? op() : withRetry(op, r.policy ?? DEFAULT_RETRY_POLICY, r);
1464
+ }
1465
+ /** The store's registry, or a typed error naming the operation that needs one. */
1466
+ requireRegistry(op) {
1467
+ if (this.registry === void 0) {
1468
+ throw new UnsupportedError(
1469
+ `${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
1470
+ );
1471
+ }
1472
+ return this.registry;
1473
+ }
1474
+ /**
1475
+ * **Cross-tier DR consistency check.** After a restore/failover, verify every registered segment's `currentGen`
1476
+ * actually has its `.crbm` present in Storage — catching a **torn restore** where the registry (`currentGen`) came
1477
+ * back ahead of the object store, so a pointer references a generation that isn't there (reads would then
1478
+ * throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
1479
+ * errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
1480
+ * store, or roll the registry back to a coherent point). Needs the store built with a **backend**
1481
+ * (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded) segments are skipped. Pair it with
1482
+ * the DR runbook (docs/guide/disaster-recovery.md).
1483
+ *
1484
+ * It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
1485
+ * `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
1486
+ * ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
1487
+ * `storage` and `registry` with a higher `maxScanSegments`.
1488
+ */
1489
+ async checkConsistency(options = {}) {
1490
+ const deps = this.lifecycleDeps("checkConsistency");
1491
+ return runConsistencyCheck({ storage: deps.storage, registry: deps.registry }, options);
1492
+ }
1493
+ /**
1494
+ * **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
1495
+ * public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
1496
+ * data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
1497
+ * (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
1498
+ * Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
1499
+ * enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
1500
+ * **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
1501
+ * Crypto-shredded segments are skipped. A segment that can't be read is isolated into the manifest's `failed[]`
1502
+ * (the run continues), so "a manifest exists" means the run finished — check `failed`. The `export-segments`
1503
+ * CLI wraps this with a filesystem sink.
1504
+ */
1505
+ async exportSegments(sink, options = {}) {
1506
+ const registry = this.requireRegistry("exportSegments");
1507
+ return runExport(this, registry, sink, {
1508
+ ...options,
1509
+ codec: options.codec ?? roaringCodec
1510
+ });
1511
+ }
1512
+ /**
1513
+ * Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
1514
+ * from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
1515
+ */
1516
+ static estimateCost(input) {
1517
+ return estimateCost(input);
1518
+ }
1519
+ };
1520
+ var failing = (err) => ({
1521
+ [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
1522
+ });
1523
+ var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
1524
+ var twoPins = (a, b) => {
1525
+ const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
1526
+ return a !== void 0 && b !== void 0 && a.generation === b.generation ? `pinned twice at generation ${a.generation ?? "none"}, as two different objects` : `${one(a)} and ${one(b)}`;
1527
+ };
1528
+ var rangeOf = (options) => ({
1529
+ after: options.after,
1530
+ through: options.through
1531
+ });
1532
+ var readOptions = (options) => ({
1533
+ ...rangeOf(options),
1534
+ concurrency: options.concurrency,
1535
+ budget: options.budget,
1536
+ allowAbsentOperands: options.allowAbsentOperands
1537
+ });
1538
+ var NO_SEGMENTS = [];
1539
+ var EMPTY_IDS = {
1540
+ async *[Symbol.asyncIterator]() {
1541
+ }
1542
+ };
1543
+ var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
1544
+ var makeSegment;
1545
+ var minting = false;
1546
+ var Segment = class _Segment {
1547
+ metricsOn;
1548
+ engine;
1549
+ ref;
1550
+ clock;
1551
+ metrics;
1552
+ materialize;
1553
+ pinned;
1554
+ combineEngine;
1555
+ /** Absolute epoch-ms deadline from {@link SegmentOptions.expiresAt}; `undefined` ⇒ this handle never expires. */
1556
+ expiresAt;
1557
+ /**
1558
+ * The generation this handle is held at, when it came from {@link Segment.pin}. Read by the store so a
1559
+ * pinned handle passed as an **operand** is still read at its pin rather than live.
1560
+ */
1561
+ pinnedAt;
1562
+ static {
1563
+ makeSegment = (parts) => {
1564
+ minting = true;
1565
+ try {
1566
+ return new _Segment(parts);
1567
+ } finally {
1568
+ minting = false;
1569
+ }
1570
+ };
1571
+ }
1572
+ /**
1573
+ * Not constructible: a handle comes from {@link CloudRoaring.segment}, which wires it to the store's engine,
1574
+ * caches and write path. The constructor takes those internals, so it is not part of the surface.
1575
+ */
1576
+ constructor(parts) {
1577
+ if (!minting) {
1578
+ throw new ValidationError2(
1579
+ "a Segment is not constructed directly; call `store.segment(name)`"
1580
+ );
1581
+ }
1582
+ this.engine = parts.engine;
1583
+ this.ref = parts.ref;
1584
+ this.clock = parts.clock;
1585
+ this.metrics = parts.metrics;
1586
+ this.materialize = parts.materialize;
1587
+ this.pinned = parts.pinned;
1588
+ this.combineEngine = parts.combineEngine;
1589
+ this.expiresAt = parts.expiresAt;
1590
+ this.pinnedAt = parts.pinnedAt;
1591
+ this.metricsOn = parts.metrics !== NOOP_METRICS;
1592
+ }
1593
+ /**
1594
+ * **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
1595
+ *
1596
+ * An ordinary handle re-resolves on `cache.genTtlMs`, so a publish part-way through a long job means its second
1597
+ * half describes a different instant than its first — every chunk whole and verified, but the answer covering
1598
+ * two moments, with nothing in the result saying so. That is fine for a dashboard and wrong for an export, a
1599
+ * reconciliation, or a send that has to match the count you reported. A pin is how you get one instant.
1600
+ *
1601
+ * ```ts
1602
+ * const snap = await store.segment('active-30d').pin();
1603
+ * const total = await snap.count(); // the number you report
1604
+ * for await (const id of snap.iterate()) { … } // …and the ids it counted, however long this takes
1605
+ * ```
1606
+ *
1607
+ * **Only this segment is pinned.** `snap.intersect([other])` reads `snap` at its pinned generation and
1608
+ * `other` at whatever is current — pin each segment if you want the whole query held. And a pinned handle
1609
+ * used as an *operand* is still read at its pin, never live.
1610
+ *
1611
+ * **One call reads a segment at one generation**, so a combine that holds this segment at two —
1612
+ * `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
1613
+ * `*Into` of one throws before it reads anything. Materialise one side first, with `intersectInto(dest, [])`.
1614
+ *
1615
+ * **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
1616
+ * 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
1617
+ * instant it pinned, and anything it would have to fetch fails with `NotFoundError`, as a swept pin's does.
1618
+ *
1619
+ * **It is a hold, not a lease.** Nothing here stops a collection deleting the generation underneath
1620
+ * you: a pinned read deliberately does **not** heal forward, because silently serving a different generation
1621
+ * is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover your longest pinned
1622
+ * job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) — or take the pin on a segment
1623
+ * you are not collecting.
1624
+ *
1625
+ * A segment with no current generation pins nothing and reads empty, exactly as it would unpinned. A pinned
1626
+ * segment whose row is later dropped or destroyed fails once it must open its object again, rather than go empty
1627
+ * part-way through a call. **A pin
1628
+ * keeps the key its reader unwrapped for as long as that reader stays open, and answers from the chunks it
1629
+ * decoded for as long as they stay cached.** This store invalidates a pin:
1630
+ *
1631
+ * - a `load`, a `rollback` or an `*Into` invalidates a pin of the segment it writes;
1632
+ * - `dropSegment` invalidates a pin of the segment it drops, and `retireExpired` a pin of each segment its ledger
1633
+ * lists, retired or not, neither on a dry run;
1634
+ * - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
1635
+ *
1636
+ * An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
1637
+ * destroyed.
1638
+ * Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
1639
+ * retirement through another store, in this process or another, the pin answers from what it holds until this
1640
+ * store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
1641
+ * {@link CloudRoaring.invalidate} is called on this store. Where the
1642
+ * object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
1643
+ * once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
1644
+ *
1645
+ * A replacement a pin has found is remembered, so its later chunk reads fail with no request, until the store
1646
+ * forgets it: an invalidation of the segment does, as does a later `pin()` of the same version that opens the object
1647
+ * again, and the store remembers at most `cache.readerMax` of them. So once a restore puts the pinned object back,
1648
+ * call {@link CloudRoaring.invalidate} on this store: the pin then reads the object again.
1649
+ *
1650
+ * `pin()` reads the registry row and opens the generation at once, so the pin knows its object before its first
1651
+ * read. With a registry, pins of one generation taken while its row is unchanged share one reader while the store
1652
+ * keeps it open, pins taken at the same moment included: the first costs a tail read, and a key unwrap for an
1653
+ * encrypted segment, whether or not it is read, and the rest cost the row read alone. Without a registry every
1654
+ * `pin()` lists the segment's objects and makes the tail read, since only the object can tell two incarnations of
1655
+ * a name apart there. It retries a
1656
+ * transient fault as the store's reads do, and heals a generation swept between the two, but it fails where the
1657
+ * generation cannot be opened: `NotFoundError` for a pointer at a missing object, `IntegrityError` for a damaged
1658
+ * or misfiled one.
1659
+ *
1660
+ * Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
1661
+ * {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
1662
+ */
1663
+ async pin() {
1664
+ return this.pinned(this.ref, this.expiresAt);
1665
+ }
1666
+ /**
1667
+ * An opaque string that names this handle's segment, namespace included: two handles of one segment have the
1668
+ * same key, a live handle and its pins among them. Use it as a `Map` key or a log field when you track handles
1669
+ * and have not kept the name you made them with. The format is not specified and is not a storage key, so
1670
+ * compare keys, never parse one.
1671
+ */
1672
+ key() {
1673
+ return segmentKey(this.ref);
1674
+ }
1675
+ /**
1676
+ * Has this handle's deadline passed? **The lazy half of expiry** — the whole check is one comparison against
1677
+ * the injected clock, so it costs nothing on a handle with no deadline and no I/O on one that has expired.
1678
+ *
1679
+ * Deliberately evaluated per call rather than cached: a long-lived handle created before its deadline must
1680
+ * start reading empty the moment the deadline passes, without the caller re-creating it.
1681
+ */
1682
+ expired() {
1683
+ return this.expiresAt !== void 0 && this.clock.now() >= this.expiresAt;
1684
+ }
1685
+ /**
1686
+ * Refuse a materialisation that involves an **expired** handle. Called by the three `*Into` verbs only.
1687
+ *
1688
+ * On the read verbs an expired handle answers empty, which is the point of lazy expiry. On a *write* the same
1689
+ * rule would be destructive in a way nobody asks for: `a.intersectInto(dest, [b])` where `b`'s deadline has
1690
+ * quietly passed would, unrefused, replace `dest` with an **empty generation** — a wipe, reported as a successful
1691
+ * write, with the cause (a deadline on a handle somewhere) nowhere in the result. Reads degrade to empty; writes must not.
1692
+ *
1693
+ * So every handle in the call is checked, `dest` included: an expired `dest` does not change the bytes written,
1694
+ * but a caller who put a deadline on the thing they are writing into has said something contradictory and is
1695
+ * better told than guessed at. Open a handle without `expiresAt` to write, or drop the deadline.
1696
+ *
1697
+ * (The broader guard — refusing to publish an empty or implausible generation over a non-empty one, with an
1698
+ * `allowEmpty` override — covers these verbs too: they route through the same guarded write path as
1699
+ * {@link CloudRoaring.load}. The two stay separate because they differ in kind. That one is a REPORTED
1700
+ * refusal a caller may legitimately override; an expired handle is a wiring mistake, so it THROWS, before
1701
+ * any object is written — and `allowEmpty: true` does not reach it.)
1702
+ *
1703
+ * The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
1704
+ * a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
1705
+ * awaiting.
1706
+ */
1707
+ refuseIfExpired(op, dest, operands) {
1708
+ const stale = [];
1709
+ for (const seg of [this, dest, ...operands]) {
1710
+ if (seg.expired())
1711
+ stale.push(seg.ref.namespace ? `${seg.ref.namespace}/${seg.ref.segment}` : seg.ref.segment);
1712
+ }
1713
+ if (stale.length > 0) {
1714
+ throw new ValidationError2(
1715
+ `${op}: refusing to publish a generation while these handles have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired handle reads as empty, so this would write an empty (or short) generation over "${dest.ref.segment}". Open the handles without \`expiresAt\` if you meant to materialise, or drop the deadline.`
1716
+ );
1717
+ }
1718
+ }
1719
+ /**
1720
+ * Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
1721
+ * entirely when no sink is wired, so the default path pays nothing.
1722
+ */
1723
+ async timed(name, fn) {
1724
+ if (!this.metricsOn) return fn();
1725
+ const startedAt = this.clock.now();
1726
+ try {
1727
+ return await fn();
1728
+ } finally {
1729
+ this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
1730
+ }
1731
+ }
1732
+ /** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
1733
+ has(id) {
1734
+ if (this.expired()) return Promise.resolve(false);
1735
+ return this.timed("has", () => this.engine.has(this.ref, id));
1736
+ }
1737
+ /**
1738
+ * Cardinality — summed from the `.crbm` index with **zero payload reads** on a loaded segment.
1739
+ *
1740
+ * What this trusts: the answer is the sum of the per-chunk cardinalities the index records, and no payload is
1741
+ * decoded to confirm it. The index is checked for internal consistency when the object is opened (each key in
1742
+ * range and ascending, each cardinality in `1..65536`, each payload inside the payload region, and, on an
1743
+ * unencrypted object, the footer's chunk count and total agreeing with the index), and a corrupt index that is
1744
+ * still internally consistent yields a wrong count. `iterate()` and the combines decode the payloads, whose
1745
+ * structure is checked.
1746
+ */
1747
+ count() {
1748
+ if (this.expired()) return Promise.resolve(0);
1749
+ return this.timed("count", () => this.engine.count(this.ref));
1750
+ }
1751
+ /**
1752
+ * Every id, ascending, streamed one chunk at a time. Pass a range to read part of the segment: the ids in
1753
+ * `(after, through]`, fetching only the chunks the range overlaps. The per-op budget is charged for every chunk in
1754
+ * the range, so give a page `through` as well as `after`. On a pinned handle it reads the pinned generation, as a
1755
+ * full read does.
1756
+ *
1757
+ * ```ts
1758
+ * // Keyset paging: each page resumes where the one before ended, and stops at its window's end.
1759
+ * for await (const id of seg.iterate({ after: previousEnd, through: windowEnd })) await send(id);
1760
+ * ```
1761
+ *
1762
+ * Each bound is optional and an integer in `0..4294967295`; a bad one throws {@link ValidationError} when the
1763
+ * stream is first read. `after >= through` is an empty range, which reads nothing. An expired handle reads empty
1764
+ * without checking its options, as every read of one does.
1765
+ */
1766
+ iterate(options) {
1767
+ if (this.expired()) return EMPTY_IDS;
1768
+ const range = options == null ? void 0 : rangeOf(options);
1769
+ const none = range === void 0 || range.after === void 0 && range.through === void 0;
1770
+ return this.engine.iterate(this.ref, none ? void 0 : range);
1771
+ }
1772
+ /**
1773
+ * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
1774
+ * it was given, already reduced to its live handles ({@link liveExcludes}) and mapped down to the plain refs
1775
+ * `core` takes. A method rather than a module function because `ref` is class-private — the encapsulation is
1776
+ * worth more than the free function.
1777
+ *
1778
+ * Each field is named rather than copied with a rest spread: a rest copy takes only own enumerable properties, so
1779
+ * a bound held in a getter or inherited from a prototype was dropped, and the read silently widened to the whole
1780
+ * segment.
1781
+ */
1782
+ refsIn(options, exclude) {
1783
+ if (options == null) return void 0;
1784
+ return {
1785
+ ...readOptions(options),
1786
+ exclude: exclude.length > 0 ? exclude.map((o) => o.ref) : void 0
1787
+ };
1788
+ }
1789
+ /**
1790
+ * The handles in `options.exclude` that have not expired. **An expired exclusion excludes nothing**, in every
1791
+ * combine, so it is dropped here, before the engine is asked for anything: it is never fetched, never
1792
+ * checked for absence, and never counted in the pin-consistency check. `exclude` is read once, so a getter sees
1793
+ * one call.
1794
+ *
1795
+ * With no `exclude` this is one property read and a shared empty list, and with none expired it returns the
1796
+ * caller's own array, so the common case allocates nothing.
1797
+ */
1798
+ liveExcludes(options) {
1799
+ const exclude = options?.exclude;
1800
+ if (exclude == null || exclude.length === 0) return NO_SEGMENTS;
1801
+ return exclude.some((e) => e.expired()) ? exclude.filter((e) => !e.expired()) : exclude;
1802
+ }
1803
+ /**
1804
+ * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
1805
+ * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
1806
+ * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
1807
+ * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
1808
+ * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
1809
+ * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
1810
+ */
1811
+ intersect(others, options) {
1812
+ if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
1813
+ const exclude = this.liveExcludes(options);
1814
+ let engine;
1815
+ try {
1816
+ engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
1817
+ } catch (err) {
1818
+ return failing(err);
1819
+ }
1820
+ return engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
1821
+ }
1822
+ /**
1823
+ * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
1824
+ * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
1825
+ * as a load is (the pointer advances to it, fenced on the row), so readers of `dest` see either the old generation
1826
+ * or the new one, never a partial. Needs the
1827
+ * store built with a backend (throws {@link UnsupportedError} otherwise).
1828
+ *
1829
+ * **An empty result does NOT overwrite a non-empty destination.** A combine that comes out empty is far more
1830
+ * often a mistake upstream — a typo'd operand, an `exclude` that swallowed everything, an operand that has
1831
+ * not loaded yet — than an intent, and once it publishes it is indistinguishable from a correct run. So the
1832
+ * write is **refused and reported**: `published: false`, `reason: 'empty'`, and `dest` keeps what it had.
1833
+ * Pass `allowEmpty: true` when emptying the destination is the point.
1834
+ *
1835
+ * `guard` adds the same plausibility bounds `load()` takes — `minCardinality` and `minRetained` — judged
1836
+ * against what `dest` held before. A refusal is **reported, not thrown**, exactly as on `load()`; branch on
1837
+ * `published`. A lost race is the one outcome that still throws ({@link WriteConflictError}), because a
1838
+ * materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
1839
+ *
1840
+ * A call involving an **expired handle** is refused before any of this. See {@link refuseIfExpired}.
1841
+ */
1842
+ async intersectInto(dest, others, options) {
1843
+ this.refuseIfExpired("intersectInto", dest, [...others, ...options?.exclude ?? []]);
1844
+ this.combineEngine([this, ...others, ...options?.exclude ?? []]);
1845
+ return this.timed(
1846
+ "intersectInto",
1847
+ () => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
1848
+ );
1849
+ }
1850
+ /**
1851
+ * `this ∪ others…`, minus `options.exclude` — streamed ascending.
1852
+ *
1853
+ * **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
1854
+ * here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
1855
+ * `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
1856
+ * same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
1857
+ * deliberately. If you find yourself unioning the same segments on every read, materializing the combined
1858
+ * segment once (`unionInto`, or a load) is the cheaper shape.
1859
+ */
1860
+ union(others, options) {
1861
+ const live = others.filter((o) => !o.expired());
1862
+ if (this.expired()) {
1863
+ if (live.length === 0) return EMPTY_IDS;
1864
+ return live[0].union(live.slice(1), options);
1865
+ }
1866
+ if (live.length === 0 && others.length > 0) {
1867
+ const exclude2 = this.liveExcludes(options);
1868
+ return exclude2.length > 0 ? this.andNot([...exclude2], options) : this.union([], options);
1869
+ }
1870
+ if (live.length !== others.length) return this.union(live, options);
1871
+ const exclude = this.liveExcludes(options);
1872
+ let engine;
1873
+ try {
1874
+ engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
1875
+ } catch (err) {
1876
+ return failing(err);
1877
+ }
1878
+ return engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
1879
+ }
1880
+ /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
1881
+ async unionInto(dest, others, options) {
1882
+ this.refuseIfExpired("unionInto", dest, [...others, ...options?.exclude ?? []]);
1883
+ this.combineEngine([this, ...others, ...options?.exclude ?? []]);
1884
+ return this.timed(
1885
+ "unionInto",
1886
+ () => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
1887
+ );
1888
+ }
1889
+ /**
1890
+ * `this \ (excludes…)` — streamed ascending. Suppression on its own.
1891
+ *
1892
+ * Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
1893
+ * overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
1894
+ * suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
1895
+ * opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
1896
+ *
1897
+ * An expired handle in `excludes` excludes nothing, and is skipped without being read; if every one has
1898
+ * expired the result is `this` whole. An expired `this` is empty. `exclude` on `intersect` and `union` follows
1899
+ * the same rule.
1900
+ *
1901
+ * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
1902
+ * the suppression folds into the same pass rather than materializing an intermediate segment first.
1903
+ */
1904
+ andNot(excludes, options) {
1905
+ if (this.expired()) return EMPTY_IDS;
1906
+ const liveExcludes = excludes.filter((e) => !e.expired());
1907
+ if (liveExcludes.length === 0 && excludes.length > 0) {
1908
+ return this.union([], options == null ? void 0 : readOptions(options));
1909
+ }
1910
+ if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
1911
+ let engine;
1912
+ try {
1913
+ engine = this.combineEngine([this, ...excludes]) ?? this.engine;
1914
+ } catch (err) {
1915
+ return failing(err);
1916
+ }
1917
+ return engine.andNot(
1918
+ this.ref,
1919
+ excludes.map((o) => o.ref),
1920
+ options == null ? void 0 : readOptions(options)
1921
+ );
1922
+ }
1923
+ /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
1924
+ async andNotInto(dest, excludes, options) {
1925
+ this.refuseIfExpired("andNotInto", dest, excludes);
1926
+ this.combineEngine([this, ...excludes]);
1927
+ return this.timed(
1928
+ "andNotInto",
1929
+ () => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
1930
+ );
1931
+ }
1932
+ /**
1933
+ * Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
1934
+ * reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
1935
+ * storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
1936
+ * refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
1937
+ * the lose-zone).
1938
+ */
1939
+ async costReport(options) {
1940
+ const canMeasure = this.engine.supportsStorageSize;
1941
+ const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
1942
+ const refreshMs = this.engine.pointerRefreshMs;
1943
+ const workload = refreshMs === void 0 || options?.workload?.genTtlMs !== void 0 ? options?.workload : { ...options?.workload, genTtlMs: refreshMs };
1944
+ return groundedReport({
1945
+ storageBytes: size?.sizeBytes ?? 0,
1946
+ grounded: canMeasure,
1947
+ workload,
1948
+ pricing: options?.pricing,
1949
+ extraNotes: canMeasure ? void 0 : ["storage source has no sizeOf() \u2014 storage not measured, reported as $0."]
1950
+ });
1951
+ }
1952
+ };
1953
+
1954
+ // src/bin/export-segments.ts
19
1955
  function parseConfig(env) {
20
1956
  const root = env.CR_EXPORT_ROOT;
21
1957
  if (root === void 0 || root === "") {
@@ -30,14 +1966,14 @@ function parseConfig(env) {
30
1966
  throw new Error(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
31
1967
  }
32
1968
  const namespace = env.CR_EXPORT_NAMESPACE || void 0;
33
- return { root, out, format, namespace, segments: parseSegments(env.CR_EXPORT_SEGMENTS) };
1969
+ return { root, out, format, namespace };
34
1970
  }
35
1971
  function fsSink(out) {
36
1972
  return {
37
1973
  async open(ref, ext) {
38
- const dir = join(out, ref.namespace ?? "_default");
1974
+ const dir = join(out, namespacePathPart(ref.namespace));
39
1975
  await mkdir(dir, { recursive: true, mode: 448 });
40
- const finalPath = join(dir, `${ref.segment}${ext}`);
1976
+ const finalPath = join(dir, `${encodeNameForPath(ref.segment)}${ext}`);
41
1977
  const tmpPath = `${finalPath}.${randomUUID()}.part`;
42
1978
  const handle = await open(tmpPath, "wx", 384);
43
1979
  return {
@@ -73,16 +2009,11 @@ var log = (obj) => {
73
2009
  };
74
2010
  async function main(env = process.env, now = () => Date.now()) {
75
2011
  const config = parseConfig(env);
76
- const registry = new src_exports.LocalFsRegistryDriver(join(config.root, "registry"));
77
- const store = new CloudRoaring({
78
- cold: new src_exports.LocalFsColdDriver(join(config.root, "cold")),
79
- warm: new src_exports.LocalFsWarmDriver(join(config.root, "warm")),
80
- registry
81
- });
2012
+ const storage = new LocalFsStorage(config.root);
2013
+ const store = new CloudRoaring({ storage });
82
2014
  const manifest = await store.exportSegments(fsSink(config.out), {
83
2015
  format: config.format,
84
- namespace: config.namespace,
85
- candidates: config.segments
2016
+ namespace: config.namespace
86
2017
  });
87
2018
  await mkdir(config.out, { recursive: true, mode: 448 });
88
2019
  const manifestPath = join(config.out, "manifest.json");
@@ -110,7 +2041,16 @@ async function main(env = process.env, now = () => Date.now()) {
110
2041
  });
111
2042
  return manifest;
112
2043
  }
113
- if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv[1]).href) {
2044
+ function invokedAsCli() {
2045
+ const entry = process.argv[1];
2046
+ if (entry === void 0) return false;
2047
+ try {
2048
+ return import.meta.url === pathToFileURL(entry).href || import.meta.url === pathToFileURL(realpathSync(entry)).href;
2049
+ } catch {
2050
+ return false;
2051
+ }
2052
+ }
2053
+ if (invokedAsCli()) {
114
2054
  main().then((manifest) => {
115
2055
  if (manifest.failed.length > 0) {
116
2056
  process.stderr.write(
@@ -125,7 +2065,9 @@ if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv
125
2065
  process.exit(1);
126
2066
  });
127
2067
  }
128
-
129
- export { fsSink, main, parseConfig };
2068
+ export {
2069
+ fsSink,
2070
+ main,
2071
+ parseConfig
2072
+ };
130
2073
  //# sourceMappingURL=export-segments.js.map
131
- //# sourceMappingURL=export-segments.js.map