@cloudbitmaps/roaring 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/PRIVACY.md +198 -72
- package/README.md +121 -57
- package/dist/bin/export-segments.d.ts +21 -0
- package/dist/bin/export-segments.js +1508 -30
- package/dist/bin/export-segments.js.map +7 -1
- package/dist/codec-bound.d.ts +38 -0
- package/dist/index.d.ts +805 -447
- package/dist/index.js +911 -325
- package/dist/index.js.map +7 -1
- package/dist/moved-options.d.ts +14 -0
- package/dist/portable/decode.d.ts +54 -0
- package/dist/roaring-codec.d.ts +61 -0
- package/dist/system-clock.d.ts +40 -0
- package/dist/testing/conformance.d.ts +34 -0
- package/dist/testing/fuzz-codec.d.ts +9 -0
- package/package.json +11 -116
- package/dist/azure/index.cjs +0 -14
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -9
- package/dist/azure/index.d.ts +0 -9
- package/dist/azure/index.js +0 -3
- package/dist/azure/index.js.map +0 -1
- package/dist/bin/compact-segments.js +0 -201
- package/dist/bin/compact-segments.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -14
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -1
- package/dist/cassandra/index.d.ts +0 -1
- package/dist/cassandra/index.js +0 -3
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-D6LIJSGF.js +0 -906
- package/dist/chunk-D6LIJSGF.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -14
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -1
- package/dist/dynamodb/index.d.ts +0 -1
- package/dist/dynamodb/index.js +0 -3
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -14
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -1
- package/dist/gcs/index.d.ts +0 -1
- package/dist/gcs/index.js +0 -3
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -894
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -734
- package/dist/mongodb/index.cjs +0 -14
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -1
- package/dist/mongodb/index.d.ts +0 -1
- package/dist/mongodb/index.js +0 -3
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -14
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -1
- package/dist/mysql/index.d.ts +0 -1
- package/dist/mysql/index.js +0 -3
- package/dist/mysql/index.js.map +0 -1
- package/dist/postgres/index.cjs +0 -14
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -1
- package/dist/postgres/index.d.ts +0 -1
- package/dist/postgres/index.js +0 -3
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -14
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -1
- package/dist/redis/index.d.ts +0 -1
- package/dist/redis/index.js +0 -3
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -14
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -1
- package/dist/s3/index.d.ts +0 -1
- package/dist/s3/index.js +0 -3
- package/dist/s3/index.js.map +0 -1
|
@@ -1,21 +1,1493 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
|
|
8
19
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
20
|
+
// src/bin/export-segments.ts
|
|
21
|
+
import { randomUUID } from "node:crypto";
|
|
22
|
+
import { realpathSync } from "node:fs";
|
|
23
|
+
import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
import { pathToFileURL } from "node:url";
|
|
26
|
+
|
|
27
|
+
// src/index.ts
|
|
28
|
+
var src_exports = {};
|
|
29
|
+
__export(src_exports, {
|
|
30
|
+
CloudRoaring: () => CloudRoaring,
|
|
31
|
+
SafeBitmap: () => SafeBitmap,
|
|
32
|
+
Segment: () => Segment,
|
|
33
|
+
VERSION: () => VERSION,
|
|
34
|
+
bulkLoadCrbmGeneration: () => bulkLoadCrbmGeneration,
|
|
35
|
+
eraseIdFromSegment: () => eraseIdFromSegment,
|
|
36
|
+
loadSegment: () => loadSegment,
|
|
37
|
+
roaringCodec: () => roaringCodec,
|
|
38
|
+
runExport: () => runExport
|
|
39
|
+
});
|
|
40
|
+
import {
|
|
41
|
+
BoundedLru,
|
|
42
|
+
PinnedStorageChunkSource,
|
|
43
|
+
segmentKey,
|
|
44
|
+
CrbmStorageChunkSource,
|
|
45
|
+
DEFAULT_BUDGET,
|
|
46
|
+
DEFAULT_RETRY_POLICY,
|
|
47
|
+
isStorageBackend,
|
|
48
|
+
NOOP_METRICS,
|
|
49
|
+
RetryingStorageChunkSource,
|
|
50
|
+
SegmentEngine,
|
|
51
|
+
UnsupportedError,
|
|
52
|
+
ValidationError,
|
|
53
|
+
WriteConflictError,
|
|
54
|
+
MIN_EXPIRES_AT_MS,
|
|
55
|
+
collectWithinBudget,
|
|
56
|
+
excludingReservedRows,
|
|
57
|
+
dropSegment,
|
|
58
|
+
estimateCost,
|
|
59
|
+
groundedReport,
|
|
60
|
+
mapWithConcurrency,
|
|
61
|
+
resolveBudget,
|
|
62
|
+
resolvePerOpBudget,
|
|
63
|
+
retireExpired,
|
|
64
|
+
runConsistencyCheck,
|
|
65
|
+
runExport as runExport2,
|
|
66
|
+
setSegmentRetention,
|
|
67
|
+
clearSegmentRetention,
|
|
68
|
+
getSegmentRetention,
|
|
69
|
+
safeMetrics,
|
|
70
|
+
splitId,
|
|
71
|
+
validateSegmentRef
|
|
72
|
+
} from "@cloudbitmaps/core";
|
|
73
|
+
|
|
74
|
+
// src/codec-bound.ts
|
|
75
|
+
import {
|
|
76
|
+
bulkLoadCrbmGeneration as coreBulkLoad,
|
|
77
|
+
eraseIdFromSegment as coreEraseIdFromSegment,
|
|
78
|
+
loadSegment as coreLoadSegment,
|
|
79
|
+
runExport as coreRunExport
|
|
80
|
+
} from "@cloudbitmaps/core";
|
|
81
|
+
|
|
82
|
+
// src/roaring-codec.ts
|
|
83
|
+
import roaring from "roaring";
|
|
84
|
+
import { IntegrityError } from "@cloudbitmaps/core";
|
|
85
|
+
var { RoaringBitmap32, SerializationFormat, DeserializationFormat } = roaring;
|
|
86
|
+
var SafeBitmap = class _SafeBitmap {
|
|
87
|
+
bitmap;
|
|
88
|
+
constructor(bitmap) {
|
|
89
|
+
this.bitmap = bitmap;
|
|
90
|
+
}
|
|
91
|
+
static empty() {
|
|
92
|
+
return new _SafeBitmap(new RoaringBitmap32());
|
|
93
|
+
}
|
|
94
|
+
static fromValues(values) {
|
|
95
|
+
return new _SafeBitmap(new RoaringBitmap32(values));
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Validate size, then deserialize with the **portable** (validated) format.
|
|
99
|
+
* Throws `IntegrityError` if the input exceeds `maxBytes` or fails to decode — the
|
|
100
|
+
* native addon is never handed unbounded or unsafe-format input.
|
|
101
|
+
*/
|
|
102
|
+
static safeDeserialize(bytes, maxBytes) {
|
|
103
|
+
if (bytes.length > maxBytes) {
|
|
104
|
+
throw new IntegrityError(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
105
|
+
}
|
|
106
|
+
try {
|
|
107
|
+
return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
|
|
108
|
+
} catch (err) {
|
|
109
|
+
throw new IntegrityError(`failed to deserialize bitmap: ${err.message}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
serialize() {
|
|
113
|
+
return this.bitmap.serialize(SerializationFormat.portable);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
117
|
+
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
118
|
+
*
|
|
119
|
+
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
120
|
+
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
121
|
+
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
122
|
+
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
123
|
+
* whichever encoding is smaller per container.
|
|
124
|
+
*
|
|
125
|
+
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
126
|
+
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
127
|
+
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
128
|
+
* columns; don't carry a figure between them without checking which denominator it had.
|
|
129
|
+
*
|
|
130
|
+
* Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
|
|
131
|
+
* path deliberately does not.
|
|
132
|
+
*/
|
|
133
|
+
optimize() {
|
|
134
|
+
this.bitmap.runOptimize();
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the container index, so the engine's
|
|
138
|
+
* per-chunk range assertion costs one call per chunk rather than a walk per id.
|
|
139
|
+
*/
|
|
140
|
+
maximum() {
|
|
141
|
+
return this.bitmap.isEmpty ? void 0 : this.bitmap.maximum();
|
|
142
|
+
}
|
|
143
|
+
add(value) {
|
|
144
|
+
this.bitmap.add(value);
|
|
145
|
+
}
|
|
146
|
+
addMany(values) {
|
|
147
|
+
this.bitmap.addMany(values);
|
|
148
|
+
}
|
|
149
|
+
remove(value) {
|
|
150
|
+
this.bitmap.remove(value);
|
|
151
|
+
}
|
|
152
|
+
removeMany(values) {
|
|
153
|
+
this.bitmap.removeMany(values);
|
|
154
|
+
}
|
|
155
|
+
has(value) {
|
|
156
|
+
return this.bitmap.has(value);
|
|
157
|
+
}
|
|
158
|
+
get size() {
|
|
159
|
+
return this.bitmap.size;
|
|
160
|
+
}
|
|
161
|
+
get isEmpty() {
|
|
162
|
+
return this.bitmap.isEmpty;
|
|
163
|
+
}
|
|
164
|
+
clone() {
|
|
165
|
+
return new _SafeBitmap(this.bitmap.clone());
|
|
166
|
+
}
|
|
167
|
+
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
168
|
+
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
169
|
+
// `bitmap` (accessible on same-class instances) for the native op.
|
|
170
|
+
/** In-place union: `this = this ∪ other`. */
|
|
171
|
+
orInPlace(other) {
|
|
172
|
+
this.bitmap.orInPlace(other.bitmap);
|
|
173
|
+
}
|
|
174
|
+
/** In-place difference: `this = this \ other`. */
|
|
175
|
+
andNotInPlace(other) {
|
|
176
|
+
this.bitmap.andNotInPlace(other.bitmap);
|
|
177
|
+
}
|
|
178
|
+
/** In-place intersection: `this = this ∩ other`. */
|
|
179
|
+
andInPlace(other) {
|
|
180
|
+
this.bitmap.andInPlace(other.bitmap);
|
|
181
|
+
}
|
|
182
|
+
/** Ascending iterator over the set values. */
|
|
183
|
+
[Symbol.iterator]() {
|
|
184
|
+
return this.bitmap[Symbol.iterator]();
|
|
185
|
+
}
|
|
186
|
+
toArray() {
|
|
187
|
+
return this.bitmap.toArray();
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
var roaringCodec = {
|
|
191
|
+
empty: () => SafeBitmap.empty(),
|
|
192
|
+
fromValues: (values) => SafeBitmap.fromValues(values),
|
|
193
|
+
safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
// src/system-clock.ts
|
|
197
|
+
var SystemClock = class {
|
|
198
|
+
now() {
|
|
199
|
+
return Date.now();
|
|
200
|
+
}
|
|
201
|
+
sleep(ms) {
|
|
202
|
+
if (ms <= 0) return Promise.resolve();
|
|
203
|
+
return new Promise((resolve) => {
|
|
204
|
+
setTimeout(resolve, ms);
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
|
|
209
|
+
*
|
|
210
|
+
* A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
|
|
211
|
+
* co-resident HTTP server keeps answering. Measured on the **synthetic yield-primitive benchmark** — a
|
|
212
|
+
* 61,035-iteration loop yielding every 1,024, isolating the primitive rather than timing a real load (the
|
|
213
|
+
* end-to-end figures are 450 ms → 19 ms; see `cooperative.ts`, which lists all three experiments):
|
|
214
|
+
*
|
|
215
|
+
* ```text
|
|
216
|
+
* no yield 568 ms wall 568.0 ms worst event-loop gap
|
|
217
|
+
* sleep(0) 555 ms wall 555.2 ms ← a microtask. Yields NOTHING.
|
|
218
|
+
* setImmediate 569 ms wall 13.8 ms ← free, and 41x less starvation
|
|
219
|
+
* setTimeout(1) 625 ms wall 15.0 ms ← +10% wall for no extra relief
|
|
220
|
+
* ```
|
|
221
|
+
*
|
|
222
|
+
* `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
|
|
223
|
+
* before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
|
|
224
|
+
* callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
|
|
225
|
+
* server wants — inbound requests are serviced before the loop resumes chewing.
|
|
226
|
+
*
|
|
227
|
+
* A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
|
|
228
|
+
* "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
|
|
229
|
+
*/
|
|
230
|
+
yieldNow() {
|
|
231
|
+
return new Promise((resolve) => {
|
|
232
|
+
setImmediate(resolve);
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
// src/codec-bound.ts
|
|
238
|
+
var bulkLoadCrbmGeneration = (driver, key, ids, options = {}) => coreBulkLoad(driver, key, ids, {
|
|
239
|
+
...options,
|
|
240
|
+
codec: options.codec ?? roaringCodec,
|
|
241
|
+
clock: options.clock ?? new SystemClock()
|
|
242
|
+
});
|
|
243
|
+
var eraseIdFromSegment = (ref, id, deps, options) => coreEraseIdFromSegment(
|
|
244
|
+
ref,
|
|
245
|
+
id,
|
|
246
|
+
{ ...deps, codec: deps.codec ?? roaringCodec, clock: deps.clock ?? new SystemClock() },
|
|
247
|
+
options
|
|
248
|
+
);
|
|
249
|
+
var runExport = (reader, registry, sink, options = {}) => coreRunExport(reader, registry, sink, { ...options, codec: options.codec ?? roaringCodec });
|
|
250
|
+
var loadSegment = (ref, ids, deps, options = {}) => coreLoadSegment(
|
|
251
|
+
ref,
|
|
252
|
+
ids,
|
|
253
|
+
{ ...deps, codec: deps.codec ?? roaringCodec, clock: deps.clock ?? new SystemClock() },
|
|
254
|
+
options
|
|
255
|
+
);
|
|
256
|
+
|
|
257
|
+
// src/index.ts
|
|
258
|
+
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
259
|
+
import { listSegments, segmentExists } from "@cloudbitmaps/core";
|
|
260
|
+
|
|
261
|
+
// src/moved-options.ts
|
|
262
|
+
var MOVED_OPTIONS = [
|
|
263
|
+
["cacheMaxChunks", "cache.maxChunks", "group"],
|
|
264
|
+
["cacheTtlMs", "cache.ttlMs", "group"],
|
|
265
|
+
// The `cold*` spellings are what `0.9.x` ACTUALLY had. The `storage*` ones below never appeared in any
|
|
266
|
+
// release — they existed between two unreleased commits of this cycle — and for a while they were the only
|
|
267
|
+
// ones here, so the guard written to catch an upgrader missed every real upgrader and caught nobody.
|
|
268
|
+
["coldGenTtlMs", "cache.genTtlMs", "group"],
|
|
269
|
+
["coldReaderCacheMax", "cache.readerMax", "group"],
|
|
270
|
+
["coldReaderCacheMaxBytes", "cache.readerMaxBytes", "group"],
|
|
271
|
+
["storageGenTtlMs", "cache.genTtlMs", "group"],
|
|
272
|
+
["storageReaderCacheMax", "cache.readerMax", "group"],
|
|
273
|
+
["storageReaderCacheMaxBytes", "cache.readerMaxBytes", "group"],
|
|
274
|
+
["keystore", "encryption.keystore", "group"],
|
|
275
|
+
["requireEncryption", "encryption.required", "group"],
|
|
276
|
+
["onRetry", "retry.onRetry", "group"],
|
|
277
|
+
["clock", "seams.clock", "group"],
|
|
278
|
+
["rng", "seams.rng", "group"],
|
|
279
|
+
[
|
|
280
|
+
"registry",
|
|
281
|
+
"the backend passed as `storage` (S3Storage, GcsStorage, \u2026), which carries it",
|
|
282
|
+
"renamed"
|
|
283
|
+
],
|
|
284
|
+
["cold", "storage", "renamed"],
|
|
285
|
+
// The live (warm) tier and its knobs. `warm` was REQUIRED in `0.9.x`, so every upgrader passes one — and
|
|
286
|
+
// an upgrader who fixes `cold` first meets this next. None has a successor, so none is offered.
|
|
287
|
+
["warm", "the live tier is gone", "gone"],
|
|
288
|
+
["warmReadConsistency", "it tuned the live tier", "gone"],
|
|
289
|
+
["maxWarmScanBytes", "it tuned the live tier", "gone"],
|
|
290
|
+
["writeConcurrency", "it bounded the live tier\u2019s flusher", "gone"],
|
|
291
|
+
["occBackoff", "it tuned the live tier\u2019s read-modify-write", "gone"]
|
|
292
|
+
];
|
|
293
|
+
|
|
294
|
+
// src/index.ts
|
|
295
|
+
__reExport(src_exports, core_star);
|
|
296
|
+
import * as core_star from "@cloudbitmaps/core";
|
|
297
|
+
var SystemRng = class {
|
|
298
|
+
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
299
|
+
next() {
|
|
300
|
+
return Math.random();
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
304
|
+
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
305
|
+
function validateConcurrency(concurrency) {
|
|
306
|
+
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
307
|
+
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
function requireScope(options, op) {
|
|
311
|
+
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
312
|
+
throw new ValidationError(
|
|
313
|
+
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
314
|
+
);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
function resolveStorageSource(options, clock) {
|
|
318
|
+
const storage = options.storage;
|
|
319
|
+
if (storage === null || typeof storage !== "object") {
|
|
320
|
+
throw new ValidationError(
|
|
321
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
const asBackend = storage;
|
|
325
|
+
const isObject = (v) => v !== null && typeof v === "object";
|
|
326
|
+
const hasGetChunk = typeof storage.getChunk === "function";
|
|
327
|
+
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
328
|
+
const isBackend = isStorageBackend(storage);
|
|
329
|
+
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
330
|
+
throw new ValidationError(
|
|
331
|
+
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
const hasStorageHalf = isObject(asBackend.storage);
|
|
335
|
+
const hasRegistryHalf = isObject(asBackend.registry);
|
|
336
|
+
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
337
|
+
throw new ValidationError(
|
|
338
|
+
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, say so: `createBackend({ storage: <your driver>, registry })`."
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
342
|
+
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
343
|
+
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
344
|
+
if (!storageOk || !registryOk) {
|
|
345
|
+
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
346
|
+
throw new ValidationError(
|
|
347
|
+
"`storage` looks like a backend but its " + bad + " \u2014 build one with a backend class, or with `createBackend({ storage, registry })`."
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
throw new ValidationError(
|
|
351
|
+
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. If you genuinely want halves of your own \u2014 an instrumented driver, a registry in a database you already run \u2014 say so with `createBackend({ storage, registry })`, which is you taking on that they agree."
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
if (isBackend) {
|
|
355
|
+
const backend = storage;
|
|
356
|
+
return {
|
|
357
|
+
source: new CrbmStorageChunkSource(backend.storage, {
|
|
358
|
+
registry: backend.registry,
|
|
359
|
+
keystore: options.encryption?.keystore,
|
|
360
|
+
requireEncryption: options.encryption?.required,
|
|
361
|
+
clock,
|
|
362
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
363
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
364
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
365
|
+
}),
|
|
366
|
+
driver: backend.storage,
|
|
367
|
+
registry: backend.registry
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
371
|
+
throw new ValidationError(
|
|
372
|
+
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
376
|
+
throw new ValidationError(
|
|
377
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
if (hasGetChunk) {
|
|
381
|
+
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
382
|
+
throw new ValidationError(
|
|
383
|
+
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
return { source: storage, driver: void 0, registry: void 0 };
|
|
387
|
+
}
|
|
388
|
+
const driver = storage;
|
|
389
|
+
return {
|
|
390
|
+
source: new CrbmStorageChunkSource(driver, {
|
|
391
|
+
keystore: options.encryption?.keystore,
|
|
392
|
+
requireEncryption: options.encryption?.required,
|
|
393
|
+
clock,
|
|
394
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
395
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
396
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
397
|
+
}),
|
|
398
|
+
driver,
|
|
399
|
+
registry: void 0
|
|
400
|
+
};
|
|
18
401
|
}
|
|
402
|
+
var CloudRoaring = class _CloudRoaring {
|
|
403
|
+
engine;
|
|
404
|
+
cache;
|
|
405
|
+
crbmSource;
|
|
406
|
+
clock;
|
|
407
|
+
metrics;
|
|
408
|
+
// The store's own drivers, kept so the lifecycle helpers and the `*Into` verbs reuse them instead of making
|
|
409
|
+
// you re-pass deps. `storageDriver` is set only when `storage` was a raw IStorageDriver (a pre-built StorageChunkSource has
|
|
410
|
+
// no underlying driver to write through).
|
|
411
|
+
storageDriver;
|
|
412
|
+
registry;
|
|
413
|
+
keystore;
|
|
414
|
+
requireEncryption;
|
|
415
|
+
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
416
|
+
budget;
|
|
417
|
+
/**
|
|
418
|
+
* Refuse an option that moved into a group, naming where it went.
|
|
419
|
+
*
|
|
420
|
+
* Silently ignoring one would be the exact failure this release exists to remove — see {@link MOVED_OPTIONS}
|
|
421
|
+
* for why each of these is unsafe to drop rather than merely untidy.
|
|
422
|
+
*/
|
|
423
|
+
static rejectMovedOptions(options) {
|
|
424
|
+
if (options === null || options === void 0 || typeof options !== "object") {
|
|
425
|
+
throw new ValidationError(
|
|
426
|
+
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
const bag = options;
|
|
430
|
+
const moved = MOVED_OPTIONS.filter(([from]) => bag[from] !== void 0);
|
|
431
|
+
if (moved.length === 0) return;
|
|
432
|
+
const clause = (kind, one, many) => {
|
|
433
|
+
const hits = moved.filter(([, , k]) => k === kind);
|
|
434
|
+
if (hits.length === 0) return null;
|
|
435
|
+
const body = hits.map(
|
|
436
|
+
([from, to]) => kind === "gone" ? `\`${from}\` (${to})` : `\`${from}\` \u2192 ${/^[\w.]+$/.test(to) ? `\`${to}\`` : to}`
|
|
437
|
+
).join(" \xB7 ");
|
|
438
|
+
return `${hits.length > 1 ? many : one}: ${body}`;
|
|
439
|
+
};
|
|
440
|
+
const parts = [
|
|
441
|
+
clause("group", "option moved into a group", "options moved into groups"),
|
|
442
|
+
clause("renamed", "option renamed", "options renamed"),
|
|
443
|
+
clause("gone", "option removed", "options removed")
|
|
444
|
+
].filter((c) => c !== null);
|
|
445
|
+
throw new ValidationError(
|
|
446
|
+
`CloudRoaring ${parts.join("; ")}. Options are now one required \`storage\` plus four optional groups \u2014 \`cache\`, \`encryption\`, \`retry\` and \`seams\`. \`metrics\` and \`budget\` are unchanged flat options; leave them as they are. Full guide: https://github.com/cloudbitmaps/cloudbitmaps/blob/main/MIGRATING.md`
|
|
447
|
+
);
|
|
448
|
+
}
|
|
449
|
+
constructor(options) {
|
|
450
|
+
_CloudRoaring.rejectMovedOptions(options);
|
|
451
|
+
const clock = options.seams?.clock ?? new SystemClock();
|
|
452
|
+
const rng = options.seams?.rng ?? new SystemRng();
|
|
453
|
+
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS);
|
|
454
|
+
const cache = new BoundedLru({
|
|
455
|
+
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
456
|
+
ttlMs: options.cache?.ttlMs,
|
|
457
|
+
clock
|
|
458
|
+
});
|
|
459
|
+
const resolved = resolveStorageSource(options, clock);
|
|
460
|
+
let storage = resolved.source;
|
|
461
|
+
if (options.retry !== false) {
|
|
462
|
+
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
463
|
+
const policy = {
|
|
464
|
+
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
465
|
+
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
466
|
+
maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
467
|
+
backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
468
|
+
jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
469
|
+
};
|
|
470
|
+
const retryOpts = {
|
|
471
|
+
clock,
|
|
472
|
+
rng,
|
|
473
|
+
policy,
|
|
474
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
475
|
+
onRetry: (info) => {
|
|
476
|
+
metrics.onEvent({
|
|
477
|
+
kind: "retry",
|
|
478
|
+
reason: "transient",
|
|
479
|
+
attempt: info.attempt,
|
|
480
|
+
delayMs: info.delayMs
|
|
481
|
+
});
|
|
482
|
+
userOnRetry?.(info);
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
storage = new RetryingStorageChunkSource(storage, retryOpts);
|
|
486
|
+
}
|
|
487
|
+
this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
|
|
488
|
+
const deps = {
|
|
489
|
+
storage,
|
|
490
|
+
cache,
|
|
491
|
+
codec: roaringCodec,
|
|
492
|
+
// the facade injects the flagship codec; core stays codec-agnostic
|
|
493
|
+
clock,
|
|
494
|
+
metrics,
|
|
495
|
+
budget: this.budget
|
|
496
|
+
};
|
|
497
|
+
this.engine = new SegmentEngine(deps);
|
|
498
|
+
this.cache = cache;
|
|
499
|
+
this.crbmSource = resolved.source instanceof CrbmStorageChunkSource ? resolved.source : void 0;
|
|
500
|
+
this.clock = clock;
|
|
501
|
+
this.metrics = metrics;
|
|
502
|
+
this.storageDriver = resolved.driver;
|
|
503
|
+
this.registry = resolved.registry;
|
|
504
|
+
this.keystore = options.encryption?.keystore;
|
|
505
|
+
this.requireEncryption = options.encryption?.required ?? false;
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* The write-side deps, from the store's own drivers, for the lifecycle helpers and the `*Into` verbs. Requires
|
|
509
|
+
* the store to have been constructed with a **backend**, which supplies both halves: an `IStorageDriver` to
|
|
510
|
+
* write generations through (a pre-built `StorageChunkSource` has none) and the registry holding the pointer
|
|
511
|
+
* every write publishes.
|
|
512
|
+
* Out-of-process callers use the free functions with explicit deps.
|
|
513
|
+
*/
|
|
514
|
+
lifecycleDeps(op) {
|
|
515
|
+
if (this.storageDriver === void 0) {
|
|
516
|
+
throw new UnsupportedError(
|
|
517
|
+
`${op} needs the store built with a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A pre-built StorageChunkSource is read-only: it has no IStorageDriver underneath to write generations through. Out of process, call the equivalent free function with explicit deps instead.`
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
if (this.registry === void 0) {
|
|
521
|
+
throw new UnsupportedError(
|
|
522
|
+
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no longer a separate \`registry\` option to add.`
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
return {
|
|
526
|
+
storage: this.storageDriver,
|
|
527
|
+
registry: this.registry,
|
|
528
|
+
clock: this.clock,
|
|
529
|
+
codec: roaringCodec,
|
|
530
|
+
// facade injects the flagship codec
|
|
531
|
+
keystore: this.keystore,
|
|
532
|
+
requireEncryption: this.requireEncryption
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
/** Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). */
|
|
536
|
+
segment(name, options) {
|
|
537
|
+
const ref = { segment: name, namespace: options?.namespace };
|
|
538
|
+
validateSegmentRef(ref);
|
|
539
|
+
const expiresAt = options?.expiresAt;
|
|
540
|
+
if (expiresAt !== void 0) {
|
|
541
|
+
if (!Number.isFinite(expiresAt) || !Number.isInteger(expiresAt) || expiresAt < MIN_EXPIRES_AT_MS) {
|
|
542
|
+
throw new ValidationError(
|
|
543
|
+
`segment: expiresAt must be an integer epoch-MILLISECONDS >= ${MIN_EXPIRES_AT_MS}; got ${expiresAt} (a value in seconds lands in 1970 and would make the segment read as permanently empty)`
|
|
544
|
+
);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
return new Segment(
|
|
548
|
+
this.engine,
|
|
549
|
+
ref,
|
|
550
|
+
this.clock,
|
|
551
|
+
this.metrics,
|
|
552
|
+
(dest, ids, op, options2) => this.materialize(dest, ids, op, options2),
|
|
553
|
+
(r, e) => this.pinSegment(r, e),
|
|
554
|
+
(handles) => this.engineForCombine(handles),
|
|
555
|
+
expiresAt
|
|
556
|
+
);
|
|
557
|
+
}
|
|
558
|
+
/**
|
|
559
|
+
* Write `ids` as a **new generation of `dest`** and publish it forward-only — the shared body of the `*Into`
|
|
560
|
+
* verbs. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
561
|
+
* within `cache.genTtlMs`).
|
|
562
|
+
*
|
|
563
|
+
* This routes through `loadSegment` rather than writing the generation itself, and that is the whole point
|
|
564
|
+
* of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
|
|
565
|
+
* everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
|
|
566
|
+
* guard runs while the old generation is still authoritative, the publish is fenced (on the pointer it
|
|
567
|
+
* judged, on the row's identity, and — where it judged an ABSENT segment — on that absence), and a refused
|
|
568
|
+
* object is reclaimed only after re-reading the row and finding the same incarnation (hard invariant 1:
|
|
569
|
+
* deleting it after a purge-and-recreate would put a live row over a missing generation).
|
|
570
|
+
*
|
|
571
|
+
* That last check narrows the window rather than closing it: the row read and the delete are two round
|
|
572
|
+
* trips, and `IStorageDriver` has no conditional delete to make them one. `gcOrphanGenerations` carries
|
|
573
|
+
* the same residual and says so. The failure it leaves is an orphan object, which costs storage until
|
|
574
|
+
* something collects it — deliberately the cheaper side of the trade.
|
|
575
|
+
*
|
|
576
|
+
* Materialising used to do none of that. It wrote and published in one step, so an empty combine — a typo'd
|
|
577
|
+
* operand, an `exclude` that swallowed everything, an operand that had not loaded yet — silently replaced
|
|
578
|
+
* `dest` with an empty generation. That is the same failure `load()`'s guard exists to prevent, on the same
|
|
579
|
+
* data, and it was reachable without passing any option at all.
|
|
580
|
+
*
|
|
581
|
+
* **A lost race still throws.** `loadSegment` reports one as `reason: 'superseded'`; the `*Into` verbs have
|
|
582
|
+
* always thrown {@link WriteConflictError} for it, and a caller who wrote `catch (WriteConflictError)` must
|
|
583
|
+
* keep working. So that one refusal is translated back into the throw, and `MaterializeResult.reason` never
|
|
584
|
+
* carries it.
|
|
585
|
+
*
|
|
586
|
+
* **A `WriteConflictError` does not by itself mean nothing was published**, and that is worth knowing
|
|
587
|
+
* before you write the retry. `'superseded'` covers four different causes — the write-once PUT collided,
|
|
588
|
+
* the pointer moved, the row's token changed, the row was purged — and only the first two are the
|
|
589
|
+
* "somebody beat us" the name suggests. A token can also change on a write that is not a supersession at
|
|
590
|
+
* all, such as a `setRetention` on the destination. On top of that, the collection pass that runs AFTER a
|
|
591
|
+
* successful publish can raise the same error. So: treat it as "re-read the destination and decide",
|
|
592
|
+
* never as "the write did not happen".
|
|
593
|
+
*
|
|
594
|
+
* **And it still collects nothing**, unlike `load()`. See the `keep` default below.
|
|
595
|
+
*/
|
|
596
|
+
async materialize(dest, ids, op, options) {
|
|
597
|
+
const deps = this.lifecycleDeps(op);
|
|
598
|
+
const result = await loadSegment(dest, ids, deps, {
|
|
599
|
+
...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
|
|
600
|
+
...options?.guard === void 0 ? {} : { guard: options.guard },
|
|
601
|
+
// COLLECT NOTHING by default, which `loadSegment` does not — it keeps a grace window of 1 and deletes
|
|
602
|
+
// the rest. A materialisation has never collected: the guide states "**It deletes nothing.** The
|
|
603
|
+
// destination's previous generation stays in the bucket until you collect it", and the ownership table
|
|
604
|
+
// puts that call on the operator. Inheriting `load()`'s collection would have silently deleted the
|
|
605
|
+
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
606
|
+
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
607
|
+
keep: options?.keep ?? KEEP_EVERY_GENERATION,
|
|
608
|
+
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
609
|
+
});
|
|
610
|
+
const reason = result.reason;
|
|
611
|
+
if (reason === "superseded") {
|
|
612
|
+
const wrote = result.size > 0 ? `generation ${result.generation} was written and did not become current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
613
|
+
throw new WriteConflictError(
|
|
614
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so it never became current: ${wrote}. The pointer may have moved, the row may have been rewritten (a retention policy does this) or purged. Re-read the destination and re-run.`
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
return {
|
|
618
|
+
generation: result.generation,
|
|
619
|
+
published: result.published,
|
|
620
|
+
...reason === void 0 ? {} : { reason },
|
|
621
|
+
cardinality: result.cardinality,
|
|
622
|
+
cardinalityBefore: result.cardinalityBefore,
|
|
623
|
+
chunkCount: result.chunkCount,
|
|
624
|
+
size: result.size,
|
|
625
|
+
collected: result.collected
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?**
|
|
630
|
+
*
|
|
631
|
+
* Enumerates the **registered** segments (via the store's own `registry`) and does a `has(id)` on each — no
|
|
632
|
+
* drivers to re-pass. Complete only over registered segments (every loaded segment has a row, so register the
|
|
633
|
+
* registry the loads used). There is deliberately **no `id → segments` reverse index** — that would tax every
|
|
634
|
+
* load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Requires a
|
|
635
|
+
* `registry` in the store config (throws {@link UnsupportedError} otherwise).
|
|
636
|
+
*/
|
|
637
|
+
async subjectReport(id, options = {}) {
|
|
638
|
+
const registry = this.requireRegistry("subjectReport");
|
|
639
|
+
requireScope(options, "subjectReport");
|
|
640
|
+
validateConcurrency(options.concurrency);
|
|
641
|
+
const budget = resolvePerOpBudget(options.budget, this.budget);
|
|
642
|
+
splitId(id);
|
|
643
|
+
const recs = await collectWithinBudget(
|
|
644
|
+
// A subject cannot be in a coordination row, and charging this request's budget for them would refuse a
|
|
645
|
+
// GDPR Art. 15 report for a reason unrelated to the subject.
|
|
646
|
+
excludingReservedRows(registry.list(options.namespace)),
|
|
647
|
+
budget,
|
|
648
|
+
"subjectReport"
|
|
649
|
+
);
|
|
650
|
+
const membership = await mapWithConcurrency(
|
|
651
|
+
recs,
|
|
652
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
653
|
+
async (rec) => {
|
|
654
|
+
if (rec.status === "destroyed") return null;
|
|
655
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
656
|
+
return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
657
|
+
}
|
|
658
|
+
);
|
|
659
|
+
const segments = membership.filter((m) => m !== null);
|
|
660
|
+
return { id, segments, scannedSegments: recs.length };
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
664
|
+
* from the bucket on return.**
|
|
665
|
+
*
|
|
666
|
+
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
667
|
+
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
668
|
+
* generation it was derived from**, and
|
|
669
|
+
* collects the generation that held the bit. The returned per-segment record is your **erasure ledger** —
|
|
670
|
+
* persist it / route it to your audit sink as the proof of deletion (a `segment.rewrite` audit event is also
|
|
671
|
+
* emitted per rewrite when you pass `audit`).
|
|
672
|
+
*
|
|
673
|
+
* Uses the backend's **own** two halves, so the membership check and the rewrite provably run
|
|
674
|
+
* over the same generation. Requires the store built with a **backend** (throws
|
|
675
|
+
* {@link UnsupportedError} otherwise; a pre-built `StorageChunkSource` store has no `IStorageDriver` to write
|
|
676
|
+
* through — use the `eraseIdFromSegment` free function there).
|
|
677
|
+
*
|
|
678
|
+
* **One contract remains** (an integrator obligation the library cannot check): **do not load the segment
|
|
679
|
+
* while erasing from it.** A load that lands after the rewrite carries whatever its source held, and the
|
|
680
|
+
* library cannot know that source was meant to exclude the id. Quiesce loads of the affected segments for the
|
|
681
|
+
* duration, or fix the source first and load after. A writer that lands *during* the rewrite is caught: the
|
|
682
|
+
* rewrite's publish is refused **by the fence** — `publishGeneration`'s `expectFrom`, which lands the CAS only
|
|
683
|
+
* while the pointer is still on the generation the rewrite streamed — and the entry says `note: 'superseded'`,
|
|
684
|
+
* so re-run. Forward-only alone would NOT refuse it: `nextGeneration` numbers above everything in the bucket,
|
|
685
|
+
* so the rewrite would out-rank the newer generation and then collect it.
|
|
686
|
+
*
|
|
687
|
+
* A racing **erasure** is caught before that, and reported the same way. It collects with `keep: 0`, taking
|
|
688
|
+
* every generation below its new pointer — the one this rewrite is streaming, and the object this rewrite
|
|
689
|
+
* just wrote — so the loser can find its own inputs deleted mid-flight. That surfaces as a reason rather than
|
|
690
|
+
* an error, read off the row: a moved pointer is `'superseded'`, a concurrent `dropSegment` `'destroyed'`,
|
|
691
|
+
* a retention sweep that purged the row `'absent'`. Re-running is the fix in every case.
|
|
692
|
+
*
|
|
693
|
+
* **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
|
|
694
|
+
* another writer (a load, or another erasure) moved the pointer mid-rewrite, so **this call** did not erase
|
|
695
|
+
* the id. Re-run: it erases the id if it is still there, and lists nothing for the segment if a racing
|
|
696
|
+
* erasure of the same id already removed it. Do not read `'superseded'` as "the id is still present" —
|
|
697
|
+
* read it as "not done by this call, and the re-run settles it".
|
|
698
|
+
* `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
|
|
699
|
+
* on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
|
|
700
|
+
* it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
|
|
701
|
+
* absent from every read and what remains is an object in the bucket that still contains the bit. A re-run
|
|
702
|
+
* then reports nothing for that segment (the id is not in the current generation), so **that residual is
|
|
703
|
+
* collected by `gcOrphanGenerations(ref, deps, { keep: 0 })` or the next retention sweep**, not by another
|
|
704
|
+
* `eraseSubject`. Re-running is otherwise safe and idempotent: a segment the id is no longer in is not listed. Admin-only path;
|
|
705
|
+
* `O(registered segments)`, no hot-path cost. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
706
|
+
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
707
|
+
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
708
|
+
*/
|
|
709
|
+
async eraseSubject(id, options = {}) {
|
|
710
|
+
const deps = this.lifecycleDeps("eraseSubject");
|
|
711
|
+
requireScope(options, "eraseSubject");
|
|
712
|
+
validateConcurrency(options.concurrency);
|
|
713
|
+
const budget = resolvePerOpBudget(options.budget, this.budget);
|
|
714
|
+
splitId(id);
|
|
715
|
+
const recs = await collectWithinBudget(
|
|
716
|
+
excludingReservedRows(deps.registry.list(options.namespace)),
|
|
717
|
+
budget,
|
|
718
|
+
"eraseSubject"
|
|
719
|
+
);
|
|
720
|
+
const entries = await mapWithConcurrency(
|
|
721
|
+
recs,
|
|
722
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
723
|
+
async (rec) => {
|
|
724
|
+
if (rec.status === "destroyed") return null;
|
|
725
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
726
|
+
try {
|
|
727
|
+
const result = await eraseIdFromSegment(ref, id, deps, { audit: options.audit });
|
|
728
|
+
if (result.reason === "not-member" || result.reason === "absent") return null;
|
|
729
|
+
if (result.reason === "no-generation") return null;
|
|
730
|
+
if (result.reason === "destroyed") return null;
|
|
731
|
+
return {
|
|
732
|
+
segment: rec.segment,
|
|
733
|
+
namespace: rec.namespace,
|
|
734
|
+
erased: result.erased,
|
|
735
|
+
fromGeneration: result.fromGeneration,
|
|
736
|
+
generation: result.generation,
|
|
737
|
+
note: result.erased ? void 0 : result.reason
|
|
738
|
+
};
|
|
739
|
+
} catch (err) {
|
|
740
|
+
return {
|
|
741
|
+
segment: rec.segment,
|
|
742
|
+
namespace: rec.namespace,
|
|
743
|
+
erased: false,
|
|
744
|
+
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
745
|
+
};
|
|
746
|
+
} finally {
|
|
747
|
+
try {
|
|
748
|
+
this.engine.invalidate(ref);
|
|
749
|
+
} catch {
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
);
|
|
754
|
+
const erasedFrom = entries.filter((e) => e !== null);
|
|
755
|
+
return { id, erasedFrom, scannedSegments: recs.length };
|
|
756
|
+
}
|
|
757
|
+
/**
|
|
758
|
+
* Every generation still in the bucket for this segment, ascending, with the current one marked.
|
|
759
|
+
*
|
|
760
|
+
* What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
|
|
761
|
+
* the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
|
|
762
|
+
* choose from, which is the reason to look at it. One `list` call; it does not open the objects.
|
|
763
|
+
*
|
|
764
|
+
* Needs a backend.
|
|
765
|
+
*/
|
|
766
|
+
async generations(ref) {
|
|
767
|
+
validateSegmentRef(ref);
|
|
768
|
+
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
769
|
+
}
|
|
770
|
+
/**
|
|
771
|
+
* Whether a read of this segment would find anything — the "do I already have this?" question, as one
|
|
772
|
+
* registry point read.
|
|
773
|
+
*
|
|
774
|
+
* There is no `create` in this library: {@link CloudRoaring.segment} is a validated address and does no I/O,
|
|
775
|
+
* so naming a segment can never collide with an existing one. A segment starts existing when something is
|
|
776
|
+
* first loaded into it, and this is how you ask whether that has happened.
|
|
777
|
+
*
|
|
778
|
+
* **Not the same as `count() > 0`.** A segment loaded with no ids exists and counts zero. Distinguishing
|
|
779
|
+
* "never loaded" from "loaded, and genuinely empty" is the thing `count()` cannot do and the reason this
|
|
780
|
+
* exists. It is `false` for a segment whose row was minted ahead of its first load (by `setRetention`) and
|
|
781
|
+
* for a `destroyed` tombstone, because a read answers empty in both cases.
|
|
782
|
+
*
|
|
783
|
+
* Two states answer `true` where a read still gives you nothing: a torn restore (a live pointer whose object
|
|
784
|
+
* was deleted) makes reads *throw* rather than answer empty — `checkConsistency` is the call for that — and
|
|
785
|
+
* a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
|
|
786
|
+
*
|
|
787
|
+
* Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
|
|
788
|
+
* `load`'s `guard`, or `expectFrom`/`expectToken` on a publish. Needs a `registry`.
|
|
789
|
+
*
|
|
790
|
+
* ```ts
|
|
791
|
+
* if (!(await store.exists({ segment: 'users' }))) {
|
|
792
|
+
* await store.load({ segment: 'users' }, idsFromUpstream);
|
|
793
|
+
* }
|
|
794
|
+
* ```
|
|
795
|
+
*/
|
|
796
|
+
async exists(ref) {
|
|
797
|
+
validateSegmentRef(ref);
|
|
798
|
+
return segmentExists(ref, this.requireRegistry("exists"));
|
|
799
|
+
}
|
|
800
|
+
/**
|
|
801
|
+
* Every segment the registry knows about, streamed — optionally scoped to one namespace.
|
|
802
|
+
*
|
|
803
|
+
* The registry is already the list of your segments, which is why you should not keep a second one beside it:
|
|
804
|
+
* a hand-maintained list is a source of truth that drifts from this one the first time a load fails halfway.
|
|
805
|
+
*
|
|
806
|
+
* **An admin/discovery call, not a request-path one.** This is the registry's own enumeration — a paged
|
|
807
|
+
* LIST over the `registry/` prefix — so its cost grows with the size of the fleet rather than with what you
|
|
808
|
+
* are looking for.
|
|
809
|
+
*
|
|
810
|
+
* Scoping to a namespace narrows the LIST prefix, so it really is the difference between reading one tenant
|
|
811
|
+
* and reading all of them.
|
|
812
|
+
*
|
|
813
|
+
* It streams, and stopping the iteration stops the scan — except behind a driver that buffers its
|
|
814
|
+
* enumeration to retry it as a unit, which `RetryingRegistryDriver` does: wrapped in that, the whole scan is
|
|
815
|
+
* paid for and resident before the first row arrives.
|
|
816
|
+
*
|
|
817
|
+
* Yields `destroyed` tombstones and rows with `currentGen: null`, because a filtered enumeration that looks
|
|
818
|
+
* complete is worse than an honest one — filter on `status`/`currentGen` yourself, or ask
|
|
819
|
+
* {@link CloudRoaring.exists} the narrower question. Needs a `registry`.
|
|
820
|
+
*
|
|
821
|
+
* ```ts
|
|
822
|
+
* for await (const s of store.segments({ namespace: 'active-daily' })) {
|
|
823
|
+
* console.log(s.segment, s.currentGen, s.status);
|
|
824
|
+
* }
|
|
825
|
+
* ```
|
|
826
|
+
*/
|
|
827
|
+
segments(options = {}) {
|
|
828
|
+
if (options.namespace !== void 0) {
|
|
829
|
+
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
830
|
+
}
|
|
831
|
+
return listSegments(this.requireRegistry("segments"), options);
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Move this segment's pointer **back** to a generation still in the bucket — the one write in the library that
|
|
835
|
+
* is not forward-only.
|
|
836
|
+
*
|
|
837
|
+
* Forward-only is right for a writer: a load whose ids came from upstream loses nothing by being out-raced, and
|
|
838
|
+
* letting it regress would let a slow loader silently undo a fast one. It is wrong for an operator who has
|
|
839
|
+
* looked at the segment, decided the current generation is wrong, and knows which one they want. So this is
|
|
840
|
+
* reachable only by asking for it by name — no sweep, retry or reconciliation calls it — and it is audited
|
|
841
|
+
* (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
|
|
842
|
+
* one cannot.
|
|
843
|
+
*
|
|
844
|
+
* It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
|
|
845
|
+
* `NotFoundError` naming what *is* available, and a crypto-shredded segment throws
|
|
846
|
+
* {@link ValidationError} because every generation of it is unreadable. Rolling to the generation already
|
|
847
|
+
* current is a no-op that reports itself.
|
|
848
|
+
*
|
|
849
|
+
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
850
|
+
* and are then *above* `currentGen`, where collection never looks, so they remain until a later load raises the
|
|
851
|
+
* pointer past them. An operator who has just undone a bad load should not have the evidence collected out from
|
|
852
|
+
* under them.
|
|
853
|
+
*
|
|
854
|
+
* Needs a backend.
|
|
855
|
+
*/
|
|
856
|
+
async rollback(ref, toGeneration, options = {}) {
|
|
857
|
+
validateSegmentRef(ref);
|
|
858
|
+
const deps = this.lifecycleDeps("rollback");
|
|
859
|
+
try {
|
|
860
|
+
return await rollbackSegment(ref, toGeneration, deps, options);
|
|
861
|
+
} finally {
|
|
862
|
+
this.engine.invalidate(ref);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* **Replace this segment's contents** with `ids`, as one new immutable generation, and make it current.
|
|
867
|
+
*
|
|
868
|
+
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
869
|
+
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
870
|
+
* the one that gets left out is the last, so segments quietly accumulate superseded generations nobody pays
|
|
871
|
+
* attention to and everybody pays for.
|
|
872
|
+
*
|
|
873
|
+
* ```ts
|
|
874
|
+
* const r = await store.load('audience:active', idsFromWarehouse, {
|
|
875
|
+
* guard: { maxShrink: 0.5 }, // refuse a load that drops more than half the segment
|
|
876
|
+
* });
|
|
877
|
+
* if (!r.published) console.warn(`load refused: ${r.reason}`);
|
|
878
|
+
* ```
|
|
879
|
+
*
|
|
880
|
+
* **A load REPLACES.** Whatever the stream contains is what the segment contains afterwards, so an upstream
|
|
881
|
+
* query that returns fewer rows than usual is a shrink nobody asked for and an empty one is a wipe — both
|
|
882
|
+
* reported as a successful write, because at the storage layer they are one. That is what `guard` and the
|
|
883
|
+
* default empty refusal are for, and why they run **between** the write and the publish: the only moment where
|
|
884
|
+
* the new content is known and the old one is still authoritative.
|
|
885
|
+
*
|
|
886
|
+
* **Branch on `published`.** A refusal is a normal outcome, not an exception — `published: false` with a
|
|
887
|
+
* `reason`, in the same shape as a success — so a discarded result is a load that silently did nothing, and
|
|
888
|
+
* with the empty guard on by default this verb refuses more readily than any other. Note the sibling `*Into`
|
|
889
|
+
* verbs *throw* on the same superseded condition rather than reporting it; this one follows
|
|
890
|
+
* `eraseIdFromSegment` instead, because three of its four refusals are expected guard outcomes rather than
|
|
891
|
+
* faults.
|
|
892
|
+
*
|
|
893
|
+
* The object written for a refused load is deleted again before returning — it sits above `currentGen`, where
|
|
894
|
+
* generation collection deliberately never looks, so nothing else would reclaim it. The exception is a load
|
|
895
|
+
* that finds the segment **re-created** underneath it: the generation number it holds may then name the new
|
|
896
|
+
* incarnation's live object, so it leaves the orphan rather than risk deleting live data.
|
|
897
|
+
*
|
|
898
|
+
* Two things it does **throw** for, rather than report: a crypto-shredded segment (`ValidationError` — there
|
|
899
|
+
* is no key to write under), and a collection pass that could not prove the segment was still the same one
|
|
900
|
+
* (`WriteConflictError`). The second can be raised **after** the publish already landed, so a throw does not
|
|
901
|
+
* by itself mean the load did not take effect — re-read the pointer rather than assuming.
|
|
902
|
+
*
|
|
903
|
+
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
904
|
+
*/
|
|
905
|
+
async load(ref, ids, options = {}) {
|
|
906
|
+
validateSegmentRef(ref);
|
|
907
|
+
const deps = this.lifecycleDeps("load");
|
|
908
|
+
try {
|
|
909
|
+
return await loadSegment(ref, ids, deps, options);
|
|
910
|
+
} finally {
|
|
911
|
+
this.engine.invalidate(ref);
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
/**
|
|
915
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** Irreversible.
|
|
916
|
+
*
|
|
917
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
|
|
918
|
+
* backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
|
|
919
|
+
* removes the storage and works on a cleartext segment. On an encrypted segment it does both.
|
|
920
|
+
*
|
|
921
|
+
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
922
|
+
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
923
|
+
* variable twice.
|
|
924
|
+
*
|
|
925
|
+
* ```ts
|
|
926
|
+
* for (const day of expiredDays) {
|
|
927
|
+
* // The family in the namespace, the date in the segment — `registry.list(namespace)` then enumerates
|
|
928
|
+
* // exactly this family's buckets, which a flat `active:2026-08-01` name cannot do without string-matching.
|
|
929
|
+
* const ref = { namespace: 'active-daily', segment: day };
|
|
930
|
+
* await store.dropSegment(ref, { confirmSegment: ref.segment });
|
|
931
|
+
* }
|
|
932
|
+
* ```
|
|
933
|
+
*
|
|
934
|
+
* **Omitting the namespace addresses a different segment and is a silent no-op** — you get
|
|
935
|
+
* `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
|
|
936
|
+
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
937
|
+
*
|
|
938
|
+
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
939
|
+
* reclaimed and the drop should be re-run. A load that was already writing when the tombstone landed still
|
|
940
|
+
* finishes its object, so a single sweep can miss it — this call re-sweeps and then reports whatever it still
|
|
941
|
+
* could not remove rather than returning a result that looks like a clean drop.
|
|
942
|
+
*
|
|
943
|
+
* Reads become empty within `cache.genTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
944
|
+
* segment may answer from its cached generation + cached chunks until that window lapses. A reader that never
|
|
945
|
+
* touched it sees empty at once. **That bound needs a clock and `cache.genTtlMs > 0`** — a store built without a
|
|
946
|
+
* clock, or with `cache.genTtlMs: 0` ("pin forever"), holds its resolved snapshot for its own lifetime and can
|
|
947
|
+
* keep answering `true` for a dropped segment indefinitely; restart it.
|
|
948
|
+
*
|
|
949
|
+
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
950
|
+
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
951
|
+
*/
|
|
952
|
+
async dropSegment(ref, options) {
|
|
953
|
+
validateSegmentRef(ref);
|
|
954
|
+
const deps = this.lifecycleDeps("dropSegment");
|
|
955
|
+
try {
|
|
956
|
+
return await dropSegment(ref, { registry: deps.registry, storage: deps.storage }, options);
|
|
957
|
+
} finally {
|
|
958
|
+
if (options.dryRun !== true) this.engine.invalidate(ref);
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* **Record when this segment becomes eligible for retirement.** One registry write; nothing is deleted here,
|
|
963
|
+
* and nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
964
|
+
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
965
|
+
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
966
|
+
*
|
|
967
|
+
* `expiresAt` is an **absolute epoch-ms you compute**, not a duration the library derives. A relative TTL would
|
|
968
|
+
* have to be anchored to something the library knows — `updatedAt`, or the current generation — and every load
|
|
969
|
+
* rewrites both, so "expire 30 days after the last write" would keep a busy daily bucket alive forever
|
|
970
|
+
* precisely because it is being reloaded.
|
|
971
|
+
*
|
|
972
|
+
* ```ts
|
|
973
|
+
* const DAY = 86_400_000;
|
|
974
|
+
* const ref = { namespace: 'active-daily', segment: '2026-08-05' };
|
|
975
|
+
* await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY });
|
|
976
|
+
* ```
|
|
977
|
+
*
|
|
978
|
+
* **Works before the first load**: a segment with no registry row yet gets one (`createdRow: true` in the
|
|
979
|
+
* result) with **no Storage generation**, so the policy is recorded ahead of the data and the segment is already
|
|
980
|
+
* enumerable by the sweep; the first load then publishes onto that row.
|
|
981
|
+
*
|
|
982
|
+
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
983
|
+
* buckets is normal. A value below `MIN_EXPIRES_AT_MS` (2001-09-09) is rejected: it is almost certainly epoch
|
|
984
|
+
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a `registry`
|
|
985
|
+
* in the store config (throws {@link UnsupportedError} otherwise), and refuses a crypto-shredded segment.
|
|
986
|
+
*/
|
|
987
|
+
async setRetention(ref, policy) {
|
|
988
|
+
validateSegmentRef(ref);
|
|
989
|
+
return setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
|
|
990
|
+
}
|
|
991
|
+
/**
|
|
992
|
+
* **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
|
|
993
|
+
* tombstone). Returns the string `'invalid'` for a row whose `expiresAt` is present but unusable — a
|
|
994
|
+
* hand-edited row, or one from a restore — so a malformed policy is visible rather than silently reading as
|
|
995
|
+
* "never expires" on a segment someone believes is expiring.
|
|
996
|
+
*/
|
|
997
|
+
async getRetention(ref) {
|
|
998
|
+
validateSegmentRef(ref);
|
|
999
|
+
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether a policy was actually removed (`false`
|
|
1003
|
+
* when there was none). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
1004
|
+
* to the setter is how a typo becomes a deletion.
|
|
1005
|
+
*/
|
|
1006
|
+
async clearRetention(ref) {
|
|
1007
|
+
validateSegmentRef(ref);
|
|
1008
|
+
return clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
|
|
1009
|
+
}
|
|
1010
|
+
/**
|
|
1011
|
+
* **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
|
|
1012
|
+
* the policies `setRetention` records — and it is a **call, not a daemon**. Nothing schedules it; you run it from
|
|
1013
|
+
* whatever heartbeat your deployment already has (an EventBridge rule, a Kubernetes CronJob, a queue consumer,
|
|
1014
|
+
* the job that runs your loads). A library that started a timer would behave differently in a Lambda, an edge
|
|
1015
|
+
* isolate and a long-lived server, which is worse than not having one.
|
|
1016
|
+
*
|
|
1017
|
+
* Each retirement goes through `dropSegment`, so the registry → Storage ordering, the re-sweep for an object a
|
|
1018
|
+
* load was still writing, and the `generationsRemaining` report all come from one implementation rather than
|
|
1019
|
+
* two.
|
|
1020
|
+
*
|
|
1021
|
+
* ```ts
|
|
1022
|
+
* // In your scheduled handler. Start with a preview in a new deployment.
|
|
1023
|
+
* const preview = await store.retireExpired({ namespace: 'active-daily', dryRun: true });
|
|
1024
|
+
* console.log(`would retire ${preview.retired} of ${preview.scanned} (limited: ${preview.limited})`);
|
|
1025
|
+
*
|
|
1026
|
+
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
1027
|
+
* for (const e of swept.entries) {
|
|
1028
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`); // invalid-policy / limit / failed: …
|
|
1029
|
+
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
1030
|
+
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
1031
|
+
* }
|
|
1032
|
+
* }
|
|
1033
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more are still eligible
|
|
1034
|
+
* ```
|
|
1035
|
+
*
|
|
1036
|
+
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
1037
|
+
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
1038
|
+
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
1039
|
+
* `limit` (default 100) cut the pass short and more are eligible. That cap is charged on **attempts**, not
|
|
1040
|
+
* successes, which is what makes it a real bound: `dropSegment` writes the tombstone before sweeping Storage, so a
|
|
1041
|
+
* fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
1042
|
+
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
1043
|
+
* dashboard summing `retired` can never show a phantom deletion.
|
|
1044
|
+
*
|
|
1045
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
1046
|
+
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
1047
|
+
* first, since nothing else ever would for a tombstoned segment. Attribution is a **positive marker the sweep
|
|
1048
|
+
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
1049
|
+
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
1050
|
+
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. Pass
|
|
1051
|
+
* `purgeTombstones: false` to keep every tombstone.
|
|
1052
|
+
*
|
|
1053
|
+
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
1054
|
+
* because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
|
|
1055
|
+
*/
|
|
1056
|
+
async retireExpired(options = {}) {
|
|
1057
|
+
const deps = this.lifecycleDeps("retireExpired");
|
|
1058
|
+
const result = await retireExpired(
|
|
1059
|
+
{ registry: deps.registry, storage: deps.storage },
|
|
1060
|
+
{ ...options, now: options.now ?? this.clock.now() }
|
|
1061
|
+
);
|
|
1062
|
+
if (options.dryRun !== true) {
|
|
1063
|
+
for (const entry of result.entries) {
|
|
1064
|
+
this.engine.invalidate({ segment: entry.segment, namespace: entry.namespace });
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
return result;
|
|
1068
|
+
}
|
|
1069
|
+
/**
|
|
1070
|
+
* **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
|
|
1071
|
+
* through a path this store did not perform itself.
|
|
1072
|
+
*
|
|
1073
|
+
* A store keeps two layers of derived state — a resolved snapshot per segment (an open `.crbm` reader, plus
|
|
1074
|
+
* the DEK it unwrapped) and decoded chunks keyed by generation. Both are built for one event: **a publish
|
|
1075
|
+
* that advances `currentGen`**, which the snapshot TTL notices and the generation-keyed cache misses on.
|
|
1076
|
+
* Neither notices an event that *destroys* what they were derived from.
|
|
1077
|
+
*
|
|
1078
|
+
* The verbs on this class handle themselves — `eraseSubject`, `dropSegment` and `retireExpired` invalidate
|
|
1079
|
+
* what they touch. This method is for the cases they cannot see:
|
|
1080
|
+
*
|
|
1081
|
+
* - **`destroySegment` / `eraseNamespace`**, which are free functions over raw drivers rather than methods
|
|
1082
|
+
* here, so a crypto-shred performed beside this store leaves it holding an open reader and an unwrapped
|
|
1083
|
+
* DEK. Until it is told, it keeps decrypting — including chunks it had never fetched before the shred.
|
|
1084
|
+
* - **Another process.** Erasing on one box invalidates nothing on the others; each store bounds its own
|
|
1085
|
+
* staleness by `cache.genTtlMs`, and a store built with no clock or `cache.genTtlMs: 0` ("pin forever") never
|
|
1086
|
+
* converges at all. If a compliance deadline depends on every reader converging, you need to signal them —
|
|
1087
|
+
* this is the call to make when your own fan-out delivers.
|
|
1088
|
+
*
|
|
1089
|
+
* Synchronous, best-effort, and safe to call for a segment this store has never read.
|
|
1090
|
+
*
|
|
1091
|
+
* ```ts
|
|
1092
|
+
* await destroySegment(ref, { storage, registry, keystore }, { confirmSegment: ref.segment });
|
|
1093
|
+
* store.invalidate(ref); // this process
|
|
1094
|
+
* await bus.publish('cloudbitmaps.invalidate', ref); // and every other one
|
|
1095
|
+
* ```
|
|
1096
|
+
*/
|
|
1097
|
+
invalidate(ref) {
|
|
1098
|
+
validateSegmentRef(ref);
|
|
1099
|
+
this.engine.invalidate(ref);
|
|
1100
|
+
}
|
|
1101
|
+
/**
|
|
1102
|
+
* Build a handle held at the generation `ref` resolves to right now. See {@link Segment.pin}.
|
|
1103
|
+
*
|
|
1104
|
+
* The pinned handle gets its own engine but **shares the store's chunk cache**, which is safe precisely
|
|
1105
|
+
* because the cache is keyed by the source's version rather than the generation number: the pinned view
|
|
1106
|
+
* reports the version captured at pin time, so its decoded chunks cannot collide with the live generation's.
|
|
1107
|
+
* Sharing it on a generation-only key was how a pinned read could resurrect an id already reported
|
|
1108
|
+
* physically gone.
|
|
1109
|
+
*/
|
|
1110
|
+
async pinSegment(ref, expiresAt) {
|
|
1111
|
+
const crbm = this.crbmSource;
|
|
1112
|
+
if (crbm === void 0) {
|
|
1113
|
+
throw new UnsupportedError(
|
|
1114
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
const at = await crbm.pinGeneration(ref);
|
|
1118
|
+
const pinnedAt = { generation: at?.generation ?? null, version: at?.version ?? null };
|
|
1119
|
+
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
1120
|
+
return new Segment(
|
|
1121
|
+
this.engineWithPins(pins),
|
|
1122
|
+
ref,
|
|
1123
|
+
this.clock,
|
|
1124
|
+
this.metrics,
|
|
1125
|
+
(dest, ids, op, options) => this.materialize(dest, ids, op, options),
|
|
1126
|
+
(r, e) => this.pinSegment(r, e),
|
|
1127
|
+
(handles) => this.engineForCombine(handles),
|
|
1128
|
+
expiresAt,
|
|
1129
|
+
pinnedAt
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
/**
|
|
1133
|
+
* An engine reading every segment in `pins` at its pinned generation and everything else live.
|
|
1134
|
+
*
|
|
1135
|
+
* It **shares the store's chunk cache**, which is safe only because that cache is keyed by the source's
|
|
1136
|
+
* version rather than the generation number: a pinned view reports the version captured at pin time, so its
|
|
1137
|
+
* decoded chunks cannot collide with the live generation's.
|
|
1138
|
+
*/
|
|
1139
|
+
engineWithPins(pins) {
|
|
1140
|
+
const crbm = this.crbmSource;
|
|
1141
|
+
if (crbm === void 0) {
|
|
1142
|
+
throw new UnsupportedError(
|
|
1143
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
return new SegmentEngine({
|
|
1147
|
+
storage: new PinnedStorageChunkSource(crbm, pins),
|
|
1148
|
+
cache: this.cache,
|
|
1149
|
+
codec: roaringCodec,
|
|
1150
|
+
clock: this.clock,
|
|
1151
|
+
metrics: this.metrics,
|
|
1152
|
+
budget: this.budget
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
/**
|
|
1156
|
+
* The engine a combine should run on, given the handles involved.
|
|
1157
|
+
*
|
|
1158
|
+
* A pinned handle passed as an **operand** must still be read at its pin. Routing by "which handle the call
|
|
1159
|
+
* was made on" reads it live instead — `snap.intersect([other])` honours the pin while
|
|
1160
|
+
* `other.intersect([snap])` silently does not, and the two are the same question. So the combine collects
|
|
1161
|
+
* every pin in play and runs on an engine that honours all of them. With no pins anywhere this is the
|
|
1162
|
+
* store's own engine and costs nothing.
|
|
1163
|
+
*/
|
|
1164
|
+
engineForCombine(handles) {
|
|
1165
|
+
const pins = /* @__PURE__ */ new Map();
|
|
1166
|
+
for (const h of handles) {
|
|
1167
|
+
const at = h.pinnedAt;
|
|
1168
|
+
if (at !== void 0) pins.set(h.key(), at);
|
|
1169
|
+
}
|
|
1170
|
+
return pins.size === 0 ? void 0 : this.engineWithPins(pins);
|
|
1171
|
+
}
|
|
1172
|
+
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
1173
|
+
requireRegistry(op) {
|
|
1174
|
+
if (this.registry === void 0) {
|
|
1175
|
+
throw new UnsupportedError(
|
|
1176
|
+
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no longer a separate \`registry\` option to add.`
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
return this.registry;
|
|
1180
|
+
}
|
|
1181
|
+
/**
|
|
1182
|
+
* **Cross-tier DR consistency check.** After a restore/failover, verify every registered segment's `currentGen`
|
|
1183
|
+
* actually has its `.crbm` present in Storage — catching a **torn restore** where the registry (`currentGen`) came
|
|
1184
|
+
* back ahead of the object store, so a pointer references a generation that isn't there (reads would then
|
|
1185
|
+
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent }` —
|
|
1186
|
+
* `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object store, or
|
|
1187
|
+
* roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
1188
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded) segments are skipped. Pair it with
|
|
1189
|
+
* the DR runbook (docs/guide/disaster-recovery.md).
|
|
1190
|
+
*/
|
|
1191
|
+
async checkConsistency(options = {}) {
|
|
1192
|
+
const deps = this.lifecycleDeps("checkConsistency");
|
|
1193
|
+
return runConsistencyCheck({ storage: deps.storage, registry: deps.registry }, options);
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
1197
|
+
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
1198
|
+
* data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
|
|
1199
|
+
* (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
|
|
1200
|
+
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
1201
|
+
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
1202
|
+
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
1203
|
+
* Crypto-shredded segments are skipped. A segment that can't be read is isolated into the manifest's `failed[]`
|
|
1204
|
+
* (the run continues), so "a manifest exists" means the run finished — check `failed`. The `export-segments`
|
|
1205
|
+
* CLI wraps this with a filesystem sink.
|
|
1206
|
+
*/
|
|
1207
|
+
async exportSegments(sink, options = {}) {
|
|
1208
|
+
const registry = this.requireRegistry("exportSegments");
|
|
1209
|
+
return runExport2(this, registry, sink, {
|
|
1210
|
+
...options,
|
|
1211
|
+
codec: options.codec ?? roaringCodec
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
|
|
1216
|
+
* from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
1217
|
+
*/
|
|
1218
|
+
static estimateCost(input) {
|
|
1219
|
+
return estimateCost(input);
|
|
1220
|
+
}
|
|
1221
|
+
};
|
|
1222
|
+
var EMPTY_IDS = {
|
|
1223
|
+
async *[Symbol.asyncIterator]() {
|
|
1224
|
+
}
|
|
1225
|
+
};
|
|
1226
|
+
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
1227
|
+
var Segment = class {
|
|
1228
|
+
constructor(engine, ref, clock, metrics, materialize, pinned, combineEngine, expiresAt, pinnedAt) {
|
|
1229
|
+
this.engine = engine;
|
|
1230
|
+
this.ref = ref;
|
|
1231
|
+
this.clock = clock;
|
|
1232
|
+
this.metrics = metrics;
|
|
1233
|
+
this.materialize = materialize;
|
|
1234
|
+
this.pinned = pinned;
|
|
1235
|
+
this.combineEngine = combineEngine;
|
|
1236
|
+
this.expiresAt = expiresAt;
|
|
1237
|
+
this.pinnedAt = pinnedAt;
|
|
1238
|
+
this.metricsOn = metrics !== NOOP_METRICS;
|
|
1239
|
+
}
|
|
1240
|
+
engine;
|
|
1241
|
+
ref;
|
|
1242
|
+
clock;
|
|
1243
|
+
metrics;
|
|
1244
|
+
materialize;
|
|
1245
|
+
pinned;
|
|
1246
|
+
combineEngine;
|
|
1247
|
+
expiresAt;
|
|
1248
|
+
pinnedAt;
|
|
1249
|
+
metricsOn;
|
|
1250
|
+
/**
|
|
1251
|
+
* **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
|
|
1252
|
+
*
|
|
1253
|
+
* An ordinary handle re-resolves on `cache.genTtlMs`, so a publish part-way through a long job means its second
|
|
1254
|
+
* half describes a different instant than its first — every chunk whole and verified, but the answer covering
|
|
1255
|
+
* two moments, with nothing in the result saying so. That is fine for a dashboard and wrong for an export, a
|
|
1256
|
+
* reconciliation, or a send that has to match the count you reported. A pin is how you get one instant.
|
|
1257
|
+
*
|
|
1258
|
+
* ```ts
|
|
1259
|
+
* const snap = await store.segment('active-30d').pin();
|
|
1260
|
+
* const total = await snap.count(); // the number you report
|
|
1261
|
+
* for await (const id of snap.iterate()) { … } // …and the ids it counted, however long this takes
|
|
1262
|
+
* ```
|
|
1263
|
+
*
|
|
1264
|
+
* **Only this segment is pinned.** `snap.intersect([other])` reads `snap` at its pinned generation and
|
|
1265
|
+
* `other` at whatever is current — pin each segment if you want the whole query held. And a pinned handle
|
|
1266
|
+
* used as an *operand* is still read at its pin, never live.
|
|
1267
|
+
*
|
|
1268
|
+
* **It is a hold, not a lease.** Nothing here stops `gcOrphanGenerations` deleting the generation underneath
|
|
1269
|
+
* you: a pinned read deliberately does **not** heal forward, because silently serving a different generation
|
|
1270
|
+
* is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover your longest pinned
|
|
1271
|
+
* job — see [Sizing `keep`](../../docs/guide/getting-started.md#sizing-keep) — or take the pin on a segment
|
|
1272
|
+
* you are not collecting.
|
|
1273
|
+
*
|
|
1274
|
+
* A segment with no current generation pins nothing and reads empty, exactly as it would unpinned. A pin
|
|
1275
|
+
* taken before a crypto-shred stops reading when the shred lands: the destroyed row is re-checked every time
|
|
1276
|
+
* the pinned reader opens, so a pin cannot outlive the key it was using.
|
|
1277
|
+
*
|
|
1278
|
+
* Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
|
|
1279
|
+
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
1280
|
+
*/
|
|
1281
|
+
async pin() {
|
|
1282
|
+
return this.pinned(this.ref, this.expiresAt);
|
|
1283
|
+
}
|
|
1284
|
+
/** This handle's segment, as the cache/pin key — `ref` stays private; the encapsulation is worth the method. */
|
|
1285
|
+
key() {
|
|
1286
|
+
return segmentKey(this.ref);
|
|
1287
|
+
}
|
|
1288
|
+
/**
|
|
1289
|
+
* Has this handle's deadline passed? **The lazy half of expiry** — the whole check is one comparison against
|
|
1290
|
+
* the injected clock, so it costs nothing on a handle with no deadline and no I/O on one that has expired.
|
|
1291
|
+
*
|
|
1292
|
+
* Deliberately evaluated per call rather than cached: a long-lived handle created before its deadline must
|
|
1293
|
+
* start reading empty the moment the deadline passes, without the caller re-creating it.
|
|
1294
|
+
*/
|
|
1295
|
+
expired() {
|
|
1296
|
+
return this.expiresAt !== void 0 && this.clock.now() >= this.expiresAt;
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Refuse a materialisation that involves an **expired** handle. Called by the three `*Into` verbs only.
|
|
1300
|
+
*
|
|
1301
|
+
* On the read verbs an expired handle answers empty, which is the point of lazy expiry. On a *write* the same
|
|
1302
|
+
* rule would be destructive in a way nobody asks for: `a.intersectInto(dest, [b])` where `b`'s deadline has
|
|
1303
|
+
* quietly passed publishes an **empty generation over `dest`** — a wipe, reported as a successful write, with
|
|
1304
|
+
* the cause (a deadline on a handle somewhere) nowhere in the result. Reads degrade to empty; writes must not.
|
|
1305
|
+
*
|
|
1306
|
+
* So every handle in the call is checked, `dest` included: an expired `dest` does not change the bytes written,
|
|
1307
|
+
* but a caller who put a deadline on the thing they are writing into has said something contradictory and is
|
|
1308
|
+
* better told than guessed at. Open a handle without `expiresAt` to write, or drop the deadline.
|
|
1309
|
+
*
|
|
1310
|
+
* (The broader guard — refusing to publish an empty or implausible generation over a non-empty one, with an
|
|
1311
|
+
* `allowEmpty` override — now covers these verbs too: they route through the same guarded write path as
|
|
1312
|
+
* {@link CloudRoaring.load}. The two stay separate because they differ in kind. That one is a REPORTED
|
|
1313
|
+
* refusal a caller may legitimately override; an expired handle is a wiring mistake, so it THROWS, before
|
|
1314
|
+
* any object is written — and `allowEmpty: true` does not reach it.)
|
|
1315
|
+
*
|
|
1316
|
+
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
1317
|
+
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
1318
|
+
* awaiting.
|
|
1319
|
+
*/
|
|
1320
|
+
refuseIfExpired(op, dest, operands) {
|
|
1321
|
+
const stale = [];
|
|
1322
|
+
for (const seg of [this, dest, ...operands]) {
|
|
1323
|
+
if (seg.expired())
|
|
1324
|
+
stale.push(seg.ref.namespace ? `${seg.ref.namespace}/${seg.ref.segment}` : seg.ref.segment);
|
|
1325
|
+
}
|
|
1326
|
+
if (stale.length > 0) {
|
|
1327
|
+
throw new ValidationError(
|
|
1328
|
+
`${op}: refusing to publish a generation while these handles have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired handle reads as empty, so this would write an empty (or short) generation over "${dest.ref.segment}". Open the handles without \`expiresAt\` if you meant to materialise, or drop the deadline.`
|
|
1329
|
+
);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
/**
|
|
1333
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
1334
|
+
* entirely when no sink is wired, so the default path pays nothing.
|
|
1335
|
+
*/
|
|
1336
|
+
async timed(name, fn) {
|
|
1337
|
+
if (!this.metricsOn) return fn();
|
|
1338
|
+
const startedAt = this.clock.now();
|
|
1339
|
+
try {
|
|
1340
|
+
return await fn();
|
|
1341
|
+
} finally {
|
|
1342
|
+
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
/** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
|
|
1346
|
+
has(id) {
|
|
1347
|
+
if (this.expired()) return Promise.resolve(false);
|
|
1348
|
+
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
1349
|
+
}
|
|
1350
|
+
/** Cardinality — summed from the `.crbm` index with **zero payload reads** on a loaded segment. */
|
|
1351
|
+
count() {
|
|
1352
|
+
if (this.expired()) return Promise.resolve(0);
|
|
1353
|
+
return this.timed("count", () => this.engine.count(this.ref));
|
|
1354
|
+
}
|
|
1355
|
+
/** Every id, ascending, streamed one chunk at a time. */
|
|
1356
|
+
iterate() {
|
|
1357
|
+
if (this.expired()) return EMPTY_IDS;
|
|
1358
|
+
return this.engine.iterate(this.ref);
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1361
|
+
* Map the facade's `Segment` handles in `exclude` down to the plain refs `core` takes. A method rather than
|
|
1362
|
+
* a module function because `ref` is class-private — the encapsulation is worth more than the free function.
|
|
1363
|
+
*/
|
|
1364
|
+
refsIn(options) {
|
|
1365
|
+
if (options === void 0) return void 0;
|
|
1366
|
+
const { exclude, ...rest } = options;
|
|
1367
|
+
return exclude === void 0 ? rest : { ...rest, exclude: exclude.map((o) => o.ref) };
|
|
1368
|
+
}
|
|
1369
|
+
/**
|
|
1370
|
+
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
1371
|
+
* Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
1372
|
+
* and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
|
|
1373
|
+
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
1374
|
+
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
|
|
1375
|
+
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
1376
|
+
*/
|
|
1377
|
+
intersect(others, options) {
|
|
1378
|
+
if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
|
|
1379
|
+
const engine = this.combineEngine([this, ...others, ...options?.exclude ?? []]) ?? this.engine;
|
|
1380
|
+
return engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
|
|
1381
|
+
}
|
|
1382
|
+
/**
|
|
1383
|
+
* Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
|
|
1384
|
+
* are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
|
|
1385
|
+
* forward-only, so readers of `dest` see either the old generation or the new one, never a partial. Needs the
|
|
1386
|
+
* store built with a backend (throws {@link UnsupportedError} otherwise).
|
|
1387
|
+
*
|
|
1388
|
+
* **An empty result does NOT overwrite a non-empty destination.** A combine that comes out empty is far more
|
|
1389
|
+
* often a mistake upstream — a typo'd operand, an `exclude` that swallowed everything, an operand that has
|
|
1390
|
+
* not loaded yet — than an intent, and once it publishes it is indistinguishable from a correct run. So the
|
|
1391
|
+
* write is **refused and reported**: `published: false`, `reason: 'empty'`, and `dest` keeps what it had.
|
|
1392
|
+
* Pass `allowEmpty: true` when emptying the destination is the point.
|
|
1393
|
+
*
|
|
1394
|
+
* `guard` adds the same plausibility bounds `load()` takes — `minCardinality` and `minRetained` — judged
|
|
1395
|
+
* against what `dest` held before. A refusal is **reported, not thrown**, exactly as on `load()`; branch on
|
|
1396
|
+
* `published`. A lost race is the one outcome that still throws ({@link WriteConflictError}), because a
|
|
1397
|
+
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
1398
|
+
*
|
|
1399
|
+
* A call involving an **expired handle** is refused before any of this. See {@link refuseIfExpired}.
|
|
1400
|
+
*/
|
|
1401
|
+
async intersectInto(dest, others, options) {
|
|
1402
|
+
this.refuseIfExpired("intersectInto", dest, [...others, ...options?.exclude ?? []]);
|
|
1403
|
+
return this.timed(
|
|
1404
|
+
"intersectInto",
|
|
1405
|
+
() => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
/**
|
|
1409
|
+
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
1410
|
+
*
|
|
1411
|
+
* **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
|
|
1412
|
+
* here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
|
|
1413
|
+
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
1414
|
+
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
1415
|
+
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
1416
|
+
* segment once (`unionInto`, or a load) is the cheaper shape.
|
|
1417
|
+
*/
|
|
1418
|
+
union(others, options) {
|
|
1419
|
+
const live = others.filter((o) => !o.expired());
|
|
1420
|
+
if (this.expired()) {
|
|
1421
|
+
if (live.length === 0) return EMPTY_IDS;
|
|
1422
|
+
return live[0].union(live.slice(1), options);
|
|
1423
|
+
}
|
|
1424
|
+
if (live.length === 0 && others.length > 0) {
|
|
1425
|
+
const exclude = options?.exclude ?? [];
|
|
1426
|
+
return exclude.length > 0 ? this.andNot([...exclude], options) : this.iterate();
|
|
1427
|
+
}
|
|
1428
|
+
if (live.length !== others.length) return this.union(live, options);
|
|
1429
|
+
const engine = this.combineEngine([this, ...others, ...options?.exclude ?? []]) ?? this.engine;
|
|
1430
|
+
return engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
|
|
1431
|
+
}
|
|
1432
|
+
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
1433
|
+
async unionInto(dest, others, options) {
|
|
1434
|
+
this.refuseIfExpired("unionInto", dest, [...others, ...options?.exclude ?? []]);
|
|
1435
|
+
return this.timed(
|
|
1436
|
+
"unionInto",
|
|
1437
|
+
() => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
/**
|
|
1441
|
+
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
1442
|
+
*
|
|
1443
|
+
* Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
|
|
1444
|
+
* overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
|
|
1445
|
+
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
1446
|
+
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
1447
|
+
*
|
|
1448
|
+
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
1449
|
+
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
1450
|
+
*/
|
|
1451
|
+
andNot(excludes, options) {
|
|
1452
|
+
if (this.expired()) return EMPTY_IDS;
|
|
1453
|
+
const liveExcludes = excludes.filter((e) => !e.expired());
|
|
1454
|
+
if (liveExcludes.length === 0 && excludes.length > 0) return this.iterate();
|
|
1455
|
+
if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
|
|
1456
|
+
const engine = this.combineEngine([this, ...excludes]) ?? this.engine;
|
|
1457
|
+
return engine.andNot(
|
|
1458
|
+
this.ref,
|
|
1459
|
+
excludes.map((o) => o.ref),
|
|
1460
|
+
options
|
|
1461
|
+
);
|
|
1462
|
+
}
|
|
1463
|
+
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
1464
|
+
async andNotInto(dest, excludes, options) {
|
|
1465
|
+
this.refuseIfExpired("andNotInto", dest, excludes);
|
|
1466
|
+
return this.timed(
|
|
1467
|
+
"andNotInto",
|
|
1468
|
+
() => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
|
|
1469
|
+
);
|
|
1470
|
+
}
|
|
1471
|
+
/**
|
|
1472
|
+
* Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
|
|
1473
|
+
* reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
|
|
1474
|
+
* storage. See {@link CostReport} — it always includes a verdict (incl. the lose-zone).
|
|
1475
|
+
*/
|
|
1476
|
+
async costReport(options) {
|
|
1477
|
+
const canMeasure = this.engine.supportsStorageSize;
|
|
1478
|
+
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
1479
|
+
return groundedReport({
|
|
1480
|
+
storageBytes: size?.sizeBytes ?? 0,
|
|
1481
|
+
grounded: canMeasure,
|
|
1482
|
+
workload: options?.workload,
|
|
1483
|
+
pricing: options?.pricing,
|
|
1484
|
+
extraNotes: canMeasure ? void 0 : ["storage source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
1485
|
+
});
|
|
1486
|
+
}
|
|
1487
|
+
};
|
|
1488
|
+
var VERSION = "0.10.0";
|
|
1489
|
+
|
|
1490
|
+
// src/bin/export-segments.ts
|
|
19
1491
|
function parseConfig(env) {
|
|
20
1492
|
const root = env.CR_EXPORT_ROOT;
|
|
21
1493
|
if (root === void 0 || root === "") {
|
|
@@ -30,14 +1502,14 @@ function parseConfig(env) {
|
|
|
30
1502
|
throw new Error(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
31
1503
|
}
|
|
32
1504
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
33
|
-
return { root, out, format, namespace
|
|
1505
|
+
return { root, out, format, namespace };
|
|
34
1506
|
}
|
|
35
1507
|
function fsSink(out) {
|
|
36
1508
|
return {
|
|
37
1509
|
async open(ref, ext) {
|
|
38
|
-
const dir = join(out, ref.namespace
|
|
1510
|
+
const dir = join(out, (0, src_exports.namespacePathPart)(ref.namespace));
|
|
39
1511
|
await mkdir(dir, { recursive: true, mode: 448 });
|
|
40
|
-
const finalPath = join(dir, `${ref.segment}${ext}`);
|
|
1512
|
+
const finalPath = join(dir, `${(0, src_exports.encodeNameForPath)(ref.segment)}${ext}`);
|
|
41
1513
|
const tmpPath = `${finalPath}.${randomUUID()}.part`;
|
|
42
1514
|
const handle = await open(tmpPath, "wx", 384);
|
|
43
1515
|
return {
|
|
@@ -73,16 +1545,11 @@ var log = (obj) => {
|
|
|
73
1545
|
};
|
|
74
1546
|
async function main(env = process.env, now = () => Date.now()) {
|
|
75
1547
|
const config = parseConfig(env);
|
|
76
|
-
const
|
|
77
|
-
const store = new CloudRoaring({
|
|
78
|
-
cold: new src_exports.LocalFsColdDriver(join(config.root, "cold")),
|
|
79
|
-
warm: new src_exports.LocalFsWarmDriver(join(config.root, "warm")),
|
|
80
|
-
registry
|
|
81
|
-
});
|
|
1548
|
+
const storage = new src_exports.LocalFsStorage(config.root);
|
|
1549
|
+
const store = new CloudRoaring({ storage });
|
|
82
1550
|
const manifest = await store.exportSegments(fsSink(config.out), {
|
|
83
1551
|
format: config.format,
|
|
84
|
-
namespace: config.namespace
|
|
85
|
-
candidates: config.segments
|
|
1552
|
+
namespace: config.namespace
|
|
86
1553
|
});
|
|
87
1554
|
await mkdir(config.out, { recursive: true, mode: 448 });
|
|
88
1555
|
const manifestPath = join(config.out, "manifest.json");
|
|
@@ -110,7 +1577,16 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
110
1577
|
});
|
|
111
1578
|
return manifest;
|
|
112
1579
|
}
|
|
113
|
-
|
|
1580
|
+
function invokedAsCli() {
|
|
1581
|
+
const entry = process.argv[1];
|
|
1582
|
+
if (entry === void 0) return false;
|
|
1583
|
+
try {
|
|
1584
|
+
return import.meta.url === pathToFileURL(entry).href || import.meta.url === pathToFileURL(realpathSync(entry)).href;
|
|
1585
|
+
} catch {
|
|
1586
|
+
return false;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
if (invokedAsCli()) {
|
|
114
1590
|
main().then((manifest) => {
|
|
115
1591
|
if (manifest.failed.length > 0) {
|
|
116
1592
|
process.stderr.write(
|
|
@@ -125,7 +1601,9 @@ if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv
|
|
|
125
1601
|
process.exit(1);
|
|
126
1602
|
});
|
|
127
1603
|
}
|
|
128
|
-
|
|
129
|
-
|
|
1604
|
+
export {
|
|
1605
|
+
fsSink,
|
|
1606
|
+
main,
|
|
1607
|
+
parseConfig
|
|
1608
|
+
};
|
|
130
1609
|
//# sourceMappingURL=export-segments.js.map
|
|
131
|
-
//# sourceMappingURL=export-segments.js.map
|