@cloudbitmaps/roaring 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/PRIVACY.md +268 -76
- package/README.md +85 -95
- package/dist/bin/export-segments.d.ts +21 -0
- package/dist/bin/export-segments.js +1972 -30
- package/dist/bin/export-segments.js.map +7 -1
- package/dist/index.d.ts +969 -458
- package/dist/index.js +1461 -354
- package/dist/index.js.map +7 -1
- package/dist/open-charging-storage.d.ts +34 -0
- package/dist/option-keys.d.ts +19 -0
- package/dist/portable/decode.d.ts +53 -0
- package/dist/portable/layout.d.ts +32 -0
- package/dist/reserved-namespace.d.ts +3 -0
- package/dist/roaring-codec.d.ts +68 -0
- package/dist/system-clock.d.ts +39 -0
- package/dist/testing/conformance.d.ts +54 -0
- package/dist/testing/fuzz-codec.d.ts +23 -0
- package/package.json +11 -117
- package/dist/azure/index.cjs +0 -14
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -9
- package/dist/azure/index.d.ts +0 -9
- package/dist/azure/index.js +0 -3
- package/dist/azure/index.js.map +0 -1
- package/dist/bin/compact-segments.js +0 -201
- package/dist/bin/compact-segments.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -14
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -1
- package/dist/cassandra/index.d.ts +0 -1
- package/dist/cassandra/index.js +0 -3
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-D6LIJSGF.js +0 -906
- package/dist/chunk-D6LIJSGF.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -14
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -1
- package/dist/dynamodb/index.d.ts +0 -1
- package/dist/dynamodb/index.js +0 -3
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -14
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -1
- package/dist/gcs/index.d.ts +0 -1
- package/dist/gcs/index.js +0 -3
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -894
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -734
- package/dist/mongodb/index.cjs +0 -14
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -1
- package/dist/mongodb/index.d.ts +0 -1
- package/dist/mongodb/index.js +0 -3
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -14
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -1
- package/dist/mysql/index.d.ts +0 -1
- package/dist/mysql/index.js +0 -3
- package/dist/mysql/index.js.map +0 -1
- package/dist/postgres/index.cjs +0 -14
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -1
- package/dist/postgres/index.d.ts +0 -1
- package/dist/postgres/index.js +0 -3
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -14
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -1
- package/dist/redis/index.d.ts +0 -1
- package/dist/redis/index.js +0 -3
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -14
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -1
- package/dist/s3/index.d.ts +0 -1
- package/dist/s3/index.js +0 -3
- package/dist/s3/index.js.map +0 -1
|
@@ -1,21 +1,1957 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { src_exports, CloudRoaring } from '../chunk-D6LIJSGF.js';
|
|
3
|
-
import { randomUUID } from 'crypto';
|
|
4
|
-
import { mkdir, writeFile, rename, rm, open } from 'fs/promises';
|
|
5
|
-
import { join } from 'path';
|
|
6
|
-
import { pathToFileURL } from 'url';
|
|
7
|
-
import { validateSegmentRef } from '@cloudbitmaps/core';
|
|
8
2
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
3
|
+
// src/bin/export-segments.ts
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { realpathSync } from "node:fs";
|
|
6
|
+
import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { pathToFileURL } from "node:url";
|
|
9
|
+
import { encodeNameForPath, namespacePathPart } from "@cloudbitmaps/core/driver-kit";
|
|
10
|
+
|
|
11
|
+
// src/index.ts
|
|
12
|
+
import {
|
|
13
|
+
BoundedLru,
|
|
14
|
+
PinnedStorageChunkSource,
|
|
15
|
+
segmentKey,
|
|
16
|
+
CrbmStorageChunkSource,
|
|
17
|
+
DEFAULT_BUDGET,
|
|
18
|
+
DEFAULT_RETRY_POLICY,
|
|
19
|
+
withRetry,
|
|
20
|
+
isStorageBackend,
|
|
21
|
+
NOOP_METRICS,
|
|
22
|
+
RetryingStorageChunkSource,
|
|
23
|
+
SegmentEngine,
|
|
24
|
+
UnsupportedError,
|
|
25
|
+
ValidationError as ValidationError2,
|
|
26
|
+
WriteConflictError,
|
|
27
|
+
MIN_EXPIRES_AT_MS,
|
|
28
|
+
checkBudget,
|
|
29
|
+
collectWithinBudget,
|
|
30
|
+
excludingReservedRows,
|
|
31
|
+
dropSegment,
|
|
32
|
+
eraseIdFromSegment,
|
|
33
|
+
estimateCost,
|
|
34
|
+
groundedReport,
|
|
35
|
+
loadSegment,
|
|
36
|
+
mapWithConcurrency,
|
|
37
|
+
resolveBudget,
|
|
38
|
+
resolvePerOpBudget,
|
|
39
|
+
retireExpired,
|
|
40
|
+
runConsistencyCheck,
|
|
41
|
+
runExport,
|
|
42
|
+
setSegmentRetention,
|
|
43
|
+
clearSegmentRetention,
|
|
44
|
+
getSegmentRetention,
|
|
45
|
+
safeMetrics,
|
|
46
|
+
splitId
|
|
47
|
+
} from "@cloudbitmaps/core";
|
|
48
|
+
import { validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
49
|
+
|
|
50
|
+
// src/open-charging-storage.ts
|
|
51
|
+
var OpenChargingStorage = class {
|
|
52
|
+
/**
|
|
53
|
+
* @param inner the driver the erasure would otherwise use
|
|
54
|
+
* @param current the generation the segment's row named when it was listed, or `null` when it named none
|
|
55
|
+
* @param charge called once per charged open; it throws to refuse
|
|
56
|
+
*/
|
|
57
|
+
constructor(inner, current, charge) {
|
|
58
|
+
this.inner = inner;
|
|
59
|
+
this.charge = charge;
|
|
60
|
+
if (current !== null) this.free.add(current);
|
|
61
|
+
}
|
|
62
|
+
inner;
|
|
63
|
+
charge;
|
|
64
|
+
free = /* @__PURE__ */ new Set();
|
|
65
|
+
capabilities() {
|
|
66
|
+
return this.inner.capabilities();
|
|
67
|
+
}
|
|
68
|
+
async putImmutable(key, write) {
|
|
69
|
+
const written = await this.inner.putImmutable(key, write);
|
|
70
|
+
this.free.add(key.generation);
|
|
71
|
+
return written;
|
|
72
|
+
}
|
|
73
|
+
getRange(key, offset, length) {
|
|
74
|
+
return this.inner.getRange(key, offset, length);
|
|
75
|
+
}
|
|
76
|
+
/** One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. */
|
|
77
|
+
async getTail(key, maxBytes) {
|
|
78
|
+
if (!this.free.has(key.generation)) this.charge();
|
|
79
|
+
return this.inner.getTail(key, maxBytes);
|
|
80
|
+
}
|
|
81
|
+
delete(key) {
|
|
82
|
+
return this.inner.delete(key);
|
|
83
|
+
}
|
|
84
|
+
list(ref) {
|
|
85
|
+
return this.inner.list(ref);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// src/index.ts
|
|
90
|
+
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
91
|
+
import { listSegments, segmentExists } from "@cloudbitmaps/core";
|
|
92
|
+
|
|
93
|
+
// src/reserved-namespace.ts
|
|
94
|
+
import { ValidationError } from "@cloudbitmaps/core";
|
|
95
|
+
var RESERVED_NAMESPACE_PREFIX = "cbm.due.";
|
|
96
|
+
function refuseReservedNamespace(namespace) {
|
|
97
|
+
if (namespace !== void 0 && namespace.startsWith(RESERVED_NAMESPACE_PREFIX)) {
|
|
98
|
+
throw new ValidationError(
|
|
99
|
+
`namespace "${namespace}" is reserved: names starting with "${RESERVED_NAMESPACE_PREFIX}" hold the library's own bookkeeping rows, which every fleet-wide scan skips, so a segment there would be invisible to an erasure, a consistency check, an export and a retention sweep. Choose another namespace.`
|
|
100
|
+
);
|
|
101
|
+
}
|
|
18
102
|
}
|
|
103
|
+
|
|
104
|
+
// src/roaring-codec.ts
|
|
105
|
+
import roaring from "roaring";
|
|
106
|
+
import { IntegrityError as IntegrityError2 } from "@cloudbitmaps/core";
|
|
107
|
+
|
|
108
|
+
// src/portable/layout.ts
|
|
109
|
+
import { IntegrityError } from "@cloudbitmaps/core";
|
|
110
|
+
var SERIAL_COOKIE_NO_RUNCONTAINER = 12346;
|
|
111
|
+
var SERIAL_COOKIE = 12347;
|
|
112
|
+
var NO_OFFSET_THRESHOLD = 4;
|
|
113
|
+
var ARRAY_MAX_CARDINALITY = 4096;
|
|
114
|
+
var BITMAP_CONTAINER_BYTES = 8192;
|
|
115
|
+
var MAX_LOW = 65535;
|
|
116
|
+
var ARRAY = 0;
|
|
117
|
+
var BITMAP = 1;
|
|
118
|
+
var RUN = 2;
|
|
119
|
+
function overrun(bytes, offset, length, what) {
|
|
120
|
+
return new IntegrityError(
|
|
121
|
+
`portable roaring: ${what} needs bytes [${offset}, ${offset + length}) of a ${bytes.byteLength}-byte buffer`
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
function checkPortableLayout(bytes) {
|
|
125
|
+
walk(bytes, null);
|
|
126
|
+
}
|
|
127
|
+
function walk(bytes, out) {
|
|
128
|
+
const length = bytes.byteLength;
|
|
129
|
+
if (length === 0) return;
|
|
130
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, length);
|
|
131
|
+
if (length < 4) throw overrun(bytes, 0, 4, "the cookie");
|
|
132
|
+
const cookie = view.getUint32(0, true);
|
|
133
|
+
let count;
|
|
134
|
+
let runFlagsAt = -1;
|
|
135
|
+
let pos;
|
|
136
|
+
if (cookie === SERIAL_COOKIE_NO_RUNCONTAINER) {
|
|
137
|
+
if (length < 8) throw overrun(bytes, 4, 4, "the container count");
|
|
138
|
+
count = view.getUint32(4, true);
|
|
139
|
+
pos = 8;
|
|
140
|
+
} else if ((cookie & 65535) === SERIAL_COOKIE) {
|
|
141
|
+
count = (cookie >>> 16) + 1;
|
|
142
|
+
runFlagsAt = 4;
|
|
143
|
+
const flagBytes = count + 7 >>> 3;
|
|
144
|
+
if (runFlagsAt + flagBytes > length) {
|
|
145
|
+
throw overrun(bytes, runFlagsAt, flagBytes, "the run-container bitmap");
|
|
146
|
+
}
|
|
147
|
+
pos = runFlagsAt + flagBytes;
|
|
148
|
+
} else {
|
|
149
|
+
throw new IntegrityError(
|
|
150
|
+
`portable roaring: unrecognized cookie 0x${cookie.toString(16)}, not a portable-format bitmap`
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
const descriptiveAt = pos;
|
|
154
|
+
if (pos + count * 4 > length) throw overrun(bytes, pos, count * 4, "the descriptive header");
|
|
155
|
+
pos += count * 4;
|
|
156
|
+
let offsetsAt = -1;
|
|
157
|
+
if (runFlagsAt < 0 || count >= NO_OFFSET_THRESHOLD) {
|
|
158
|
+
if (pos + count * 4 > length) throw overrun(bytes, pos, count * 4, "the offset header");
|
|
159
|
+
offsetsAt = pos;
|
|
160
|
+
pos += count * 4;
|
|
161
|
+
}
|
|
162
|
+
let previousKey = -1;
|
|
163
|
+
for (let i = 0; i < count; i++) {
|
|
164
|
+
const key = view.getUint16(descriptiveAt + i * 4, true);
|
|
165
|
+
if (key <= previousKey) {
|
|
166
|
+
throw new IntegrityError(
|
|
167
|
+
`portable roaring: container keys are not strictly ascending (${previousKey} then ${key} at index ${i})`
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
previousKey = key;
|
|
171
|
+
const cardinality = view.getUint16(descriptiveAt + i * 4 + 2, true) + 1;
|
|
172
|
+
const isRun = runFlagsAt >= 0 && (bytes[runFlagsAt + (i >>> 3)] & 1 << (i & 7)) !== 0;
|
|
173
|
+
const kind = isRun ? RUN : cardinality > ARRAY_MAX_CARDINALITY ? BITMAP : ARRAY;
|
|
174
|
+
if (offsetsAt >= 0) {
|
|
175
|
+
const stated = view.getUint32(offsetsAt + i * 4, true);
|
|
176
|
+
if (stated !== pos) {
|
|
177
|
+
throw new IntegrityError(
|
|
178
|
+
`portable roaring: the offset header puts container ${i} at byte ${stated}, but it starts at ${pos}`
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
let size;
|
|
183
|
+
if (kind === ARRAY) {
|
|
184
|
+
size = cardinality * 2;
|
|
185
|
+
if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (array)`);
|
|
186
|
+
checkArray(view, pos, cardinality, i);
|
|
187
|
+
} else if (kind === BITMAP) {
|
|
188
|
+
size = BITMAP_CONTAINER_BYTES;
|
|
189
|
+
if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (bitmap)`);
|
|
190
|
+
const bits = popcount(view, pos);
|
|
191
|
+
if (bits !== cardinality) {
|
|
192
|
+
throw new IntegrityError(
|
|
193
|
+
`portable roaring: container ${i} (bitmap) holds ${bits} values, but its header says ${cardinality}`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
} else {
|
|
197
|
+
if (pos + 2 > length) throw overrun(bytes, pos, 2, `container ${i} (run count)`);
|
|
198
|
+
const runs = view.getUint16(pos, true);
|
|
199
|
+
size = 2 + runs * 4;
|
|
200
|
+
if (pos + size > length) throw overrun(bytes, pos, size, `container ${i} (run)`);
|
|
201
|
+
const covered = checkRuns(view, pos + 2, runs, i);
|
|
202
|
+
if (covered !== cardinality) {
|
|
203
|
+
throw new IntegrityError(
|
|
204
|
+
`portable roaring: container ${i} (run) covers ${covered} values, but its header says ${cardinality}`
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
out?.push({ key, cardinality, kind, offset: pos });
|
|
209
|
+
pos += size;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function checkArray(view, at, cardinality, index) {
|
|
213
|
+
let previous = -1;
|
|
214
|
+
let p = at;
|
|
215
|
+
const end = at + cardinality * 2;
|
|
216
|
+
for (; p + 4 <= end; p += 4) {
|
|
217
|
+
const pair = view.getUint32(p, true);
|
|
218
|
+
const first = pair & 65535;
|
|
219
|
+
const second = pair >>> 16;
|
|
220
|
+
if (first <= previous || second <= first) throw unordered(index, (p - at) / 2);
|
|
221
|
+
previous = second;
|
|
222
|
+
}
|
|
223
|
+
if (p < end && view.getUint16(p, true) <= previous) throw unordered(index, (p - at) / 2);
|
|
224
|
+
}
|
|
225
|
+
function unordered(index, element) {
|
|
226
|
+
return new IntegrityError(
|
|
227
|
+
`portable roaring: container ${index} (array) is not strictly ascending at or after element ${element}`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
function checkRuns(view, at, runs, index) {
|
|
231
|
+
let covered = 0;
|
|
232
|
+
let lowest = 0;
|
|
233
|
+
for (let r = 0; r < runs; r++) {
|
|
234
|
+
const run = view.getUint32(at + r * 4, true);
|
|
235
|
+
const start = run & 65535;
|
|
236
|
+
const last = start + (run >>> 16);
|
|
237
|
+
if (start < lowest) {
|
|
238
|
+
throw new IntegrityError(
|
|
239
|
+
`portable roaring: container ${index} (run) has run ${r} out of order, overlapping or touching the one before`
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
if (last > MAX_LOW) {
|
|
243
|
+
throw new IntegrityError(
|
|
244
|
+
`portable roaring: container ${index} (run) has run ${r} running past the end of the container`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
covered += last - start + 1;
|
|
248
|
+
lowest = last + 2;
|
|
249
|
+
}
|
|
250
|
+
return covered;
|
|
251
|
+
}
|
|
252
|
+
function popcount(view, at) {
|
|
253
|
+
let bits = 0;
|
|
254
|
+
for (let p = at, end = at + BITMAP_CONTAINER_BYTES; p < end; p += 4) {
|
|
255
|
+
let w = view.getUint32(p, true);
|
|
256
|
+
w -= w >>> 1 & 1431655765;
|
|
257
|
+
w = (w & 858993459) + (w >>> 2 & 858993459);
|
|
258
|
+
bits += Math.imul(w + (w >>> 4) & 252645135, 16843009) >>> 24;
|
|
259
|
+
}
|
|
260
|
+
return bits;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// src/roaring-codec.ts
|
|
264
|
+
var { RoaringBitmap32, SerializationFormat, DeserializationFormat } = roaring;
|
|
265
|
+
var SafeBitmap = class _SafeBitmap {
|
|
266
|
+
bitmap;
|
|
267
|
+
constructor(bitmap) {
|
|
268
|
+
this.bitmap = bitmap;
|
|
269
|
+
}
|
|
270
|
+
static empty() {
|
|
271
|
+
return new _SafeBitmap(new RoaringBitmap32());
|
|
272
|
+
}
|
|
273
|
+
static fromValues(values) {
|
|
274
|
+
return new _SafeBitmap(new RoaringBitmap32(values));
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Cap the size, check the structure, then deserialize with the **portable** format.
|
|
278
|
+
* Throws `IntegrityError` if the input exceeds `maxBytes`, is not a well-formed portable bitmap, or fails to
|
|
279
|
+
* decode — the native addon is never handed unbounded, malformed or unsafe-format input.
|
|
280
|
+
*
|
|
281
|
+
* The structural check is what makes the decoded bitmap's answers mean anything. The native deserializer
|
|
282
|
+
* bounds its reads and checks nothing else, so bytes of the right length and the wrong shape (containers or
|
|
283
|
+
* values out of order, overlapping runs, a cardinality that disagrees with the bits) decode into a bitmap
|
|
284
|
+
* whose `maximum()`, `has()` and `size` are wrong, and some of those shapes crash the process when used.
|
|
285
|
+
*/
|
|
286
|
+
static safeDeserialize(bytes, maxBytes) {
|
|
287
|
+
if (bytes.length > maxBytes) {
|
|
288
|
+
throw new IntegrityError2(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
289
|
+
}
|
|
290
|
+
checkPortableLayout(bytes);
|
|
291
|
+
try {
|
|
292
|
+
return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
|
|
293
|
+
} catch (err) {
|
|
294
|
+
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
serialize() {
|
|
298
|
+
return this.bitmap.serialize(SerializationFormat.portable);
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
302
|
+
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
303
|
+
*
|
|
304
|
+
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
305
|
+
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
306
|
+
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
307
|
+
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
308
|
+
* whichever encoding is smaller per container.
|
|
309
|
+
*
|
|
310
|
+
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
311
|
+
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
312
|
+
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
313
|
+
* columns; don't carry a figure between them without checking which denominator it had.
|
|
314
|
+
*
|
|
315
|
+
* Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
|
|
316
|
+
* path deliberately does not.
|
|
317
|
+
*/
|
|
318
|
+
optimize() {
|
|
319
|
+
this.bitmap.runOptimize();
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the last container, so the engine's
|
|
323
|
+
* per-chunk range assertion costs one call per chunk rather than a walk per id. The last container holds the
|
|
324
|
+
* largest value only because {@link SafeBitmap.safeDeserialize} refuses bytes whose containers or values are
|
|
325
|
+
* out of order.
|
|
326
|
+
*/
|
|
327
|
+
maximum() {
|
|
328
|
+
return this.bitmap.isEmpty ? void 0 : this.bitmap.maximum();
|
|
329
|
+
}
|
|
330
|
+
add(value) {
|
|
331
|
+
this.bitmap.add(value);
|
|
332
|
+
}
|
|
333
|
+
addMany(values) {
|
|
334
|
+
this.bitmap.addMany(values);
|
|
335
|
+
}
|
|
336
|
+
remove(value) {
|
|
337
|
+
this.bitmap.remove(value);
|
|
338
|
+
}
|
|
339
|
+
removeMany(values) {
|
|
340
|
+
this.bitmap.removeMany(values);
|
|
341
|
+
}
|
|
342
|
+
has(value) {
|
|
343
|
+
return this.bitmap.has(value);
|
|
344
|
+
}
|
|
345
|
+
get size() {
|
|
346
|
+
return this.bitmap.size;
|
|
347
|
+
}
|
|
348
|
+
get isEmpty() {
|
|
349
|
+
return this.bitmap.isEmpty;
|
|
350
|
+
}
|
|
351
|
+
clone() {
|
|
352
|
+
return new _SafeBitmap(this.bitmap.clone());
|
|
353
|
+
}
|
|
354
|
+
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
355
|
+
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
356
|
+
// `bitmap` (accessible on same-class instances) for the native op.
|
|
357
|
+
/** In-place union: `this = this ∪ other`. */
|
|
358
|
+
orInPlace(other) {
|
|
359
|
+
this.bitmap.orInPlace(other.bitmap);
|
|
360
|
+
}
|
|
361
|
+
/** In-place difference: `this = this \ other`. */
|
|
362
|
+
andNotInPlace(other) {
|
|
363
|
+
this.bitmap.andNotInPlace(other.bitmap);
|
|
364
|
+
}
|
|
365
|
+
/** In-place intersection: `this = this ∩ other`. */
|
|
366
|
+
andInPlace(other) {
|
|
367
|
+
this.bitmap.andInPlace(other.bitmap);
|
|
368
|
+
}
|
|
369
|
+
/** Ascending iterator over the set values. */
|
|
370
|
+
[Symbol.iterator]() {
|
|
371
|
+
return this.bitmap[Symbol.iterator]();
|
|
372
|
+
}
|
|
373
|
+
toArray() {
|
|
374
|
+
return this.bitmap.toArray();
|
|
375
|
+
}
|
|
376
|
+
};
|
|
377
|
+
var roaringCodec = {
|
|
378
|
+
empty: () => SafeBitmap.empty(),
|
|
379
|
+
fromValues: (values) => SafeBitmap.fromValues(values),
|
|
380
|
+
safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
|
|
381
|
+
};
|
|
382
|
+
|
|
383
|
+
// src/system-clock.ts
|
|
384
|
+
var SystemClock = class {
|
|
385
|
+
now() {
|
|
386
|
+
return Date.now();
|
|
387
|
+
}
|
|
388
|
+
sleep(ms) {
|
|
389
|
+
if (ms <= 0) return Promise.resolve();
|
|
390
|
+
return new Promise((resolve) => {
|
|
391
|
+
setTimeout(resolve, ms);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
|
|
396
|
+
*
|
|
397
|
+
* A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
|
|
398
|
+
* co-resident HTTP server keeps answering. Measured on the **synthetic yield-primitive benchmark** — a
|
|
399
|
+
* 61,035-iteration loop yielding every 1,024, isolating the primitive rather than timing a real load (the
|
|
400
|
+
* end-to-end figures are 519 ms → 24 ms; see `cooperative.ts`, which lists all three experiments):
|
|
401
|
+
*
|
|
402
|
+
* ```text
|
|
403
|
+
* no yield 568 ms wall 568.0 ms worst event-loop gap
|
|
404
|
+
* sleep(0) 555 ms wall 555.2 ms ← a microtask. Yields NOTHING.
|
|
405
|
+
* setImmediate 569 ms wall 13.8 ms ← free, and 41x less starvation
|
|
406
|
+
* setTimeout(1) 625 ms wall 15.0 ms ← +10% wall for no extra relief
|
|
407
|
+
* ```
|
|
408
|
+
*
|
|
409
|
+
* `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
|
|
410
|
+
* before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
|
|
411
|
+
* callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
|
|
412
|
+
* server wants — inbound requests are serviced before the loop resumes chewing.
|
|
413
|
+
*
|
|
414
|
+
* A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
|
|
415
|
+
* "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
|
|
416
|
+
*/
|
|
417
|
+
yieldNow() {
|
|
418
|
+
return new Promise((resolve) => {
|
|
419
|
+
setImmediate(resolve);
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
// src/option-keys.ts
|
|
425
|
+
var OPTION_KEYS = {
|
|
426
|
+
top: ["storage", "cache", "encryption", "retry", "metrics", "budget", "seams"],
|
|
427
|
+
cache: ["maxChunks", "ttlMs", "genTtlMs", "readerMax", "readerMaxBytes"],
|
|
428
|
+
encryption: ["keystore", "required"],
|
|
429
|
+
retry: ["maxAttempts", "baseDelayMs", "maxDelayMs", "backoffFactor", "jitter", "onRetry"],
|
|
430
|
+
budget: ["maxRequests"],
|
|
431
|
+
seams: ["clock", "rng"]
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// src/index.ts
|
|
435
|
+
import {
|
|
436
|
+
MemoryStorage,
|
|
437
|
+
LocalFsStorage,
|
|
438
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource2,
|
|
439
|
+
destroySegment,
|
|
440
|
+
eraseNamespace,
|
|
441
|
+
excludingReservedRows as excludingReservedRows2,
|
|
442
|
+
readRetentionPolicy,
|
|
443
|
+
MIN_EXPIRES_AT_MS as MIN_EXPIRES_AT_MS2,
|
|
444
|
+
InProcessKeystore,
|
|
445
|
+
NodeAead,
|
|
446
|
+
aadFor,
|
|
447
|
+
CloudRoaringError,
|
|
448
|
+
ValidationError as ValidationError3,
|
|
449
|
+
WriteConflictError as WriteConflictError2,
|
|
450
|
+
IntegrityError as IntegrityError3,
|
|
451
|
+
NotFoundError,
|
|
452
|
+
UnsupportedError as UnsupportedError2,
|
|
453
|
+
CapabilityError,
|
|
454
|
+
TransientError,
|
|
455
|
+
KeyUnavailableError,
|
|
456
|
+
BudgetExceededError,
|
|
457
|
+
isCloudRoaringError,
|
|
458
|
+
isWriteConflictError,
|
|
459
|
+
isTransientError,
|
|
460
|
+
isNotFoundError,
|
|
461
|
+
isIntegrityError,
|
|
462
|
+
isValidationError,
|
|
463
|
+
CrbmReader,
|
|
464
|
+
BufferReader,
|
|
465
|
+
CountingMetricsSink,
|
|
466
|
+
RecordingAuditSink,
|
|
467
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2,
|
|
468
|
+
AWS_US_EAST_1_ONDEMAND,
|
|
469
|
+
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
470
|
+
ONE_REDIS_HA_CLUSTER
|
|
471
|
+
} from "@cloudbitmaps/core";
|
|
472
|
+
var SystemRng = class {
|
|
473
|
+
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
474
|
+
next() {
|
|
475
|
+
return Math.random();
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
479
|
+
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
480
|
+
function validateConcurrency(concurrency) {
|
|
481
|
+
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
482
|
+
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
function requireScope(options, op) {
|
|
486
|
+
refuseReservedNamespace(options.namespace);
|
|
487
|
+
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
488
|
+
throw new ValidationError2(
|
|
489
|
+
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
function resolveStorageSource(options, clock) {
|
|
494
|
+
const storage = options.storage;
|
|
495
|
+
if (storage === null || typeof storage !== "object") {
|
|
496
|
+
throw new ValidationError2(
|
|
497
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
498
|
+
);
|
|
499
|
+
}
|
|
500
|
+
const asBackend = storage;
|
|
501
|
+
const isObject = (v) => v !== null && typeof v === "object";
|
|
502
|
+
const hasGetChunk = typeof storage.getChunk === "function";
|
|
503
|
+
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
504
|
+
const isBackend = isStorageBackend(storage);
|
|
505
|
+
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
506
|
+
throw new ValidationError2(
|
|
507
|
+
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
508
|
+
);
|
|
509
|
+
}
|
|
510
|
+
const hasStorageHalf = isObject(asBackend.storage);
|
|
511
|
+
const hasRegistryHalf = isObject(asBackend.registry);
|
|
512
|
+
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
513
|
+
throw new ValidationError2(
|
|
514
|
+
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
518
|
+
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
519
|
+
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
520
|
+
if (!storageOk || !registryOk) {
|
|
521
|
+
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
522
|
+
throw new ValidationError2(
|
|
523
|
+
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
throw new ValidationError2(
|
|
527
|
+
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
528
|
+
);
|
|
529
|
+
}
|
|
530
|
+
if (isBackend) {
|
|
531
|
+
const backend = storage;
|
|
532
|
+
return {
|
|
533
|
+
source: new CrbmStorageChunkSource(backend.storage, {
|
|
534
|
+
registry: backend.registry,
|
|
535
|
+
keystore: options.encryption?.keystore,
|
|
536
|
+
requireEncryption: options.encryption?.required,
|
|
537
|
+
clock,
|
|
538
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
539
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
540
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
541
|
+
}),
|
|
542
|
+
driver: backend.storage,
|
|
543
|
+
registry: backend.registry
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
547
|
+
throw new ValidationError2(
|
|
548
|
+
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
552
|
+
throw new ValidationError2(
|
|
553
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
if (hasGetChunk) {
|
|
557
|
+
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
558
|
+
throw new ValidationError2(
|
|
559
|
+
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
return { source: storage, driver: void 0, registry: void 0 };
|
|
563
|
+
}
|
|
564
|
+
const driver = storage;
|
|
565
|
+
return {
|
|
566
|
+
source: new CrbmStorageChunkSource(driver, {
|
|
567
|
+
keystore: options.encryption?.keystore,
|
|
568
|
+
requireEncryption: options.encryption?.required,
|
|
569
|
+
clock,
|
|
570
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
571
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
572
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
573
|
+
}),
|
|
574
|
+
driver,
|
|
575
|
+
registry: void 0
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
var NOT_A_GROUP = {
|
|
579
|
+
Null: "null",
|
|
580
|
+
Array: "an array",
|
|
581
|
+
Map: "a Map",
|
|
582
|
+
Set: "a Set",
|
|
583
|
+
Boolean: "a boxed boolean",
|
|
584
|
+
Number: "a boxed number",
|
|
585
|
+
String: "a boxed string"
|
|
586
|
+
};
|
|
587
|
+
var CloudRoaring = class _CloudRoaring {
|
|
588
|
+
engine;
|
|
589
|
+
cache;
|
|
590
|
+
/** The store's retry options, for a pinned engine to read through as the store's own does; undefined when off. */
|
|
591
|
+
retryOptions;
|
|
592
|
+
crbmSource;
|
|
593
|
+
clock;
|
|
594
|
+
metrics;
|
|
595
|
+
// The store's own drivers, kept so the lifecycle helpers and the `*Into` verbs reuse them instead of making
|
|
596
|
+
// you re-pass deps. `storageDriver` is set only when `storage` was a raw IStorageDriver (a pre-built StorageChunkSource has
|
|
597
|
+
// no underlying driver to write through).
|
|
598
|
+
storageDriver;
|
|
599
|
+
registry;
|
|
600
|
+
keystore;
|
|
601
|
+
requireEncryption;
|
|
602
|
+
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
603
|
+
budget;
|
|
604
|
+
/**
|
|
605
|
+
* Refuse any option key the store does not take, at the top level or inside a group, naming each one.
|
|
606
|
+
*
|
|
607
|
+
* An option the store ignored would do nothing and look as if it had — a typo'd `cache.maxChunk`, a key from
|
|
608
|
+
* another library's config spread into this one — so every key is checked against {@link OPTION_KEYS}.
|
|
609
|
+
*/
|
|
610
|
+
static rejectUnknownOptions(options) {
|
|
611
|
+
if (options === null || options === void 0 || typeof options !== "object") {
|
|
612
|
+
throw new ValidationError2(
|
|
613
|
+
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
614
|
+
);
|
|
615
|
+
}
|
|
616
|
+
const bag = options;
|
|
617
|
+
const unknown = [];
|
|
618
|
+
for (const key of Object.keys(bag)) {
|
|
619
|
+
if (!OPTION_KEYS.top.includes(key)) unknown.push({ group: null, key });
|
|
620
|
+
}
|
|
621
|
+
const groups = Object.keys(OPTION_KEYS).filter((g) => g !== "top");
|
|
622
|
+
for (const group of groups) {
|
|
623
|
+
const value = bag[group];
|
|
624
|
+
if (value === void 0) continue;
|
|
625
|
+
const offOk = group === "retry" || group === "budget";
|
|
626
|
+
if (value === false && offOk) continue;
|
|
627
|
+
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
628
|
+
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
629
|
+
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
630
|
+
throw new ValidationError2(
|
|
631
|
+
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
const known = OPTION_KEYS[group];
|
|
635
|
+
for (const key of Object.keys(value)) if (!known.includes(key)) unknown.push({ group, key });
|
|
636
|
+
}
|
|
637
|
+
if (unknown.length === 0) return;
|
|
638
|
+
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
639
|
+
const named = [...new Set(unknown.map((u) => u.group))];
|
|
640
|
+
throw new ValidationError2(
|
|
641
|
+
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
642
|
+
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
643
|
+
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
constructor(options) {
|
|
647
|
+
_CloudRoaring.rejectUnknownOptions(options);
|
|
648
|
+
const clock = options.seams?.clock ?? new SystemClock();
|
|
649
|
+
const rng = options.seams?.rng ?? new SystemRng();
|
|
650
|
+
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS);
|
|
651
|
+
const cache = new BoundedLru({
|
|
652
|
+
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
653
|
+
ttlMs: options.cache?.ttlMs,
|
|
654
|
+
clock
|
|
655
|
+
});
|
|
656
|
+
const resolved = resolveStorageSource(options, clock);
|
|
657
|
+
let storage = resolved.source;
|
|
658
|
+
if (options.retry !== false) {
|
|
659
|
+
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
660
|
+
const policy = {
|
|
661
|
+
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
662
|
+
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
663
|
+
maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
664
|
+
backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
665
|
+
jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
666
|
+
};
|
|
667
|
+
const retryOpts = {
|
|
668
|
+
clock,
|
|
669
|
+
rng,
|
|
670
|
+
policy,
|
|
671
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
672
|
+
onRetry: (info) => {
|
|
673
|
+
metrics.onEvent({
|
|
674
|
+
kind: "retry",
|
|
675
|
+
reason: "transient",
|
|
676
|
+
attempt: info.attempt,
|
|
677
|
+
delayMs: info.delayMs
|
|
678
|
+
});
|
|
679
|
+
userOnRetry?.(info);
|
|
680
|
+
}
|
|
681
|
+
};
|
|
682
|
+
storage = new RetryingStorageChunkSource(storage, retryOpts);
|
|
683
|
+
this.retryOptions = retryOpts;
|
|
684
|
+
}
|
|
685
|
+
this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
|
|
686
|
+
const deps = {
|
|
687
|
+
storage,
|
|
688
|
+
cache,
|
|
689
|
+
codec: roaringCodec,
|
|
690
|
+
// the facade injects the flagship codec; core stays codec-agnostic
|
|
691
|
+
clock,
|
|
692
|
+
metrics,
|
|
693
|
+
budget: this.budget
|
|
694
|
+
};
|
|
695
|
+
this.engine = new SegmentEngine(deps);
|
|
696
|
+
this.cache = cache;
|
|
697
|
+
this.crbmSource = resolved.source instanceof CrbmStorageChunkSource ? resolved.source : void 0;
|
|
698
|
+
this.clock = clock;
|
|
699
|
+
this.metrics = metrics;
|
|
700
|
+
this.storageDriver = resolved.driver;
|
|
701
|
+
this.registry = resolved.registry;
|
|
702
|
+
this.keystore = options.encryption?.keystore;
|
|
703
|
+
this.requireEncryption = options.encryption?.required ?? false;
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* The write-side deps, from the store's own drivers, for the lifecycle helpers and the `*Into` verbs. Requires
|
|
707
|
+
* the store to have been constructed with a **backend**, which supplies both halves: an `IStorageDriver` to
|
|
708
|
+
* write generations through (a pre-built `StorageChunkSource` has none) and the registry holding the pointer
|
|
709
|
+
* every write publishes.
|
|
710
|
+
*/
|
|
711
|
+
lifecycleDeps(op) {
|
|
712
|
+
if (this.storageDriver === void 0) {
|
|
713
|
+
throw new UnsupportedError(
|
|
714
|
+
`${op} needs the store built with a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A pre-built StorageChunkSource is read-only: it has no IStorageDriver underneath to write generations through. Build the store on one of those backend classes to write.`
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
if (this.registry === void 0) {
|
|
718
|
+
throw new UnsupportedError(
|
|
719
|
+
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
720
|
+
);
|
|
721
|
+
}
|
|
722
|
+
return {
|
|
723
|
+
storage: this.storageDriver,
|
|
724
|
+
registry: this.registry,
|
|
725
|
+
clock: this.clock,
|
|
726
|
+
codec: roaringCodec,
|
|
727
|
+
// facade injects the flagship codec
|
|
728
|
+
keystore: this.keystore,
|
|
729
|
+
requireEncryption: this.requireEncryption
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
/** Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). */
|
|
733
|
+
segment(name, options) {
|
|
734
|
+
const ref = { segment: name, namespace: options?.namespace };
|
|
735
|
+
validateSegmentRef(ref);
|
|
736
|
+
refuseReservedNamespace(ref.namespace);
|
|
737
|
+
const expiresAt = options?.expiresAt;
|
|
738
|
+
if (expiresAt !== void 0) {
|
|
739
|
+
if (!Number.isFinite(expiresAt) || !Number.isInteger(expiresAt) || expiresAt < MIN_EXPIRES_AT_MS) {
|
|
740
|
+
throw new ValidationError2(
|
|
741
|
+
`segment: expiresAt must be an integer epoch-MILLISECONDS >= ${MIN_EXPIRES_AT_MS}; got ${expiresAt} (a value in seconds lands in 1970 and would make the segment read as permanently empty)`
|
|
742
|
+
);
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
return makeSegment({
|
|
746
|
+
engine: this.engine,
|
|
747
|
+
ref,
|
|
748
|
+
clock: this.clock,
|
|
749
|
+
metrics: this.metrics,
|
|
750
|
+
materialize: (dest, ids, op, options2) => this.materialize(dest, ids, op, options2),
|
|
751
|
+
pinned: (r, e) => this.pinSegment(r, e),
|
|
752
|
+
combineEngine: (handles) => this.engineForCombine(handles),
|
|
753
|
+
expiresAt
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
|
|
758
|
+
* materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
|
|
759
|
+
* publish on the row's token; a guarded load (the default, since the empty refusal reads the current generation)
|
|
760
|
+
* also fences on the pointer it judged (`expectFrom`), and one that found no row fences on that absence instead.
|
|
761
|
+
* Only an unguarded load (`allowEmpty: true` and no `guard.minRetained`) onto a segment with no row publishes bare
|
|
762
|
+
* forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
763
|
+
* within `cache.genTtlMs`).
|
|
764
|
+
*
|
|
765
|
+
* This routes through `loadSegment` rather than writing the generation itself, and that is the whole point
|
|
766
|
+
* of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
|
|
767
|
+
* everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
|
|
768
|
+
* guard runs while the old generation is still authoritative, the publish is fenced as above, and a refused
|
|
769
|
+
* object is reclaimed only when a re-read finds the row unchanged (the same token) or gone (hard invariant 1:
|
|
770
|
+
* deleting it after a purge-and-recreate would put a live row over a missing generation).
|
|
771
|
+
*
|
|
772
|
+
* That last check narrows the window rather than closing it: the row read and the delete are two round
|
|
773
|
+
* trips, and `IStorageDriver` has no conditional delete to make them one. the collection a load runs carries
|
|
774
|
+
* the same residual. The failure it leaves is an orphan object, which costs storage until
|
|
775
|
+
* something collects it — deliberately the cheaper side of the trade.
|
|
776
|
+
*
|
|
777
|
+
* Written and published in one step, with no guard, an empty combine — a typo'd operand, an `exclude` that
|
|
778
|
+
* swallowed everything, an operand that had not loaded yet — would silently replace `dest` with an empty
|
|
779
|
+
* generation: the same failure `load()`'s guard exists to prevent, on the same data, reachable without passing
|
|
780
|
+
* any option at all.
|
|
781
|
+
*
|
|
782
|
+
* **A lost race throws.** `loadSegment` reports one as `reason: 'superseded'`; the `*Into` verbs throw
|
|
783
|
+
* {@link WriteConflictError} for it, so a caller can `catch (WriteConflictError)`. That one refusal is
|
|
784
|
+
* translated into the throw, and `MaterializeResult.reason` never carries it.
|
|
785
|
+
*
|
|
786
|
+
* **A `WriteConflictError` does not by itself mean nothing was published**, and that is worth knowing
|
|
787
|
+
* before you write the retry. `'superseded'` covers five different causes — the write-once PUT collided,
|
|
788
|
+
* the pointer moved, a row appeared where the load found none, the row's token changed, the row was purged —
|
|
789
|
+
* and only the first three are the "somebody beat us" the name suggests. A token can also change on a write
|
|
790
|
+
* that is not a supersession at all, such as a `setRetention` on the destination. On top of that, the collection pass that runs AFTER a
|
|
791
|
+
* successful publish can raise the same error. So: treat it as "re-read the destination and decide",
|
|
792
|
+
* never as "the write did not happen".
|
|
793
|
+
*
|
|
794
|
+
* **And by default it collects nothing**, unlike `load()`. See the `keep` default below.
|
|
795
|
+
*/
|
|
796
|
+
async materialize(dest, ids, op, options) {
|
|
797
|
+
const deps = this.lifecycleDeps(op);
|
|
798
|
+
let result;
|
|
799
|
+
try {
|
|
800
|
+
result = await loadSegment(dest, ids, deps, {
|
|
801
|
+
...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
|
|
802
|
+
...options?.guard === void 0 ? {} : { guard: options.guard },
|
|
803
|
+
// COLLECT NOTHING by default, which `loadSegment` does not — it keeps a grace window of 1 and deletes
|
|
804
|
+
// the rest. By default a materialisation collects nothing: the guide states "**It deletes nothing**,
|
|
805
|
+
// unlike `load()`. The destination's previous generations stay in the bucket until you collect them", and
|
|
806
|
+
// the ownership table
|
|
807
|
+
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
808
|
+
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
809
|
+
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
810
|
+
keep: options?.keep ?? KEEP_EVERY_GENERATION,
|
|
811
|
+
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
812
|
+
});
|
|
813
|
+
} finally {
|
|
814
|
+
this.engine.invalidate(dest);
|
|
815
|
+
}
|
|
816
|
+
const reason = result.reason;
|
|
817
|
+
if (reason === "superseded") {
|
|
818
|
+
const wrote = result.size > 0 ? `generation ${result.generation} was written and did not become current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
819
|
+
throw new WriteConflictError(
|
|
820
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so it never became current: ${wrote}. The pointer may have moved, the row may have been rewritten (a retention policy does this) or purged. Re-read the destination and re-run.`
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
return {
|
|
824
|
+
generation: result.generation,
|
|
825
|
+
published: result.published,
|
|
826
|
+
...reason === void 0 ? {} : { reason },
|
|
827
|
+
cardinality: result.cardinality,
|
|
828
|
+
cardinalityBefore: result.cardinalityBefore,
|
|
829
|
+
chunkCount: result.chunkCount,
|
|
830
|
+
size: result.size,
|
|
831
|
+
collected: result.collected
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
/**
|
|
835
|
+
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?**
|
|
836
|
+
*
|
|
837
|
+
* Enumerates the **registered** segments (via the store's own `registry`) and does a `has(id)` on each — no
|
|
838
|
+
* drivers to re-pass. Complete only over registered segments (every loaded segment has a row, so build the
|
|
839
|
+
* store on the backend the loads used). There is deliberately **no `id → segments` reverse index** — that would
|
|
840
|
+
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
841
|
+
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
842
|
+
*
|
|
843
|
+
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
844
|
+
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
845
|
+
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
846
|
+
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
847
|
+
* generation 0 while this store held its old generation 0 is told apart too.
|
|
848
|
+
*/
|
|
849
|
+
async subjectReport(id, options = {}) {
|
|
850
|
+
const registry = this.requireRegistry("subjectReport");
|
|
851
|
+
requireScope(options, "subjectReport");
|
|
852
|
+
validateConcurrency(options.concurrency);
|
|
853
|
+
const budget = resolvePerOpBudget(options.budget, this.budget);
|
|
854
|
+
splitId(id);
|
|
855
|
+
const recs = await collectWithinBudget(
|
|
856
|
+
// A subject cannot be in a due-index row, and charging this request's budget for them would refuse a
|
|
857
|
+
// GDPR Art. 15 report for a reason unrelated to the subject.
|
|
858
|
+
excludingReservedRows(registry.list(options.namespace)),
|
|
859
|
+
budget,
|
|
860
|
+
"subjectReport"
|
|
861
|
+
);
|
|
862
|
+
const membership = await mapWithConcurrency(
|
|
863
|
+
recs,
|
|
864
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
865
|
+
async (rec) => {
|
|
866
|
+
if (rec.status === "destroyed") return null;
|
|
867
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
868
|
+
if (this.crbmSource !== void 0) {
|
|
869
|
+
const held = await this.crbmSource.currentVersion(ref);
|
|
870
|
+
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${String(rec.token)}`;
|
|
871
|
+
if (held !== listed) this.engine.invalidate(ref);
|
|
872
|
+
}
|
|
873
|
+
return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
874
|
+
}
|
|
875
|
+
);
|
|
876
|
+
const segments = membership.filter((m) => m !== null);
|
|
877
|
+
return { id, segments, scannedSegments: recs.length };
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
881
|
+
* from the bucket on return.**
|
|
882
|
+
*
|
|
883
|
+
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
884
|
+
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
885
|
+
* generation it was derived from**, and collects the generation that held the bit. A segment whose current
|
|
886
|
+
* generation does not hold the id is searched anyway, every generation in its bucket: a retained superseded one
|
|
887
|
+
* can still hold it, and so can one above the pointer after a {@link CloudRoaring.rollback}, which the rollback
|
|
888
|
+
* could make current again. Each holder is deleted — below the pointer by a `keep: 0` collection, above it one
|
|
889
|
+
* by one, re-proved against the row first — and the generations above the pointer that never held the id stay
|
|
890
|
+
* as rollback targets. An entry says `erased: true` only once the call has listed the bucket and read what is
|
|
891
|
+
* left: **no generation of the segment holds the id**. The returned per-segment record is your **erasure ledger** —
|
|
892
|
+
* persist it / route it to your audit sink as the proof of deletion (a `segment.rewrite` audit event is also
|
|
893
|
+
* emitted per rewrite when you pass `audit`).
|
|
894
|
+
*
|
|
895
|
+
* Uses the backend's **own** two halves, so the membership check and the rewrite provably run
|
|
896
|
+
* over the same generation. Requires the store built with a **backend** (throws
|
|
897
|
+
* {@link UnsupportedError} otherwise; a pre-built `StorageChunkSource` store has no `IStorageDriver` to write
|
|
898
|
+
* through; build the store on a backend instead).
|
|
899
|
+
*
|
|
900
|
+
* **One contract remains** (an integrator obligation the library cannot check): **do not load the segment
|
|
901
|
+
* while erasing from it.** A load that lands after the rewrite carries whatever its source held, and the
|
|
902
|
+
* library cannot know that source was meant to exclude the id. Quiesce loads of the affected segments for the
|
|
903
|
+
* duration, or fix the source first and load after. A writer that lands *during* the rewrite is caught: the
|
|
904
|
+
* rewrite's publish is refused **by the fence** — the publish lands its compare-and-swap only
|
|
905
|
+
* while the pointer is still on the generation the rewrite streamed — and the entry says `note: 'superseded'`,
|
|
906
|
+
* so re-run. Forward-only alone would NOT refuse it: the rewrite's number goes above everything in the bucket,
|
|
907
|
+
* so the rewrite would out-rank the newer generation and then collect it.
|
|
908
|
+
*
|
|
909
|
+
* A racing **erasure** is caught before that, and reported the same way. It collects with `keep: 0`, taking
|
|
910
|
+
* every generation below its new pointer — the one this rewrite is streaming, and the object this rewrite
|
|
911
|
+
* just wrote — so the loser can find its own inputs deleted mid-flight. That surfaces as an outcome rather than
|
|
912
|
+
* an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
|
|
913
|
+
* that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger, as a fresh
|
|
914
|
+
* call would leave it out.
|
|
915
|
+
*
|
|
916
|
+
* **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
|
|
917
|
+
* another writer (a load, another erasure, or a rollback) moved the pointer mid-call, so **this call** did not
|
|
918
|
+
* erase the id. Re-run: it erases the id if it is still there, and lists nothing for the segment if a racing
|
|
919
|
+
* erasure of the same id already removed it. Do not read `'superseded'` as "the id is still present" —
|
|
920
|
+
* read it as "not done by this call, and the re-run settles it".
|
|
921
|
+
* `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
|
|
922
|
+
* on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
|
|
923
|
+
* it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
|
|
924
|
+
* absent from every read and what remains is an object in the bucket that still contains the bit. The same
|
|
925
|
+
* note, after a publish, can mean a rollback moved the pointer back onto a generation that still holds the id
|
|
926
|
+
* while the collection ran — then the id is served again, and the entry says so rather than `erased: true`. A
|
|
927
|
+
* re-run searches every generation in the bucket, so it rewrites or collects what is left and usually reports
|
|
928
|
+
* `erased: true`; {@link SubjectErasureEntry.note} says what else a re-run can report. Re-running is otherwise safe
|
|
929
|
+
* and idempotent: a segment the id is no longer in is not listed. Admin-only path;
|
|
930
|
+
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
931
|
+
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
932
|
+
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
933
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
934
|
+
* so a fleet with long histories can exhaust it where the segment count alone would not. A segment that does is
|
|
935
|
+
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
936
|
+
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
937
|
+
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
938
|
+
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
939
|
+
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
940
|
+
*/
|
|
941
|
+
async eraseSubject(id, options = {}) {
|
|
942
|
+
const deps = this.lifecycleDeps("eraseSubject");
|
|
943
|
+
requireScope(options, "eraseSubject");
|
|
944
|
+
validateConcurrency(options.concurrency);
|
|
945
|
+
const budget = resolvePerOpBudget(options.budget, this.budget);
|
|
946
|
+
splitId(id);
|
|
947
|
+
const recs = await collectWithinBudget(
|
|
948
|
+
excludingReservedRows(deps.registry.list(options.namespace)),
|
|
949
|
+
budget,
|
|
950
|
+
"eraseSubject"
|
|
951
|
+
);
|
|
952
|
+
let opens = 0;
|
|
953
|
+
const charge = () => {
|
|
954
|
+
opens += 1;
|
|
955
|
+
checkBudget(budget, recs.length + opens, "eraseSubject");
|
|
956
|
+
};
|
|
957
|
+
const entries = await mapWithConcurrency(
|
|
958
|
+
recs,
|
|
959
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
960
|
+
async (rec) => {
|
|
961
|
+
if (rec.status === "destroyed") return null;
|
|
962
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
963
|
+
try {
|
|
964
|
+
const result = await eraseIdFromSegment(
|
|
965
|
+
ref,
|
|
966
|
+
id,
|
|
967
|
+
budget === null ? deps : { ...deps, storage: new OpenChargingStorage(deps.storage, rec.currentGen, charge) },
|
|
968
|
+
{ audit: options.audit }
|
|
969
|
+
);
|
|
970
|
+
if (result.reason === "not-member" || result.reason === "absent") return null;
|
|
971
|
+
if (result.reason === "no-generation") return null;
|
|
972
|
+
if (result.reason === "destroyed") return null;
|
|
973
|
+
return {
|
|
974
|
+
segment: rec.segment,
|
|
975
|
+
namespace: rec.namespace,
|
|
976
|
+
erased: result.erased,
|
|
977
|
+
fromGeneration: result.fromGeneration,
|
|
978
|
+
generation: result.generation,
|
|
979
|
+
note: result.erased ? void 0 : result.reason
|
|
980
|
+
};
|
|
981
|
+
} catch (err) {
|
|
982
|
+
return {
|
|
983
|
+
segment: rec.segment,
|
|
984
|
+
namespace: rec.namespace,
|
|
985
|
+
erased: false,
|
|
986
|
+
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
987
|
+
};
|
|
988
|
+
} finally {
|
|
989
|
+
try {
|
|
990
|
+
this.engine.invalidate(ref);
|
|
991
|
+
} catch {
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
);
|
|
996
|
+
const erasedFrom = entries.filter((e) => e !== null);
|
|
997
|
+
return { id, erasedFrom, scannedSegments: recs.length };
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* Every generation still in the bucket for this segment, ascending, with the current one marked.
|
|
1001
|
+
*
|
|
1002
|
+
* What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
|
|
1003
|
+
* the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
|
|
1004
|
+
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1005
|
+
*
|
|
1006
|
+
* Needs a backend.
|
|
1007
|
+
*/
|
|
1008
|
+
async generations(ref) {
|
|
1009
|
+
validateSegmentRef(ref);
|
|
1010
|
+
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Whether a read of this segment would find anything — the "do I already have this?" question, as one
|
|
1014
|
+
* registry point read.
|
|
1015
|
+
*
|
|
1016
|
+
* There is no `create` in this library: {@link CloudRoaring.segment} is a validated address and does no I/O,
|
|
1017
|
+
* so naming a segment can never collide with an existing one. A segment starts existing when something is
|
|
1018
|
+
* first loaded into it, and this is how you ask whether that has happened.
|
|
1019
|
+
*
|
|
1020
|
+
* **Not the same as `count() > 0`.** A segment loaded with no ids exists and counts zero. Distinguishing
|
|
1021
|
+
* "never loaded" from "loaded, and genuinely empty" is the thing `count()` cannot do and the reason this
|
|
1022
|
+
* exists. It is `false` for a segment whose row was minted ahead of its first load (by `setRetention`) and
|
|
1023
|
+
* for a `destroyed` tombstone, because a read answers empty in both cases.
|
|
1024
|
+
*
|
|
1025
|
+
* Two states answer `true` where a read still gives you nothing: a torn restore (a live pointer whose object
|
|
1026
|
+
* was deleted) makes reads *throw* rather than answer empty — `checkConsistency` is the call for that — and
|
|
1027
|
+
* a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
|
|
1028
|
+
*
|
|
1029
|
+
* Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
|
|
1030
|
+
* `load`'s `guard`, or `expectFrom`/`expectToken` on a publish. Needs a storage backend.
|
|
1031
|
+
*
|
|
1032
|
+
* ```ts
|
|
1033
|
+
* if (!(await store.exists({ segment: 'users' }))) {
|
|
1034
|
+
* await store.load({ segment: 'users' }, idsFromUpstream);
|
|
1035
|
+
* }
|
|
1036
|
+
* ```
|
|
1037
|
+
*/
|
|
1038
|
+
async exists(ref) {
|
|
1039
|
+
validateSegmentRef(ref);
|
|
1040
|
+
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1041
|
+
}
|
|
1042
|
+
/**
|
|
1043
|
+
* Every segment the registry knows about, streamed — optionally scoped to one namespace.
|
|
1044
|
+
*
|
|
1045
|
+
* The registry is already the list of your segments, which is why you should not keep a second one beside it:
|
|
1046
|
+
* a hand-maintained list is a source of truth that drifts from this one the first time a load fails halfway.
|
|
1047
|
+
*
|
|
1048
|
+
* **An admin/discovery call, not a request-path one.** This is the registry's own enumeration — a paged
|
|
1049
|
+
* LIST over the `registry/` prefix — so its cost grows with the size of the fleet rather than with what you
|
|
1050
|
+
* are looking for.
|
|
1051
|
+
*
|
|
1052
|
+
* Scoping to a namespace narrows the LIST prefix, so it really is the difference between reading one tenant
|
|
1053
|
+
* and reading all of them.
|
|
1054
|
+
*
|
|
1055
|
+
* It streams, and stopping the iteration stops the scan. It reads the registry directly and is not retried: a
|
|
1056
|
+
* transient fault part-way through ends the loop with that error, and calling `segments()` again scans from the
|
|
1057
|
+
* start.
|
|
1058
|
+
*
|
|
1059
|
+
* Yields `destroyed` tombstones and rows with `currentGen: null`, because a filtered enumeration that looks
|
|
1060
|
+
* complete is worse than an honest one — filter on `status`/`currentGen` yourself, or ask
|
|
1061
|
+
* {@link CloudRoaring.exists} the narrower question. Needs a storage backend.
|
|
1062
|
+
*
|
|
1063
|
+
* ```ts
|
|
1064
|
+
* for await (const s of store.segments({ namespace: 'active-daily' })) {
|
|
1065
|
+
* console.log(s.segment, s.currentGen, s.status);
|
|
1066
|
+
* }
|
|
1067
|
+
* ```
|
|
1068
|
+
*/
|
|
1069
|
+
segments(options = {}) {
|
|
1070
|
+
if (options.namespace !== void 0) {
|
|
1071
|
+
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
1072
|
+
refuseReservedNamespace(options.namespace);
|
|
1073
|
+
}
|
|
1074
|
+
return listSegments(this.requireRegistry("segments"), options);
|
|
1075
|
+
}
|
|
1076
|
+
/**
|
|
1077
|
+
* Move this segment's pointer **back** to a generation still in the bucket — the one write in the library that
|
|
1078
|
+
* is not forward-only.
|
|
1079
|
+
*
|
|
1080
|
+
* Forward-only is right for a writer: a load whose ids came from upstream loses nothing by being out-raced, and
|
|
1081
|
+
* letting it regress would let a slow loader silently undo a fast one. It is wrong for an operator who has
|
|
1082
|
+
* looked at the segment, decided the current generation is wrong, and knows which one they want. So this is
|
|
1083
|
+
* reachable only by asking for it by name — no sweep, retry or reconciliation calls it — and it is audited
|
|
1084
|
+
* (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
|
|
1085
|
+
* one cannot.
|
|
1086
|
+
*
|
|
1087
|
+
* It refuses rather than guesses: a generation not in the bucket (collected, or never written) throws
|
|
1088
|
+
* `NotFoundError` naming what *is* available, and a crypto-shredded segment throws
|
|
1089
|
+
* {@link ValidationError} because every generation of it is unreadable. Rolling to the generation already
|
|
1090
|
+
* current is a no-op that reports itself.
|
|
1091
|
+
*
|
|
1092
|
+
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
1093
|
+
* and are then *above* `currentGen`, where collection never looks. They remain until a load numbers above them
|
|
1094
|
+
* (collection then keeps the newest `keep` of what is below its pointer), {@link CloudRoaring.dropSegment}
|
|
1095
|
+
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and only those holding
|
|
1096
|
+
* the id when the current generation does not. An operator who has just undone a bad load should not have the
|
|
1097
|
+
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
1098
|
+
* erasure undoable.
|
|
1099
|
+
*
|
|
1100
|
+
* A target **above** the pointer needs `{ allowForward: true }`, and is refused with {@link ValidationError}
|
|
1101
|
+
* without it: that is also where objects live that were never published, such as a load that wrote its object
|
|
1102
|
+
* and died before the publish. Undoing an earlier rollback is what the opt-in is for.
|
|
1103
|
+
*
|
|
1104
|
+
* Needs a backend.
|
|
1105
|
+
*/
|
|
1106
|
+
async rollback(ref, toGeneration, options = {}) {
|
|
1107
|
+
validateSegmentRef(ref);
|
|
1108
|
+
const deps = this.lifecycleDeps("rollback");
|
|
1109
|
+
try {
|
|
1110
|
+
return await rollbackSegment(ref, toGeneration, deps, options);
|
|
1111
|
+
} finally {
|
|
1112
|
+
this.engine.invalidate(ref);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* **Replace this segment's contents** with `ids`, as one new immutable generation, and make it current.
|
|
1117
|
+
*
|
|
1118
|
+
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1119
|
+
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
1120
|
+
* the one that gets left out is the last, so segments quietly accumulate superseded generations nobody pays
|
|
1121
|
+
* attention to and everybody pays for.
|
|
1122
|
+
*
|
|
1123
|
+
* ```ts
|
|
1124
|
+
* const r = await store.load({ segment: 'audience:active' }, idsFromWarehouse, {
|
|
1125
|
+
* guard: { minRetained: 0.5 }, // refuse a load that would drop more than half the segment
|
|
1126
|
+
* });
|
|
1127
|
+
* if (!r.published) console.warn(`load refused: ${r.reason}`);
|
|
1128
|
+
* ```
|
|
1129
|
+
*
|
|
1130
|
+
* **A load REPLACES.** Whatever the stream contains is what the segment contains afterwards, so an upstream
|
|
1131
|
+
* query that returns fewer rows than usual is a shrink nobody asked for and an empty one is a wipe — both
|
|
1132
|
+
* reported as a successful write, because at the storage layer they are one. That is what `guard` and the
|
|
1133
|
+
* default empty refusal are for, and why they run **between** the write and the publish: the only moment where
|
|
1134
|
+
* the new content is known and the old one is still authoritative.
|
|
1135
|
+
*
|
|
1136
|
+
* **Branch on `published`.** A refusal is a normal outcome, not an exception — `published: false` with a
|
|
1137
|
+
* `reason`, in the same shape as a success — so a discarded result is a load that silently did nothing, and
|
|
1138
|
+
* with the empty guard on by default this verb refuses more readily than any other. Note the sibling `*Into`
|
|
1139
|
+
* verbs *throw* on the same superseded condition rather than reporting it; this one follows
|
|
1140
|
+
* `eraseIdFromSegment` instead, because three of its four refusals are expected guard outcomes rather than
|
|
1141
|
+
* faults.
|
|
1142
|
+
*
|
|
1143
|
+
* `'superseded'` means another writer got there first: another load took the same generation number, so this
|
|
1144
|
+
* one wrote nothing (`size: 0`), or the segment's registry row changed while the load was writing — another load
|
|
1145
|
+
* published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
|
|
1146
|
+
*
|
|
1147
|
+
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
1148
|
+
* collection deliberately never looks — but only while the segment's registry row is unchanged or gone. Once
|
|
1149
|
+
* another write has changed the row, the generation number it holds may name another incarnation's live object,
|
|
1150
|
+
* so it leaves the orphan rather than risk deleting live data. The orphan is an ordinary generation once a later
|
|
1151
|
+
* one is current above it, and collection counts it within `keep`.
|
|
1152
|
+
*
|
|
1153
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
|
|
1154
|
+
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
1155
|
+
* that will not open when a guard reads its size (`IntegrityError`); a driver failure; and a collection pass
|
|
1156
|
+
* that could not prove the segment was still the same one (`WriteConflictError`). The last can be raised
|
|
1157
|
+
* **after** the publish already landed, so a throw does not by itself mean the load did not take effect —
|
|
1158
|
+
* re-read the pointer rather than assuming.
|
|
1159
|
+
*
|
|
1160
|
+
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
1161
|
+
*/
|
|
1162
|
+
async load(ref, ids, options = {}) {
|
|
1163
|
+
validateSegmentRef(ref);
|
|
1164
|
+
const deps = this.lifecycleDeps("load");
|
|
1165
|
+
try {
|
|
1166
|
+
return await loadSegment(ref, ids, deps, options);
|
|
1167
|
+
} finally {
|
|
1168
|
+
this.engine.invalidate(ref);
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
/**
|
|
1172
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** Irreversible.
|
|
1173
|
+
*
|
|
1174
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (bytes unreadable everywhere including
|
|
1175
|
+
* backups, but still sitting in your bucket and still billed, and it requires encryption), while this one
|
|
1176
|
+
* removes the storage and works on a cleartext segment. On an encrypted segment it does both.
|
|
1177
|
+
*
|
|
1178
|
+
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
1179
|
+
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
1180
|
+
* variable twice.
|
|
1181
|
+
*
|
|
1182
|
+
* ```ts
|
|
1183
|
+
* for (const day of expiredDays) {
|
|
1184
|
+
* // The family in the namespace, the date in the segment — `registry.list(namespace)` then enumerates
|
|
1185
|
+
* // exactly this family's buckets, which a flat `active:2026-08-01` name cannot do without string-matching.
|
|
1186
|
+
* const ref = { namespace: 'active-daily', segment: day };
|
|
1187
|
+
* await store.dropSegment(ref, { confirmSegment: ref.segment });
|
|
1188
|
+
* }
|
|
1189
|
+
* ```
|
|
1190
|
+
*
|
|
1191
|
+
* **Omitting the namespace addresses a different segment and is a silent no-op** — you get
|
|
1192
|
+
* `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
|
|
1193
|
+
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
1194
|
+
*
|
|
1195
|
+
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
1196
|
+
* reclaimed and the drop should be re-run. A load that was already writing when the tombstone landed still
|
|
1197
|
+
* finishes its object, so a single sweep can miss it — this call re-sweeps and then reports whatever it still
|
|
1198
|
+
* could not remove rather than returning a result that looks like a clean drop.
|
|
1199
|
+
*
|
|
1200
|
+
* Reads become empty within `cache.genTtlMs` (default 2 s), not instantly: a store that had already read this
|
|
1201
|
+
* segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
|
|
1202
|
+
* cannot be read because of a transient fault, until a retry 500 ms apart reaches it again. A reader that never
|
|
1203
|
+
* touched it sees empty at once. **That bound needs a registry and `cache.genTtlMs > 0`.** A store with no registry
|
|
1204
|
+
* (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a storage source built with no clock, has no timed
|
|
1205
|
+
* refresh. It notices the drop only when a read has to fetch
|
|
1206
|
+
* from a deleted generation or its reader cache evicts the segment, and until then a chunk it has cached
|
|
1207
|
+
* answers without reaching storage, so it can keep answering `true` for a dropped segment indefinitely. Tell
|
|
1208
|
+
* it with {@link invalidate}, or restart it.
|
|
1209
|
+
*
|
|
1210
|
+
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
1211
|
+
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
1212
|
+
*/
|
|
1213
|
+
async dropSegment(ref, options) {
|
|
1214
|
+
validateSegmentRef(ref);
|
|
1215
|
+
const deps = this.lifecycleDeps("dropSegment");
|
|
1216
|
+
try {
|
|
1217
|
+
return await dropSegment(ref, { registry: deps.registry, storage: deps.storage }, options);
|
|
1218
|
+
} finally {
|
|
1219
|
+
if (options.dryRun !== true) this.engine.invalidate(ref);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* **Record when this segment becomes eligible for retirement.** One registry write; nothing is deleted here,
|
|
1224
|
+
* and nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
1225
|
+
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
1226
|
+
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
1227
|
+
*
|
|
1228
|
+
* `expiresAt` is an **absolute epoch-ms you compute**, not a duration the library derives. A relative TTL would
|
|
1229
|
+
* have to be anchored to something the library knows — `updatedAt`, or the current generation — and every load
|
|
1230
|
+
* rewrites both, so "expire 30 days after the last write" would keep a busy daily bucket alive forever
|
|
1231
|
+
* precisely because it is being reloaded.
|
|
1232
|
+
*
|
|
1233
|
+
* ```ts
|
|
1234
|
+
* const DAY = 86_400_000;
|
|
1235
|
+
* const ref = { namespace: 'active-daily', segment: '2026-08-05' };
|
|
1236
|
+
* await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY });
|
|
1237
|
+
* ```
|
|
1238
|
+
*
|
|
1239
|
+
* **Works before the first load**: a segment with no registry row yet gets one (`createdRow: true` in the
|
|
1240
|
+
* result) with **no Storage generation**, so the policy is recorded ahead of the data and the segment is already
|
|
1241
|
+
* enumerable by the sweep; the first load then publishes onto that row.
|
|
1242
|
+
*
|
|
1243
|
+
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
1244
|
+
* buckets is normal. A value below `MIN_EXPIRES_AT_MS` (2001-09-09) is rejected: it is almost certainly epoch
|
|
1245
|
+
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
1246
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a crypto-shredded segment.
|
|
1247
|
+
*/
|
|
1248
|
+
async setRetention(ref, policy) {
|
|
1249
|
+
validateSegmentRef(ref);
|
|
1250
|
+
return setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
|
|
1254
|
+
* tombstone). Returns the string `'invalid'` for a row whose `expiresAt` is present but unusable — a
|
|
1255
|
+
* hand-edited row, or one from a restore — so a malformed policy is visible rather than silently reading as
|
|
1256
|
+
* "never expires" on a segment someone believes is expiring.
|
|
1257
|
+
*/
|
|
1258
|
+
async getRetention(ref) {
|
|
1259
|
+
validateSegmentRef(ref);
|
|
1260
|
+
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
1261
|
+
}
|
|
1262
|
+
/**
|
|
1263
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether a policy was actually removed (`false`
|
|
1264
|
+
* when there was none). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
1265
|
+
* to the setter is how a typo becomes a deletion.
|
|
1266
|
+
*/
|
|
1267
|
+
async clearRetention(ref) {
|
|
1268
|
+
validateSegmentRef(ref);
|
|
1269
|
+
return clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
|
|
1273
|
+
* the policies `setRetention` records — and it is a **call, not a daemon**. Nothing schedules it; you run it from
|
|
1274
|
+
* whatever heartbeat your deployment already has (an EventBridge rule, a Kubernetes CronJob, a queue consumer,
|
|
1275
|
+
* the job that runs your loads). A library that started a timer would behave differently in a Lambda, an edge
|
|
1276
|
+
* isolate and a long-lived server, which is worse than not having one.
|
|
1277
|
+
*
|
|
1278
|
+
* Each retirement goes through `dropSegment`, so the registry → Storage ordering, the re-sweep for an object a
|
|
1279
|
+
* load was still writing, and the `generationsRemaining` report all come from one implementation rather than
|
|
1280
|
+
* two.
|
|
1281
|
+
*
|
|
1282
|
+
* ```ts
|
|
1283
|
+
* // In your scheduled handler. Start with a preview in a new deployment.
|
|
1284
|
+
* const preview = await store.retireExpired({ namespace: 'active-daily', dryRun: true });
|
|
1285
|
+
* console.log(`would retire ${preview.wouldRetire} of ${preview.scanned} (limited: ${preview.limited})`);
|
|
1286
|
+
*
|
|
1287
|
+
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
1288
|
+
* for (const e of swept.entries) {
|
|
1289
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`); // invalid-policy / limit / failed: …
|
|
1290
|
+
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
1291
|
+
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
1292
|
+
* }
|
|
1293
|
+
* }
|
|
1294
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more are still eligible
|
|
1295
|
+
* ```
|
|
1296
|
+
*
|
|
1297
|
+
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
1298
|
+
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
1299
|
+
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
1300
|
+
* `limit` (default 100) cut the pass short and more are eligible. That cap is charged on **attempts**, not
|
|
1301
|
+
* successes, which is what makes it a real bound: `dropSegment` writes the tombstone before sweeping Storage, so a
|
|
1302
|
+
* fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
1303
|
+
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
1304
|
+
* dashboard summing `retired` can never show a phantom deletion.
|
|
1305
|
+
*
|
|
1306
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
1307
|
+
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
1308
|
+
* first, since nothing else ever would for a tombstoned segment. Attribution is a **positive marker the sweep
|
|
1309
|
+
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
1310
|
+
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
1311
|
+
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
1312
|
+
* segment that held nothing has its row deleted in the pass that retires it, since that row would only fence the name.
|
|
1313
|
+
* Pass `purgeTombstones: false` to keep every tombstone, that row included.
|
|
1314
|
+
*
|
|
1315
|
+
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
1316
|
+
* because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
|
|
1317
|
+
*/
|
|
1318
|
+
async retireExpired(options = {}) {
|
|
1319
|
+
const deps = this.lifecycleDeps("retireExpired");
|
|
1320
|
+
const result = await retireExpired(
|
|
1321
|
+
{ registry: deps.registry, storage: deps.storage },
|
|
1322
|
+
{ ...options, now: options.now ?? this.clock.now() }
|
|
1323
|
+
);
|
|
1324
|
+
if (options.dryRun !== true) {
|
|
1325
|
+
for (const entry of result.entries) {
|
|
1326
|
+
this.engine.invalidate({ segment: entry.segment, namespace: entry.namespace });
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
return result;
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
|
|
1333
|
+
* through a path this store did not perform itself.
|
|
1334
|
+
*
|
|
1335
|
+
* A store keeps two layers of derived state — a resolved snapshot per segment (an open `.crbm` reader, plus
|
|
1336
|
+
* the DEK it unwrapped) and decoded chunks keyed by generation. Both are built for one event: **a publish
|
|
1337
|
+
* that advances `currentGen`**, which the snapshot TTL notices and the generation-keyed cache misses on.
|
|
1338
|
+
* Neither notices an event that *destroys* what they were derived from.
|
|
1339
|
+
*
|
|
1340
|
+
* The store's own writes handle themselves: `load`, `rollback`, `eraseSubject`, `dropSegment` and `retireExpired`
|
|
1341
|
+
* here, and a segment's `*Into` verbs, invalidate what they touch. This method is for the cases they cannot see:
|
|
1342
|
+
*
|
|
1343
|
+
* - **`destroySegment` / `eraseNamespace`**, which are free functions over raw drivers rather than methods
|
|
1344
|
+
* here, so a crypto-shred performed beside this store invalidates nothing in it. It keeps its open reader and
|
|
1345
|
+
* the unwrapped DEK, and decrypts with them, chunks it had never fetched before the shred included, until its
|
|
1346
|
+
* snapshot re-resolves: within `cache.genTtlMs` when the store has a registry and a `genTtlMs` above 0, and
|
|
1347
|
+
* otherwise only when its caches let the segment go or a read has to fetch from a generation a sweep collected.
|
|
1348
|
+
* A pin answers with no timed bound, until it is told or both caches let it go.
|
|
1349
|
+
* - **Another process.** Erasing on one box invalidates nothing on the others; each store bounds its own
|
|
1350
|
+
* staleness by `cache.genTtlMs`, and a store with no registry (a bare `IStorageDriver`), with
|
|
1351
|
+
* `cache.genTtlMs: 0`, or on a storage source built with no clock has no bound at all: it converges only when a read happens to miss its caches. If a compliance deadline depends on every
|
|
1352
|
+
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
1353
|
+
*
|
|
1354
|
+
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
1355
|
+
* object back, the pin reads it again.
|
|
1356
|
+
*
|
|
1357
|
+
* Synchronous, best-effort, and safe to call for a segment this store has never read.
|
|
1358
|
+
*
|
|
1359
|
+
* ```ts
|
|
1360
|
+
* await destroySegment(ref, { registry: backend.registry }, { confirmSegment: ref.segment });
|
|
1361
|
+
* store.invalidate(ref); // this process
|
|
1362
|
+
* await bus.publish('cloudbitmaps.invalidate', ref); // and every other one
|
|
1363
|
+
* ```
|
|
1364
|
+
*/
|
|
1365
|
+
invalidate(ref) {
|
|
1366
|
+
validateSegmentRef(ref);
|
|
1367
|
+
refuseReservedNamespace(ref.namespace);
|
|
1368
|
+
this.engine.invalidate(ref);
|
|
1369
|
+
}
|
|
1370
|
+
/**
|
|
1371
|
+
* Build a handle held at the generation `ref` resolves to right now. See {@link Segment.pin}.
|
|
1372
|
+
*
|
|
1373
|
+
* The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
|
|
1374
|
+
* pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
|
|
1375
|
+
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
1376
|
+
*/
|
|
1377
|
+
async pinSegment(ref, expiresAt) {
|
|
1378
|
+
const crbm = this.crbmSource;
|
|
1379
|
+
if (crbm === void 0) {
|
|
1380
|
+
throw new UnsupportedError(
|
|
1381
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
const at = await this.withRetries(() => crbm.pinGeneration(ref));
|
|
1385
|
+
const pinnedAt = {
|
|
1386
|
+
generation: at?.generation ?? null,
|
|
1387
|
+
version: at?.version ?? null,
|
|
1388
|
+
fingerprint: at?.fingerprint ?? null
|
|
1389
|
+
};
|
|
1390
|
+
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
1391
|
+
return makeSegment({
|
|
1392
|
+
engine: this.engineWithPins(pins),
|
|
1393
|
+
ref,
|
|
1394
|
+
clock: this.clock,
|
|
1395
|
+
metrics: this.metrics,
|
|
1396
|
+
materialize: (dest, ids, op, options) => this.materialize(dest, ids, op, options),
|
|
1397
|
+
pinned: (r, e) => this.pinSegment(r, e),
|
|
1398
|
+
combineEngine: (handles) => this.engineForCombine(handles),
|
|
1399
|
+
expiresAt,
|
|
1400
|
+
pinnedAt
|
|
1401
|
+
});
|
|
1402
|
+
}
|
|
1403
|
+
/**
|
|
1404
|
+
* An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
|
|
1405
|
+
* retries, so a transient fault on a pinned read is retried as it would be on a live one.
|
|
1406
|
+
*
|
|
1407
|
+
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment.
|
|
1408
|
+
*/
|
|
1409
|
+
engineWithPins(pins) {
|
|
1410
|
+
const crbm = this.crbmSource;
|
|
1411
|
+
if (crbm === void 0) {
|
|
1412
|
+
throw new UnsupportedError(
|
|
1413
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
|
|
1414
|
+
);
|
|
1415
|
+
}
|
|
1416
|
+
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
1417
|
+
return new SegmentEngine({
|
|
1418
|
+
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource(pinned, this.retryOptions),
|
|
1419
|
+
cache: this.cache,
|
|
1420
|
+
codec: roaringCodec,
|
|
1421
|
+
clock: this.clock,
|
|
1422
|
+
metrics: this.metrics,
|
|
1423
|
+
budget: this.budget
|
|
1424
|
+
});
|
|
1425
|
+
}
|
|
1426
|
+
/**
|
|
1427
|
+
* The engine a combine should run on, given the handles involved.
|
|
1428
|
+
*
|
|
1429
|
+
* A pinned handle passed as an **operand** must still be read at its pin. Routing by "which handle the call
|
|
1430
|
+
* was made on" reads it live instead — `snap.intersect([other])` honours the pin while
|
|
1431
|
+
* `other.intersect([snap])` silently does not, and the two are the same question. So the combine collects
|
|
1432
|
+
* every pin in play and runs on an engine that honours all of them. With no pins anywhere this is the
|
|
1433
|
+
* store's own engine and costs nothing.
|
|
1434
|
+
*
|
|
1435
|
+
* One engine reads a segment at one generation, so a combine that holds the same segment at two — two pins of
|
|
1436
|
+
* it, or a pin and a live handle — is refused. Keeping one of them would answer the question for the wrong
|
|
1437
|
+
* instant, and say nothing: diffing two snapshots of a segment returned no ids at all.
|
|
1438
|
+
*/
|
|
1439
|
+
engineForCombine(handles) {
|
|
1440
|
+
if (!handles.some((h) => h.pinnedAt !== void 0)) return void 0;
|
|
1441
|
+
const pins = /* @__PURE__ */ new Map();
|
|
1442
|
+
const held = /* @__PURE__ */ new Map();
|
|
1443
|
+
for (const h of handles) {
|
|
1444
|
+
const key = h.key();
|
|
1445
|
+
const at = h.pinnedAt;
|
|
1446
|
+
if (held.has(key)) {
|
|
1447
|
+
const before = held.get(key);
|
|
1448
|
+
if (!samePin(before, at)) {
|
|
1449
|
+
throw new ValidationError2(
|
|
1450
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`intersectInto(dest, [])\``
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
1453
|
+
} else {
|
|
1454
|
+
held.set(key, at);
|
|
1455
|
+
}
|
|
1456
|
+
if (at !== void 0) pins.set(key, at);
|
|
1457
|
+
}
|
|
1458
|
+
return this.engineWithPins(pins);
|
|
1459
|
+
}
|
|
1460
|
+
/** `op` under the store's retries, as the store's own reads run; bare when the store has retries off. */
|
|
1461
|
+
withRetries(op) {
|
|
1462
|
+
const r = this.retryOptions;
|
|
1463
|
+
return r === void 0 ? op() : withRetry(op, r.policy ?? DEFAULT_RETRY_POLICY, r);
|
|
1464
|
+
}
|
|
1465
|
+
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
1466
|
+
requireRegistry(op) {
|
|
1467
|
+
if (this.registry === void 0) {
|
|
1468
|
+
throw new UnsupportedError(
|
|
1469
|
+
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
1470
|
+
);
|
|
1471
|
+
}
|
|
1472
|
+
return this.registry;
|
|
1473
|
+
}
|
|
1474
|
+
/**
|
|
1475
|
+
* **Cross-tier DR consistency check.** After a restore/failover, verify every registered segment's `currentGen`
|
|
1476
|
+
* actually has its `.crbm` present in Storage — catching a **torn restore** where the registry (`currentGen`) came
|
|
1477
|
+
* back ahead of the object store, so a pointer references a generation that isn't there (reads would then
|
|
1478
|
+
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
1479
|
+
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
1480
|
+
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
1481
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded) segments are skipped. Pair it with
|
|
1482
|
+
* the DR runbook (docs/guide/disaster-recovery.md).
|
|
1483
|
+
*
|
|
1484
|
+
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
1485
|
+
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
1486
|
+
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
1487
|
+
* `storage` and `registry` with a higher `maxScanSegments`.
|
|
1488
|
+
*/
|
|
1489
|
+
async checkConsistency(options = {}) {
|
|
1490
|
+
const deps = this.lifecycleDeps("checkConsistency");
|
|
1491
|
+
return runConsistencyCheck({ storage: deps.storage, registry: deps.registry }, options);
|
|
1492
|
+
}
|
|
1493
|
+
/**
|
|
1494
|
+
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
1495
|
+
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
1496
|
+
* data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
|
|
1497
|
+
* (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
|
|
1498
|
+
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
1499
|
+
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
1500
|
+
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
1501
|
+
* Crypto-shredded segments are skipped. A segment that can't be read is isolated into the manifest's `failed[]`
|
|
1502
|
+
* (the run continues), so "a manifest exists" means the run finished — check `failed`. The `export-segments`
|
|
1503
|
+
* CLI wraps this with a filesystem sink.
|
|
1504
|
+
*/
|
|
1505
|
+
async exportSegments(sink, options = {}) {
|
|
1506
|
+
const registry = this.requireRegistry("exportSegments");
|
|
1507
|
+
return runExport(this, registry, sink, {
|
|
1508
|
+
...options,
|
|
1509
|
+
codec: options.codec ?? roaringCodec
|
|
1510
|
+
});
|
|
1511
|
+
}
|
|
1512
|
+
/**
|
|
1513
|
+
* Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
|
|
1514
|
+
* from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
1515
|
+
*/
|
|
1516
|
+
static estimateCost(input) {
|
|
1517
|
+
return estimateCost(input);
|
|
1518
|
+
}
|
|
1519
|
+
};
|
|
1520
|
+
var failing = (err) => ({
|
|
1521
|
+
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
|
|
1522
|
+
});
|
|
1523
|
+
var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
|
|
1524
|
+
var twoPins = (a, b) => {
|
|
1525
|
+
const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
|
|
1526
|
+
return a !== void 0 && b !== void 0 && a.generation === b.generation ? `pinned twice at generation ${a.generation ?? "none"}, as two different objects` : `${one(a)} and ${one(b)}`;
|
|
1527
|
+
};
|
|
1528
|
+
var rangeOf = (options) => ({
|
|
1529
|
+
after: options.after,
|
|
1530
|
+
through: options.through
|
|
1531
|
+
});
|
|
1532
|
+
var readOptions = (options) => ({
|
|
1533
|
+
...rangeOf(options),
|
|
1534
|
+
concurrency: options.concurrency,
|
|
1535
|
+
budget: options.budget,
|
|
1536
|
+
allowAbsentOperands: options.allowAbsentOperands
|
|
1537
|
+
});
|
|
1538
|
+
var NO_SEGMENTS = [];
|
|
1539
|
+
var EMPTY_IDS = {
|
|
1540
|
+
async *[Symbol.asyncIterator]() {
|
|
1541
|
+
}
|
|
1542
|
+
};
|
|
1543
|
+
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
1544
|
+
var makeSegment;
|
|
1545
|
+
var minting = false;
|
|
1546
|
+
var Segment = class _Segment {
|
|
1547
|
+
metricsOn;
|
|
1548
|
+
engine;
|
|
1549
|
+
ref;
|
|
1550
|
+
clock;
|
|
1551
|
+
metrics;
|
|
1552
|
+
materialize;
|
|
1553
|
+
pinned;
|
|
1554
|
+
combineEngine;
|
|
1555
|
+
/** Absolute epoch-ms deadline from {@link SegmentOptions.expiresAt}; `undefined` ⇒ this handle never expires. */
|
|
1556
|
+
expiresAt;
|
|
1557
|
+
/**
|
|
1558
|
+
* The generation this handle is held at, when it came from {@link Segment.pin}. Read by the store so a
|
|
1559
|
+
* pinned handle passed as an **operand** is still read at its pin rather than live.
|
|
1560
|
+
*/
|
|
1561
|
+
pinnedAt;
|
|
1562
|
+
static {
|
|
1563
|
+
makeSegment = (parts) => {
|
|
1564
|
+
minting = true;
|
|
1565
|
+
try {
|
|
1566
|
+
return new _Segment(parts);
|
|
1567
|
+
} finally {
|
|
1568
|
+
minting = false;
|
|
1569
|
+
}
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1573
|
+
* Not constructible: a handle comes from {@link CloudRoaring.segment}, which wires it to the store's engine,
|
|
1574
|
+
* caches and write path. The constructor takes those internals, so it is not part of the surface.
|
|
1575
|
+
*/
|
|
1576
|
+
constructor(parts) {
|
|
1577
|
+
if (!minting) {
|
|
1578
|
+
throw new ValidationError2(
|
|
1579
|
+
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1582
|
+
this.engine = parts.engine;
|
|
1583
|
+
this.ref = parts.ref;
|
|
1584
|
+
this.clock = parts.clock;
|
|
1585
|
+
this.metrics = parts.metrics;
|
|
1586
|
+
this.materialize = parts.materialize;
|
|
1587
|
+
this.pinned = parts.pinned;
|
|
1588
|
+
this.combineEngine = parts.combineEngine;
|
|
1589
|
+
this.expiresAt = parts.expiresAt;
|
|
1590
|
+
this.pinnedAt = parts.pinnedAt;
|
|
1591
|
+
this.metricsOn = parts.metrics !== NOOP_METRICS;
|
|
1592
|
+
}
|
|
1593
|
+
/**
|
|
1594
|
+
* **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
|
|
1595
|
+
*
|
|
1596
|
+
* An ordinary handle re-resolves on `cache.genTtlMs`, so a publish part-way through a long job means its second
|
|
1597
|
+
* half describes a different instant than its first — every chunk whole and verified, but the answer covering
|
|
1598
|
+
* two moments, with nothing in the result saying so. That is fine for a dashboard and wrong for an export, a
|
|
1599
|
+
* reconciliation, or a send that has to match the count you reported. A pin is how you get one instant.
|
|
1600
|
+
*
|
|
1601
|
+
* ```ts
|
|
1602
|
+
* const snap = await store.segment('active-30d').pin();
|
|
1603
|
+
* const total = await snap.count(); // the number you report
|
|
1604
|
+
* for await (const id of snap.iterate()) { … } // …and the ids it counted, however long this takes
|
|
1605
|
+
* ```
|
|
1606
|
+
*
|
|
1607
|
+
* **Only this segment is pinned.** `snap.intersect([other])` reads `snap` at its pinned generation and
|
|
1608
|
+
* `other` at whatever is current — pin each segment if you want the whole query held. And a pinned handle
|
|
1609
|
+
* used as an *operand* is still read at its pin, never live.
|
|
1610
|
+
*
|
|
1611
|
+
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
1612
|
+
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
1613
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `intersectInto(dest, [])`.
|
|
1614
|
+
*
|
|
1615
|
+
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
1616
|
+
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
1617
|
+
* instant it pinned, and anything it would have to fetch fails with `NotFoundError`, as a swept pin's does.
|
|
1618
|
+
*
|
|
1619
|
+
* **It is a hold, not a lease.** Nothing here stops a collection deleting the generation underneath
|
|
1620
|
+
* you: a pinned read deliberately does **not** heal forward, because silently serving a different generation
|
|
1621
|
+
* is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover your longest pinned
|
|
1622
|
+
* job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) — or take the pin on a segment
|
|
1623
|
+
* you are not collecting.
|
|
1624
|
+
*
|
|
1625
|
+
* A segment with no current generation pins nothing and reads empty, exactly as it would unpinned. A pinned
|
|
1626
|
+
* segment whose row is later dropped or destroyed fails once it must open its object again, rather than go empty
|
|
1627
|
+
* part-way through a call. **A pin
|
|
1628
|
+
* keeps the key its reader unwrapped for as long as that reader stays open, and answers from the chunks it
|
|
1629
|
+
* decoded for as long as they stay cached.** This store invalidates a pin:
|
|
1630
|
+
*
|
|
1631
|
+
* - a `load`, a `rollback` or an `*Into` invalidates a pin of the segment it writes;
|
|
1632
|
+
* - `dropSegment` invalidates a pin of the segment it drops, and `retireExpired` a pin of each segment its ledger
|
|
1633
|
+
* lists, retired or not, neither on a dry run;
|
|
1634
|
+
* - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
|
|
1635
|
+
*
|
|
1636
|
+
* An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
|
|
1637
|
+
* destroyed.
|
|
1638
|
+
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
1639
|
+
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
1640
|
+
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
1641
|
+
* {@link CloudRoaring.invalidate} is called on this store. Where the
|
|
1642
|
+
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
1643
|
+
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
1644
|
+
*
|
|
1645
|
+
* A replacement a pin has found is remembered, so its later chunk reads fail with no request, until the store
|
|
1646
|
+
* forgets it: an invalidation of the segment does, as does a later `pin()` of the same version that opens the object
|
|
1647
|
+
* again, and the store remembers at most `cache.readerMax` of them. So once a restore puts the pinned object back,
|
|
1648
|
+
* call {@link CloudRoaring.invalidate} on this store: the pin then reads the object again.
|
|
1649
|
+
*
|
|
1650
|
+
* `pin()` reads the registry row and opens the generation at once, so the pin knows its object before its first
|
|
1651
|
+
* read. With a registry, pins of one generation taken while its row is unchanged share one reader while the store
|
|
1652
|
+
* keeps it open, pins taken at the same moment included: the first costs a tail read, and a key unwrap for an
|
|
1653
|
+
* encrypted segment, whether or not it is read, and the rest cost the row read alone. Without a registry every
|
|
1654
|
+
* `pin()` lists the segment's objects and makes the tail read, since only the object can tell two incarnations of
|
|
1655
|
+
* a name apart there. It retries a
|
|
1656
|
+
* transient fault as the store's reads do, and heals a generation swept between the two, but it fails where the
|
|
1657
|
+
* generation cannot be opened: `NotFoundError` for a pointer at a missing object, `IntegrityError` for a damaged
|
|
1658
|
+
* or misfiled one.
|
|
1659
|
+
*
|
|
1660
|
+
* Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
|
|
1661
|
+
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
1662
|
+
*/
|
|
1663
|
+
async pin() {
|
|
1664
|
+
return this.pinned(this.ref, this.expiresAt);
|
|
1665
|
+
}
|
|
1666
|
+
/**
|
|
1667
|
+
* An opaque string that names this handle's segment, namespace included: two handles of one segment have the
|
|
1668
|
+
* same key, a live handle and its pins among them. Use it as a `Map` key or a log field when you track handles
|
|
1669
|
+
* and have not kept the name you made them with. The format is not specified and is not a storage key, so
|
|
1670
|
+
* compare keys, never parse one.
|
|
1671
|
+
*/
|
|
1672
|
+
key() {
|
|
1673
|
+
return segmentKey(this.ref);
|
|
1674
|
+
}
|
|
1675
|
+
/**
|
|
1676
|
+
* Has this handle's deadline passed? **The lazy half of expiry** — the whole check is one comparison against
|
|
1677
|
+
* the injected clock, so it costs nothing on a handle with no deadline and no I/O on one that has expired.
|
|
1678
|
+
*
|
|
1679
|
+
* Deliberately evaluated per call rather than cached: a long-lived handle created before its deadline must
|
|
1680
|
+
* start reading empty the moment the deadline passes, without the caller re-creating it.
|
|
1681
|
+
*/
|
|
1682
|
+
expired() {
|
|
1683
|
+
return this.expiresAt !== void 0 && this.clock.now() >= this.expiresAt;
|
|
1684
|
+
}
|
|
1685
|
+
/**
|
|
1686
|
+
* Refuse a materialisation that involves an **expired** handle. Called by the three `*Into` verbs only.
|
|
1687
|
+
*
|
|
1688
|
+
* On the read verbs an expired handle answers empty, which is the point of lazy expiry. On a *write* the same
|
|
1689
|
+
* rule would be destructive in a way nobody asks for: `a.intersectInto(dest, [b])` where `b`'s deadline has
|
|
1690
|
+
* quietly passed would, unrefused, replace `dest` with an **empty generation** — a wipe, reported as a successful
|
|
1691
|
+
* write, with the cause (a deadline on a handle somewhere) nowhere in the result. Reads degrade to empty; writes must not.
|
|
1692
|
+
*
|
|
1693
|
+
* So every handle in the call is checked, `dest` included: an expired `dest` does not change the bytes written,
|
|
1694
|
+
* but a caller who put a deadline on the thing they are writing into has said something contradictory and is
|
|
1695
|
+
* better told than guessed at. Open a handle without `expiresAt` to write, or drop the deadline.
|
|
1696
|
+
*
|
|
1697
|
+
* (The broader guard — refusing to publish an empty or implausible generation over a non-empty one, with an
|
|
1698
|
+
* `allowEmpty` override — covers these verbs too: they route through the same guarded write path as
|
|
1699
|
+
* {@link CloudRoaring.load}. The two stay separate because they differ in kind. That one is a REPORTED
|
|
1700
|
+
* refusal a caller may legitimately override; an expired handle is a wiring mistake, so it THROWS, before
|
|
1701
|
+
* any object is written — and `allowEmpty: true` does not reach it.)
|
|
1702
|
+
*
|
|
1703
|
+
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
1704
|
+
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
1705
|
+
* awaiting.
|
|
1706
|
+
*/
|
|
1707
|
+
refuseIfExpired(op, dest, operands) {
|
|
1708
|
+
const stale = [];
|
|
1709
|
+
for (const seg of [this, dest, ...operands]) {
|
|
1710
|
+
if (seg.expired())
|
|
1711
|
+
stale.push(seg.ref.namespace ? `${seg.ref.namespace}/${seg.ref.segment}` : seg.ref.segment);
|
|
1712
|
+
}
|
|
1713
|
+
if (stale.length > 0) {
|
|
1714
|
+
throw new ValidationError2(
|
|
1715
|
+
`${op}: refusing to publish a generation while these handles have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired handle reads as empty, so this would write an empty (or short) generation over "${dest.ref.segment}". Open the handles without \`expiresAt\` if you meant to materialise, or drop the deadline.`
|
|
1716
|
+
);
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
1721
|
+
* entirely when no sink is wired, so the default path pays nothing.
|
|
1722
|
+
*/
|
|
1723
|
+
async timed(name, fn) {
|
|
1724
|
+
if (!this.metricsOn) return fn();
|
|
1725
|
+
const startedAt = this.clock.now();
|
|
1726
|
+
try {
|
|
1727
|
+
return await fn();
|
|
1728
|
+
} finally {
|
|
1729
|
+
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
/** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
|
|
1733
|
+
has(id) {
|
|
1734
|
+
if (this.expired()) return Promise.resolve(false);
|
|
1735
|
+
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
1736
|
+
}
|
|
1737
|
+
/**
|
|
1738
|
+
* Cardinality — summed from the `.crbm` index with **zero payload reads** on a loaded segment.
|
|
1739
|
+
*
|
|
1740
|
+
* What this trusts: the answer is the sum of the per-chunk cardinalities the index records, and no payload is
|
|
1741
|
+
* decoded to confirm it. The index is checked for internal consistency when the object is opened (each key in
|
|
1742
|
+
* range and ascending, each cardinality in `1..65536`, each payload inside the payload region, and, on an
|
|
1743
|
+
* unencrypted object, the footer's chunk count and total agreeing with the index), and a corrupt index that is
|
|
1744
|
+
* still internally consistent yields a wrong count. `iterate()` and the combines decode the payloads, whose
|
|
1745
|
+
* structure is checked.
|
|
1746
|
+
*/
|
|
1747
|
+
count() {
|
|
1748
|
+
if (this.expired()) return Promise.resolve(0);
|
|
1749
|
+
return this.timed("count", () => this.engine.count(this.ref));
|
|
1750
|
+
}
|
|
1751
|
+
/**
|
|
1752
|
+
* Every id, ascending, streamed one chunk at a time. Pass a range to read part of the segment: the ids in
|
|
1753
|
+
* `(after, through]`, fetching only the chunks the range overlaps. The per-op budget is charged for every chunk in
|
|
1754
|
+
* the range, so give a page `through` as well as `after`. On a pinned handle it reads the pinned generation, as a
|
|
1755
|
+
* full read does.
|
|
1756
|
+
*
|
|
1757
|
+
* ```ts
|
|
1758
|
+
* // Keyset paging: each page resumes where the one before ended, and stops at its window's end.
|
|
1759
|
+
* for await (const id of seg.iterate({ after: previousEnd, through: windowEnd })) await send(id);
|
|
1760
|
+
* ```
|
|
1761
|
+
*
|
|
1762
|
+
* Each bound is optional and an integer in `0..4294967295`; a bad one throws {@link ValidationError} when the
|
|
1763
|
+
* stream is first read. `after >= through` is an empty range, which reads nothing. An expired handle reads empty
|
|
1764
|
+
* without checking its options, as every read of one does.
|
|
1765
|
+
*/
|
|
1766
|
+
iterate(options) {
|
|
1767
|
+
if (this.expired()) return EMPTY_IDS;
|
|
1768
|
+
const range = options == null ? void 0 : rangeOf(options);
|
|
1769
|
+
const none = range === void 0 || range.after === void 0 && range.through === void 0;
|
|
1770
|
+
return this.engine.iterate(this.ref, none ? void 0 : range);
|
|
1771
|
+
}
|
|
1772
|
+
/**
|
|
1773
|
+
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
1774
|
+
* it was given, already reduced to its live handles ({@link liveExcludes}) and mapped down to the plain refs
|
|
1775
|
+
* `core` takes. A method rather than a module function because `ref` is class-private — the encapsulation is
|
|
1776
|
+
* worth more than the free function.
|
|
1777
|
+
*
|
|
1778
|
+
* Each field is named rather than copied with a rest spread: a rest copy takes only own enumerable properties, so
|
|
1779
|
+
* a bound held in a getter or inherited from a prototype was dropped, and the read silently widened to the whole
|
|
1780
|
+
* segment.
|
|
1781
|
+
*/
|
|
1782
|
+
refsIn(options, exclude) {
|
|
1783
|
+
if (options == null) return void 0;
|
|
1784
|
+
return {
|
|
1785
|
+
...readOptions(options),
|
|
1786
|
+
exclude: exclude.length > 0 ? exclude.map((o) => o.ref) : void 0
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1789
|
+
/**
|
|
1790
|
+
* The handles in `options.exclude` that have not expired. **An expired exclusion excludes nothing**, in every
|
|
1791
|
+
* combine, so it is dropped here, before the engine is asked for anything: it is never fetched, never
|
|
1792
|
+
* checked for absence, and never counted in the pin-consistency check. `exclude` is read once, so a getter sees
|
|
1793
|
+
* one call.
|
|
1794
|
+
*
|
|
1795
|
+
* With no `exclude` this is one property read and a shared empty list, and with none expired it returns the
|
|
1796
|
+
* caller's own array, so the common case allocates nothing.
|
|
1797
|
+
*/
|
|
1798
|
+
liveExcludes(options) {
|
|
1799
|
+
const exclude = options?.exclude;
|
|
1800
|
+
if (exclude == null || exclude.length === 0) return NO_SEGMENTS;
|
|
1801
|
+
return exclude.some((e) => e.expired()) ? exclude.filter((e) => !e.expired()) : exclude;
|
|
1802
|
+
}
|
|
1803
|
+
/**
|
|
1804
|
+
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
1805
|
+
* Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
1806
|
+
* and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
|
|
1807
|
+
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
1808
|
+
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
|
|
1809
|
+
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
1810
|
+
*/
|
|
1811
|
+
intersect(others, options) {
|
|
1812
|
+
if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
|
|
1813
|
+
const exclude = this.liveExcludes(options);
|
|
1814
|
+
let engine;
|
|
1815
|
+
try {
|
|
1816
|
+
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
1817
|
+
} catch (err) {
|
|
1818
|
+
return failing(err);
|
|
1819
|
+
}
|
|
1820
|
+
return engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
|
|
1821
|
+
}
|
|
1822
|
+
/**
|
|
1823
|
+
* Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
|
|
1824
|
+
* are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
|
|
1825
|
+
* as a load is (the pointer advances to it, fenced on the row), so readers of `dest` see either the old generation
|
|
1826
|
+
* or the new one, never a partial. Needs the
|
|
1827
|
+
* store built with a backend (throws {@link UnsupportedError} otherwise).
|
|
1828
|
+
*
|
|
1829
|
+
* **An empty result does NOT overwrite a non-empty destination.** A combine that comes out empty is far more
|
|
1830
|
+
* often a mistake upstream — a typo'd operand, an `exclude` that swallowed everything, an operand that has
|
|
1831
|
+
* not loaded yet — than an intent, and once it publishes it is indistinguishable from a correct run. So the
|
|
1832
|
+
* write is **refused and reported**: `published: false`, `reason: 'empty'`, and `dest` keeps what it had.
|
|
1833
|
+
* Pass `allowEmpty: true` when emptying the destination is the point.
|
|
1834
|
+
*
|
|
1835
|
+
* `guard` adds the same plausibility bounds `load()` takes — `minCardinality` and `minRetained` — judged
|
|
1836
|
+
* against what `dest` held before. A refusal is **reported, not thrown**, exactly as on `load()`; branch on
|
|
1837
|
+
* `published`. A lost race is the one outcome that still throws ({@link WriteConflictError}), because a
|
|
1838
|
+
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
1839
|
+
*
|
|
1840
|
+
* A call involving an **expired handle** is refused before any of this. See {@link refuseIfExpired}.
|
|
1841
|
+
*/
|
|
1842
|
+
async intersectInto(dest, others, options) {
|
|
1843
|
+
this.refuseIfExpired("intersectInto", dest, [...others, ...options?.exclude ?? []]);
|
|
1844
|
+
this.combineEngine([this, ...others, ...options?.exclude ?? []]);
|
|
1845
|
+
return this.timed(
|
|
1846
|
+
"intersectInto",
|
|
1847
|
+
() => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
|
|
1848
|
+
);
|
|
1849
|
+
}
|
|
1850
|
+
/**
|
|
1851
|
+
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
1852
|
+
*
|
|
1853
|
+
* **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
|
|
1854
|
+
* here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
|
|
1855
|
+
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
1856
|
+
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
1857
|
+
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
1858
|
+
* segment once (`unionInto`, or a load) is the cheaper shape.
|
|
1859
|
+
*/
|
|
1860
|
+
union(others, options) {
|
|
1861
|
+
const live = others.filter((o) => !o.expired());
|
|
1862
|
+
if (this.expired()) {
|
|
1863
|
+
if (live.length === 0) return EMPTY_IDS;
|
|
1864
|
+
return live[0].union(live.slice(1), options);
|
|
1865
|
+
}
|
|
1866
|
+
if (live.length === 0 && others.length > 0) {
|
|
1867
|
+
const exclude2 = this.liveExcludes(options);
|
|
1868
|
+
return exclude2.length > 0 ? this.andNot([...exclude2], options) : this.union([], options);
|
|
1869
|
+
}
|
|
1870
|
+
if (live.length !== others.length) return this.union(live, options);
|
|
1871
|
+
const exclude = this.liveExcludes(options);
|
|
1872
|
+
let engine;
|
|
1873
|
+
try {
|
|
1874
|
+
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
1875
|
+
} catch (err) {
|
|
1876
|
+
return failing(err);
|
|
1877
|
+
}
|
|
1878
|
+
return engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
|
|
1879
|
+
}
|
|
1880
|
+
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
1881
|
+
async unionInto(dest, others, options) {
|
|
1882
|
+
this.refuseIfExpired("unionInto", dest, [...others, ...options?.exclude ?? []]);
|
|
1883
|
+
this.combineEngine([this, ...others, ...options?.exclude ?? []]);
|
|
1884
|
+
return this.timed(
|
|
1885
|
+
"unionInto",
|
|
1886
|
+
() => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
|
|
1887
|
+
);
|
|
1888
|
+
}
|
|
1889
|
+
/**
|
|
1890
|
+
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
1891
|
+
*
|
|
1892
|
+
* Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
|
|
1893
|
+
* overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
|
|
1894
|
+
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
1895
|
+
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
1896
|
+
*
|
|
1897
|
+
* An expired handle in `excludes` excludes nothing, and is skipped without being read; if every one has
|
|
1898
|
+
* expired the result is `this` whole. An expired `this` is empty. `exclude` on `intersect` and `union` follows
|
|
1899
|
+
* the same rule.
|
|
1900
|
+
*
|
|
1901
|
+
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
1902
|
+
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
1903
|
+
*/
|
|
1904
|
+
andNot(excludes, options) {
|
|
1905
|
+
if (this.expired()) return EMPTY_IDS;
|
|
1906
|
+
const liveExcludes = excludes.filter((e) => !e.expired());
|
|
1907
|
+
if (liveExcludes.length === 0 && excludes.length > 0) {
|
|
1908
|
+
return this.union([], options == null ? void 0 : readOptions(options));
|
|
1909
|
+
}
|
|
1910
|
+
if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
|
|
1911
|
+
let engine;
|
|
1912
|
+
try {
|
|
1913
|
+
engine = this.combineEngine([this, ...excludes]) ?? this.engine;
|
|
1914
|
+
} catch (err) {
|
|
1915
|
+
return failing(err);
|
|
1916
|
+
}
|
|
1917
|
+
return engine.andNot(
|
|
1918
|
+
this.ref,
|
|
1919
|
+
excludes.map((o) => o.ref),
|
|
1920
|
+
options == null ? void 0 : readOptions(options)
|
|
1921
|
+
);
|
|
1922
|
+
}
|
|
1923
|
+
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
1924
|
+
async andNotInto(dest, excludes, options) {
|
|
1925
|
+
this.refuseIfExpired("andNotInto", dest, excludes);
|
|
1926
|
+
this.combineEngine([this, ...excludes]);
|
|
1927
|
+
return this.timed(
|
|
1928
|
+
"andNotInto",
|
|
1929
|
+
() => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
|
|
1930
|
+
);
|
|
1931
|
+
}
|
|
1932
|
+
/**
|
|
1933
|
+
* Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
|
|
1934
|
+
* reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
|
|
1935
|
+
* storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
|
|
1936
|
+
* refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
|
|
1937
|
+
* the lose-zone).
|
|
1938
|
+
*/
|
|
1939
|
+
async costReport(options) {
|
|
1940
|
+
const canMeasure = this.engine.supportsStorageSize;
|
|
1941
|
+
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
1942
|
+
const refreshMs = this.engine.pointerRefreshMs;
|
|
1943
|
+
const workload = refreshMs === void 0 || options?.workload?.genTtlMs !== void 0 ? options?.workload : { ...options?.workload, genTtlMs: refreshMs };
|
|
1944
|
+
return groundedReport({
|
|
1945
|
+
storageBytes: size?.sizeBytes ?? 0,
|
|
1946
|
+
grounded: canMeasure,
|
|
1947
|
+
workload,
|
|
1948
|
+
pricing: options?.pricing,
|
|
1949
|
+
extraNotes: canMeasure ? void 0 : ["storage source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
1952
|
+
};
|
|
1953
|
+
|
|
1954
|
+
// src/bin/export-segments.ts
|
|
19
1955
|
function parseConfig(env) {
|
|
20
1956
|
const root = env.CR_EXPORT_ROOT;
|
|
21
1957
|
if (root === void 0 || root === "") {
|
|
@@ -30,14 +1966,14 @@ function parseConfig(env) {
|
|
|
30
1966
|
throw new Error(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
31
1967
|
}
|
|
32
1968
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
33
|
-
return { root, out, format, namespace
|
|
1969
|
+
return { root, out, format, namespace };
|
|
34
1970
|
}
|
|
35
1971
|
function fsSink(out) {
|
|
36
1972
|
return {
|
|
37
1973
|
async open(ref, ext) {
|
|
38
|
-
const dir = join(out, ref.namespace
|
|
1974
|
+
const dir = join(out, namespacePathPart(ref.namespace));
|
|
39
1975
|
await mkdir(dir, { recursive: true, mode: 448 });
|
|
40
|
-
const finalPath = join(dir, `${ref.segment}${ext}`);
|
|
1976
|
+
const finalPath = join(dir, `${encodeNameForPath(ref.segment)}${ext}`);
|
|
41
1977
|
const tmpPath = `${finalPath}.${randomUUID()}.part`;
|
|
42
1978
|
const handle = await open(tmpPath, "wx", 384);
|
|
43
1979
|
return {
|
|
@@ -73,16 +2009,11 @@ var log = (obj) => {
|
|
|
73
2009
|
};
|
|
74
2010
|
async function main(env = process.env, now = () => Date.now()) {
|
|
75
2011
|
const config = parseConfig(env);
|
|
76
|
-
const
|
|
77
|
-
const store = new CloudRoaring({
|
|
78
|
-
cold: new src_exports.LocalFsColdDriver(join(config.root, "cold")),
|
|
79
|
-
warm: new src_exports.LocalFsWarmDriver(join(config.root, "warm")),
|
|
80
|
-
registry
|
|
81
|
-
});
|
|
2012
|
+
const storage = new LocalFsStorage(config.root);
|
|
2013
|
+
const store = new CloudRoaring({ storage });
|
|
82
2014
|
const manifest = await store.exportSegments(fsSink(config.out), {
|
|
83
2015
|
format: config.format,
|
|
84
|
-
namespace: config.namespace
|
|
85
|
-
candidates: config.segments
|
|
2016
|
+
namespace: config.namespace
|
|
86
2017
|
});
|
|
87
2018
|
await mkdir(config.out, { recursive: true, mode: 448 });
|
|
88
2019
|
const manifestPath = join(config.out, "manifest.json");
|
|
@@ -110,7 +2041,16 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
110
2041
|
});
|
|
111
2042
|
return manifest;
|
|
112
2043
|
}
|
|
113
|
-
|
|
2044
|
+
function invokedAsCli() {
|
|
2045
|
+
const entry = process.argv[1];
|
|
2046
|
+
if (entry === void 0) return false;
|
|
2047
|
+
try {
|
|
2048
|
+
return import.meta.url === pathToFileURL(entry).href || import.meta.url === pathToFileURL(realpathSync(entry)).href;
|
|
2049
|
+
} catch {
|
|
2050
|
+
return false;
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
if (invokedAsCli()) {
|
|
114
2054
|
main().then((manifest) => {
|
|
115
2055
|
if (manifest.failed.length > 0) {
|
|
116
2056
|
process.stderr.write(
|
|
@@ -125,7 +2065,9 @@ if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv
|
|
|
125
2065
|
process.exit(1);
|
|
126
2066
|
});
|
|
127
2067
|
}
|
|
128
|
-
|
|
129
|
-
|
|
2068
|
+
export {
|
|
2069
|
+
fsSink,
|
|
2070
|
+
main,
|
|
2071
|
+
parseConfig
|
|
2072
|
+
};
|
|
130
2073
|
//# sourceMappingURL=export-segments.js.map
|
|
131
|
-
//# sourceMappingURL=export-segments.js.map
|