@cloudbitmaps/core 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -36
- package/dist/{chunk-BUPDGZRR.js → chunk-5OMC5U7U.js} +87 -40
- package/dist/chunk-5OMC5U7U.js.map +7 -0
- package/dist/core/audit.d.ts +15 -14
- package/dist/core/budget.d.ts +5 -3
- package/dist/core/codec.d.ts +18 -10
- package/dist/core/consistency.d.ts +3 -3
- package/dist/core/cooperative.d.ts +9 -9
- package/dist/core/cost.d.ts +167 -15
- package/dist/core/crbm/format.d.ts +7 -9
- package/dist/core/crbm/reader.d.ts +18 -1
- package/dist/core/crbm/writer.d.ts +2 -2
- package/dist/core/crbm-storage-source.d.ts +150 -60
- package/dist/core/determinism.d.ts +1 -1
- package/dist/core/discover.d.ts +7 -5
- package/dist/core/due-index.d.ts +3 -5
- package/dist/core/engine.d.ts +55 -13
- package/dist/core/erase-id.d.ts +76 -41
- package/dist/core/erasure.d.ts +22 -16
- package/dist/core/errors.d.ts +13 -20
- package/dist/core/generation-gc.d.ts +9 -5
- package/dist/core/keys.d.ts +2 -2
- package/dist/core/load.d.ts +28 -9
- package/dist/core/metrics.d.ts +8 -4
- package/dist/core/name-codec.d.ts +21 -31
- package/dist/core/pinned-storage-source.d.ts +29 -8
- package/dist/core/ports.d.ts +142 -59
- package/dist/core/reader-defaults.d.ts +18 -0
- package/dist/core/registry-scan.d.ts +15 -6
- package/dist/core/retention-sweep.d.ts +26 -18
- package/dist/core/retry.d.ts +8 -4
- package/dist/core/rollback.d.ts +11 -7
- package/dist/core/validate.d.ts +27 -3
- package/dist/driver-kit.d.ts +2 -3
- package/dist/driver-kit.js +13 -52
- package/dist/driver-kit.js.map +3 -3
- package/dist/drivers/_shared/keys.d.ts +4 -5
- package/dist/drivers/_shared/object-registry.d.ts +10 -5
- package/dist/drivers/_shared/registry.d.ts +9 -5
- package/dist/drivers/backends.d.ts +10 -16
- package/dist/drivers/localfs/fs-util.d.ts +3 -2
- package/dist/drivers/localfs/registry.d.ts +4 -4
- package/dist/drivers/memory.d.ts +2 -10
- package/dist/drivers/retry/retrying-chunk-source.d.ts +59 -0
- package/dist/export/index.d.ts +7 -6
- package/dist/index.d.ts +15 -24
- package/dist/index.js +1014 -373
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +1 -1
- package/package.json +1 -1
- package/dist/chunk-BUPDGZRR.js.map +0 -7
- package/dist/drivers/_shared/aws-errors.d.ts +0 -18
- package/dist/drivers/retry/retrying-drivers.d.ts +0 -90
package/dist/index.js
CHANGED
|
@@ -7,14 +7,13 @@ import {
|
|
|
7
7
|
KeyUnavailableError,
|
|
8
8
|
MAX_NAME_LENGTH,
|
|
9
9
|
NotFoundError,
|
|
10
|
-
|
|
10
|
+
RESERVED_NAMESPACE_PREFIX,
|
|
11
11
|
TransientError,
|
|
12
12
|
UnsupportedError,
|
|
13
13
|
ValidationError,
|
|
14
14
|
WriteConflictError,
|
|
15
15
|
applyRegistryPatch,
|
|
16
16
|
brandAsBackend,
|
|
17
|
-
createBackend,
|
|
18
17
|
decodeNameFromPath,
|
|
19
18
|
encodeNameForKey,
|
|
20
19
|
encodeNameForPath,
|
|
@@ -22,6 +21,7 @@ import {
|
|
|
22
21
|
isCloudRoaringError,
|
|
23
22
|
isIntegrityError,
|
|
24
23
|
isNotFoundError,
|
|
24
|
+
isReservedNamespace,
|
|
25
25
|
isStorageBackend,
|
|
26
26
|
isTransientError,
|
|
27
27
|
isValidationError,
|
|
@@ -35,8 +35,10 @@ import {
|
|
|
35
35
|
validateChunkRef,
|
|
36
36
|
validateNewRegistryRecord,
|
|
37
37
|
validateRegistryPatch,
|
|
38
|
-
validateSegmentRef
|
|
39
|
-
|
|
38
|
+
validateSegmentRef,
|
|
39
|
+
validateUserNamespace,
|
|
40
|
+
validateUserRef
|
|
41
|
+
} from "./chunk-5OMC5U7U.js";
|
|
40
42
|
|
|
41
43
|
// src/core/bit-route.ts
|
|
42
44
|
var U32_MAX = 4294967295;
|
|
@@ -202,6 +204,57 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
202
204
|
var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
|
|
203
205
|
var DEFAULT_INTERSECT_CONCURRENCY = 8;
|
|
204
206
|
var ZERO_CLOCK = { now: () => 0 };
|
|
207
|
+
function windowOf(range) {
|
|
208
|
+
const after = range?.after;
|
|
209
|
+
const through = range?.through;
|
|
210
|
+
if (after === void 0 && through === void 0) return null;
|
|
211
|
+
for (const [name, bound] of [
|
|
212
|
+
["after", after],
|
|
213
|
+
["through", through]
|
|
214
|
+
]) {
|
|
215
|
+
if (bound !== void 0 && (!Number.isInteger(bound) || bound < 0 || bound > U32_MAX)) {
|
|
216
|
+
const got = typeof bound === "number" ? String(bound) : `a ${typeof bound}`;
|
|
217
|
+
throw new ValidationError(`${name} must be an integer in 0..${U32_MAX}; got ${got}`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
const lo = after === void 0 ? 0 : after + 1;
|
|
221
|
+
const hi = through ?? U32_MAX;
|
|
222
|
+
if (lo > hi) return "empty";
|
|
223
|
+
return {
|
|
224
|
+
loKey: Math.floor(lo / CHUNK_COUNT),
|
|
225
|
+
loRem: lo % CHUNK_COUNT,
|
|
226
|
+
hiKey: Math.floor(hi / CHUNK_COUNT),
|
|
227
|
+
hiRem: hi % CHUNK_COUNT
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
var WHOLE_ID_SPACE = {
|
|
231
|
+
loKey: 0,
|
|
232
|
+
loRem: 0,
|
|
233
|
+
hiKey: CHUNK_COUNT - 1,
|
|
234
|
+
hiRem: MAX_REMAINDER
|
|
235
|
+
};
|
|
236
|
+
var isEdge = (chunkKey, w) => chunkKey === w.loKey && w.loRem > 0 || chunkKey === w.hiKey && w.hiRem < MAX_REMAINDER;
|
|
237
|
+
function* edgeIds(chunk, chunkKey, w) {
|
|
238
|
+
const from = chunkKey === w.loKey ? w.loRem : 0;
|
|
239
|
+
const to = chunkKey === w.hiKey ? w.hiRem : MAX_REMAINDER;
|
|
240
|
+
for (const remainder of chunk) {
|
|
241
|
+
if (remainder > to) return;
|
|
242
|
+
if (remainder >= from) yield joinId(chunkKey, remainder);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
function keysWithin(keys, w) {
|
|
246
|
+
const firstAtLeast = (key) => {
|
|
247
|
+
let lo = 0;
|
|
248
|
+
let hi = keys.length;
|
|
249
|
+
while (lo < hi) {
|
|
250
|
+
const mid = lo + hi >>> 1;
|
|
251
|
+
if (keys[mid] < key) lo = mid + 1;
|
|
252
|
+
else hi = mid;
|
|
253
|
+
}
|
|
254
|
+
return lo;
|
|
255
|
+
};
|
|
256
|
+
return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
|
|
257
|
+
}
|
|
205
258
|
var SegmentEngine = class {
|
|
206
259
|
storage;
|
|
207
260
|
cache;
|
|
@@ -232,6 +285,10 @@ var SegmentEngine = class {
|
|
|
232
285
|
* Cardinality. When the Storage source can serve per-chunk cardinality from its `.crbm` index (the `.crbm`
|
|
233
286
|
* source can), the count is summed from the index with **zero payload reads**. A source without that
|
|
234
287
|
* capability (the in-memory source) falls back to fetching every chunk.
|
|
288
|
+
*
|
|
289
|
+
* The index sum is trusted, not confirmed: the reader checked the index for internal consistency when it opened
|
|
290
|
+
* the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
|
|
291
|
+
* combines decode the payloads.
|
|
235
292
|
*/
|
|
236
293
|
async count(seg) {
|
|
237
294
|
const cardinalities = this.storage.cardinalities ? await this.storage.cardinalities(seg) : null;
|
|
@@ -256,12 +313,26 @@ var SegmentEngine = class {
|
|
|
256
313
|
get supportsStorageSize() {
|
|
257
314
|
return typeof this.storage.sizeOf === "function";
|
|
258
315
|
}
|
|
316
|
+
/** How often the Storage source re-reads a pointer while reading (cost reporting); undefined if it does not say. */
|
|
317
|
+
get pointerRefreshMs() {
|
|
318
|
+
return this.storage.pointerRefreshMs;
|
|
319
|
+
}
|
|
259
320
|
/** Grounded Storage size of a segment's current generation (cost reporting), or null if it has no generation. */
|
|
260
321
|
segmentSize(seg) {
|
|
261
322
|
return this.storage.sizeOf ? this.storage.sizeOf(seg) : Promise.resolve(null);
|
|
262
323
|
}
|
|
263
|
-
/**
|
|
264
|
-
|
|
324
|
+
/**
|
|
325
|
+
* Every id, ascending, one chunk at a time; with `range`, only the ids in `(after, through]`, fetching only the
|
|
326
|
+
* chunks the range overlaps (see {@link IdRange}).
|
|
327
|
+
*
|
|
328
|
+
* Two generators, not one with a branch: a second `yield` site in the full read's generator grows its frame, and
|
|
329
|
+
* every id of a read that asked for no range paid for it (measured at about 5% per id). The full read keeps the
|
|
330
|
+
* loop it always had.
|
|
331
|
+
*/
|
|
332
|
+
iterate(seg, range) {
|
|
333
|
+
return range === void 0 ? this.iterateAll(seg) : this.iterateRange(seg, range);
|
|
334
|
+
}
|
|
335
|
+
async *iterateAll(seg) {
|
|
265
336
|
const chunkKeys = await this.chunkKeys(seg);
|
|
266
337
|
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
267
338
|
const gen = await this.cacheVersion(seg);
|
|
@@ -271,6 +342,22 @@ var SegmentEngine = class {
|
|
|
271
342
|
for (const remainder of chunk) yield joinId(chunkKey, remainder);
|
|
272
343
|
}
|
|
273
344
|
}
|
|
345
|
+
async *iterateRange(seg, range) {
|
|
346
|
+
const w = windowOf(range) ?? WHOLE_ID_SPACE;
|
|
347
|
+
if (w === "empty") return;
|
|
348
|
+
const chunkKeys = keysWithin(await this.chunkKeys(seg), w);
|
|
349
|
+
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
350
|
+
const gen = await this.cacheVersion(seg);
|
|
351
|
+
for (const chunkKey of chunkKeys) {
|
|
352
|
+
const chunk = await this.storageChunk({ ...seg, chunkKey }, gen);
|
|
353
|
+
if (chunk === null) continue;
|
|
354
|
+
if (isEdge(chunkKey, w)) {
|
|
355
|
+
for (const id of edgeIds(chunk, chunkKey, w)) yield id;
|
|
356
|
+
} else {
|
|
357
|
+
for (const remainder of chunk) yield joinId(chunkKey, remainder);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
274
361
|
/**
|
|
275
362
|
* `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
|
|
276
363
|
*
|
|
@@ -294,11 +381,13 @@ var SegmentEngine = class {
|
|
|
294
381
|
* Generation-consistent within the call (normal case): each operand's current generation is resolved **once**
|
|
295
382
|
* up front (before the fan-out) and threaded into every chunk read, so a concurrent load can't corrupt or tear
|
|
296
383
|
* the result — every chunk read is a whole, checksum-verified, immutable generation. The edge a *long* call can
|
|
297
|
-
* hit: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
|
|
298
|
-
* chunks may re-resolve forward to the newer generation (a generation hop within one long
|
|
299
|
-
* never crashes or returns a torn object, but may mix generations.
|
|
300
|
-
*
|
|
301
|
-
*
|
|
384
|
+
* hit is invariant 3's: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
|
|
385
|
+
* operand's not-yet-read chunks may re-resolve forward to the newer generation (a generation hop within one long
|
|
386
|
+
* call) — the call never crashes or returns a torn object, but may mix generations. Three things hop it without
|
|
387
|
+
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
|
|
388
|
+
* (`maxOpenSegments`), whose re-read re-resolves fresh; a sweep deleting the generation it was reading, which
|
|
389
|
+
* heals the read forward; and an invalidation, which this store's own `load`, `rollback` and `eraseSubject` make
|
|
390
|
+
* and `invalidate()` makes on request. Still whole/immutable per read, never torn.
|
|
302
391
|
*/
|
|
303
392
|
intersect(segs, options) {
|
|
304
393
|
return this.combine(segs, options?.exclude ?? [], "all", "intersect", options);
|
|
@@ -337,10 +426,13 @@ var SegmentEngine = class {
|
|
|
337
426
|
);
|
|
338
427
|
}
|
|
339
428
|
const budget = resolvePerOpBudget(options?.budget, this.budget);
|
|
429
|
+
const w = windowOf(options);
|
|
430
|
+
if (w === "empty") return;
|
|
340
431
|
const extract = async (seg) => {
|
|
341
|
-
const
|
|
432
|
+
const all = await this.chunkKeys(seg);
|
|
342
433
|
const gen = await this.cacheVersion(seg);
|
|
343
|
-
|
|
434
|
+
const keys = w === null ? all : keysWithin(all, w);
|
|
435
|
+
return { seg, keys: new Set(keys), chunkless: all.length === 0, gen };
|
|
344
436
|
};
|
|
345
437
|
const [operands, excludes] = await Promise.all([
|
|
346
438
|
Promise.all(segs.map(extract)),
|
|
@@ -392,7 +484,11 @@ var SegmentEngine = class {
|
|
|
392
484
|
if (next < common.length) inFlight.push(startAt(common[next++]));
|
|
393
485
|
if (slot.error !== void 0) throw slot.error;
|
|
394
486
|
if (slot.result && !slot.result.isEmpty) {
|
|
395
|
-
|
|
487
|
+
if (w !== null && isEdge(slot.key, w)) {
|
|
488
|
+
for (const id of edgeIds(slot.result, slot.key, w)) yield id;
|
|
489
|
+
} else {
|
|
490
|
+
for (const remainder of slot.result) yield joinId(slot.key, remainder);
|
|
491
|
+
}
|
|
396
492
|
}
|
|
397
493
|
}
|
|
398
494
|
}
|
|
@@ -442,7 +538,13 @@ var SegmentEngine = class {
|
|
|
442
538
|
async chunkKeys(seg) {
|
|
443
539
|
const keys = [...await this.storage.listChunkKeys(seg)];
|
|
444
540
|
for (const k of keys) this.assertChunkKeyInRange(k);
|
|
445
|
-
|
|
541
|
+
keys.sort((a, b) => a - b);
|
|
542
|
+
for (let i = 1; i < keys.length; i++) {
|
|
543
|
+
if (keys[i] === keys[i - 1]) {
|
|
544
|
+
throw new IntegrityError(`chunk key from a tier is listed twice: ${keys[i]}`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
return keys;
|
|
446
548
|
}
|
|
447
549
|
/**
|
|
448
550
|
* Tier-derived keys are untrusted (invariant 5) — fail fast on a corrupt/out-of-range key rather than
|
|
@@ -465,6 +567,11 @@ var SegmentEngine = class {
|
|
|
465
567
|
*
|
|
466
568
|
* Costs one `maximum()` per chunk, not per id — `maximum` is optional on the codec seam precisely so a codec
|
|
467
569
|
* that cannot answer in O(1) opts out instead of making the read path walk every value.
|
|
570
|
+
*
|
|
571
|
+
* One value stands for all of them only because the decode is structurally checked. Roaring answers `maximum()`
|
|
572
|
+
* from its last container, so a payload listing its containers out of order would report the wrong one's
|
|
573
|
+
* largest value and pass here while holding values above 65,535. The codec's `safeDeserialize` refuses that
|
|
574
|
+
* payload, and every other shape that would make this answer wrong, before it reaches this check.
|
|
468
575
|
*/
|
|
469
576
|
assertChunkPayloadInRange(bitmap, chunkKey) {
|
|
470
577
|
const max = bitmap.maximum?.();
|
|
@@ -491,11 +598,12 @@ var SegmentEngine = class {
|
|
|
491
598
|
* **Costs nothing on a normal combine.** `exists` is consulted only for an operand that resolved to zero
|
|
492
599
|
* chunks — a segment with data is self-evidently registered — so the check is a registry read exactly when
|
|
493
600
|
* a combine was about to do something suspicious, and no reads at all otherwise. A source that cannot answer
|
|
494
|
-
* `exists` (it has no notion of registration) skips the check rather than guessing.
|
|
601
|
+
* `exists` (it has no notion of registration) skips the check rather than guessing. An operand is judged by
|
|
602
|
+
* all its chunks, not those inside a range: one with nothing in range but data elsewhere is not suspicious.
|
|
495
603
|
*/
|
|
496
604
|
async refuseAbsentOperands(resolved, allow, op) {
|
|
497
605
|
if (allow === true || this.storage.exists === void 0) return;
|
|
498
|
-
const empty = resolved.filter((o) => o.
|
|
606
|
+
const empty = resolved.filter((o) => o.chunkless);
|
|
499
607
|
if (empty.length === 0) return;
|
|
500
608
|
const checked = await Promise.all(
|
|
501
609
|
empty.map(async (o) => ({ seg: o.seg, exists: await this.storage.exists(o.seg) }))
|
|
@@ -531,8 +639,8 @@ var SegmentEngine = class {
|
|
|
531
639
|
* The cache-key component identifying **which bytes** this op will read: the source's `currentVersion` when
|
|
532
640
|
* it has one, else the generation alone.
|
|
533
641
|
*
|
|
534
|
-
* The generation alone is not an identity.
|
|
535
|
-
* bucket emptied, so a retired-and-re-created name serves different data at the same `currentGen` — and a
|
|
642
|
+
* The generation alone is not an identity. A load's generation number restarts at 0 once a row is purged
|
|
643
|
+
* and the bucket emptied, so a retired-and-re-created name serves different data at the same `currentGen` — and a
|
|
536
644
|
* decoded chunk cached under `(segment, chunk, 0)` is handed straight back to the new incarnation. That is an
|
|
537
645
|
* erased id reappearing with no read to intercept it, which is why this is keyed on the version rather than
|
|
538
646
|
* the number.
|
|
@@ -547,7 +655,7 @@ var SegmentEngine = class {
|
|
|
547
655
|
* keyed by it, so a load that advances the generation misses the cache and re-reads the new bytes instead of
|
|
548
656
|
* serving a stale decoded chunk (an erased id can't resurrect from a cached superseded chunk). `gen === null`
|
|
549
657
|
* ⇒ the source reports no current generation ⇒ no storage bytes for any chunk, so skip the fetch entirely.
|
|
550
|
-
* `gen === undefined` ⇒ the source can't report a generation (
|
|
658
|
+
* `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
|
|
551
659
|
* generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
|
|
552
660
|
*
|
|
553
661
|
* The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
|
|
@@ -721,18 +829,58 @@ var BoundedLru = class {
|
|
|
721
829
|
}
|
|
722
830
|
};
|
|
723
831
|
|
|
832
|
+
// src/core/reader-defaults.ts
|
|
833
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
834
|
+
var REFRESH_RETRY_MS = 500;
|
|
835
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
836
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
837
|
+
|
|
724
838
|
// src/core/cost.ts
|
|
725
|
-
|
|
839
|
+
function deepFreeze(value) {
|
|
840
|
+
if (value !== null && typeof value === "object" && !Object.isFrozen(value)) {
|
|
841
|
+
Object.freeze(value);
|
|
842
|
+
for (const inner of Object.values(value)) deepFreeze(inner);
|
|
843
|
+
}
|
|
844
|
+
return value;
|
|
845
|
+
}
|
|
846
|
+
var ELASTICACHE_REDIS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
847
|
+
source: "ElastiCache for Redis OSS, us-east-1 on-demand, AWS price list 20260914063714",
|
|
848
|
+
replicasPerShard: 2,
|
|
849
|
+
reservedMemoryFraction: 0.25,
|
|
850
|
+
nodeTypes: [
|
|
851
|
+
{ name: "cache.t4g.micro", memoryGiB: 0.5, hourlyUSD: 0.016, maxShards: 1 },
|
|
852
|
+
{ name: "cache.t4g.small", memoryGiB: 1.37, hourlyUSD: 0.032, maxShards: 1 },
|
|
853
|
+
{ name: "cache.t4g.medium", memoryGiB: 3.09, hourlyUSD: 0.065, maxShards: 1 },
|
|
854
|
+
{ name: "cache.m6g.large", memoryGiB: 6.38, hourlyUSD: 0.149 },
|
|
855
|
+
{ name: "cache.r6g.large", memoryGiB: 13.07, hourlyUSD: 0.206 },
|
|
856
|
+
{ name: "cache.r6g.xlarge", memoryGiB: 26.32, hourlyUSD: 0.411 },
|
|
857
|
+
{ name: "cache.r6g.2xlarge", memoryGiB: 52.82, hourlyUSD: 0.821 },
|
|
858
|
+
{ name: "cache.r6g.4xlarge", memoryGiB: 105.81, hourlyUSD: 1.642 },
|
|
859
|
+
{ name: "cache.r6g.8xlarge", memoryGiB: 209.55, hourlyUSD: 3.284 },
|
|
860
|
+
{ name: "cache.r6g.12xlarge", memoryGiB: 317.77, hourlyUSD: 4.925 },
|
|
861
|
+
{ name: "cache.r6g.16xlarge", memoryGiB: 419.09, hourlyUSD: 6.567 },
|
|
862
|
+
{ name: "cache.r6gd.xlarge", memoryGiB: 26.32, ssdGiB: 99.33, hourlyUSD: 0.781 },
|
|
863
|
+
{ name: "cache.r6gd.2xlarge", memoryGiB: 52.82, ssdGiB: 199.07, hourlyUSD: 1.56 },
|
|
864
|
+
{ name: "cache.r6gd.4xlarge", memoryGiB: 105.81, ssdGiB: 398.14, hourlyUSD: 3.12 },
|
|
865
|
+
{ name: "cache.r6gd.8xlarge", memoryGiB: 209.55, ssdGiB: 796.28, hourlyUSD: 6.24 },
|
|
866
|
+
{ name: "cache.r6gd.12xlarge", memoryGiB: 317.77, ssdGiB: 1194.42, hourlyUSD: 9.358 },
|
|
867
|
+
{ name: "cache.r6gd.16xlarge", memoryGiB: 419.09, ssdGiB: 1592.56, hourlyUSD: 12.477 }
|
|
868
|
+
]
|
|
869
|
+
});
|
|
870
|
+
var ONE_REDIS_HA_CLUSTER = deepFreeze({ monthlyUSD: 346 });
|
|
871
|
+
var AWS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
726
872
|
name: "aws-us-east-1-ondemand",
|
|
727
873
|
storage: { getPerMillion: 0.4, putPerMillion: 5, storagePerGiBMonth: 0.023 },
|
|
728
|
-
redis: {
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
var
|
|
732
|
-
var SECONDS_PER_MONTH = 730 * 3600;
|
|
874
|
+
redis: { sizedToData: ELASTICACHE_REDIS_US_EAST_1_ONDEMAND }
|
|
875
|
+
});
|
|
876
|
+
var HOURS_PER_MONTH = 730;
|
|
877
|
+
var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
|
|
733
878
|
var GIB = 1024 ** 3;
|
|
879
|
+
var SIZED_READS_PER_COLD_OPERAND = 2;
|
|
880
|
+
var STORE_LOAD_PUT_CLASS = 3;
|
|
881
|
+
var STORE_LOAD_SIZED_READS = 9;
|
|
734
882
|
function requireFiniteNonNeg(n, field) {
|
|
735
|
-
if (!Number.isFinite(n) || n < 0) {
|
|
883
|
+
if (n === void 0 || !Number.isFinite(n) || n < 0) {
|
|
736
884
|
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
737
885
|
}
|
|
738
886
|
return n;
|
|
@@ -740,6 +888,147 @@ function requireFiniteNonNeg(n, field) {
|
|
|
740
888
|
function clamp01(n) {
|
|
741
889
|
return n < 0 ? 0 : n > 1 ? 1 : n;
|
|
742
890
|
}
|
|
891
|
+
var TIE_USD = 1e-6;
|
|
892
|
+
function usableGiB(sizing, nodeType) {
|
|
893
|
+
return nodeType.memoryGiB * (1 - sizing.reservedMemoryFraction) + (nodeType.ssdGiB ?? 0);
|
|
894
|
+
}
|
|
895
|
+
function validateSizing(sizing) {
|
|
896
|
+
const at = "pricing.redis.sizedToData";
|
|
897
|
+
if (sizing === null || typeof sizing !== "object") {
|
|
898
|
+
throw new ValidationError(`${at} must be a RedisSizing`);
|
|
899
|
+
}
|
|
900
|
+
if (typeof sizing.source !== "string" || sizing.source === "") {
|
|
901
|
+
throw new ValidationError(`${at}.source must say where the prices come from`);
|
|
902
|
+
}
|
|
903
|
+
if (!Array.isArray(sizing.nodeTypes) || sizing.nodeTypes.length === 0) {
|
|
904
|
+
throw new ValidationError(`${at}.nodeTypes must list at least one node type`);
|
|
905
|
+
}
|
|
906
|
+
if (!Number.isInteger(sizing.replicasPerShard) || sizing.replicasPerShard < 0) {
|
|
907
|
+
throw new ValidationError(
|
|
908
|
+
`${at}.replicasPerShard must be an integer >= 0; got ${sizing.replicasPerShard}`
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
const reserved = requireFiniteNonNeg(
|
|
912
|
+
sizing.reservedMemoryFraction,
|
|
913
|
+
`${at}.reservedMemoryFraction`
|
|
914
|
+
);
|
|
915
|
+
if (reserved >= 1) {
|
|
916
|
+
throw new ValidationError(`${at}.reservedMemoryFraction must be below 1; got ${reserved}`);
|
|
917
|
+
}
|
|
918
|
+
const names = /* @__PURE__ */ new Set();
|
|
919
|
+
for (const n of sizing.nodeTypes) {
|
|
920
|
+
if (n === null || typeof n !== "object" || typeof n.name !== "string" || n.name === "") {
|
|
921
|
+
throw new ValidationError(`${at}.nodeTypes: every node type needs a name`);
|
|
922
|
+
}
|
|
923
|
+
const row = `${at}.nodeTypes[${JSON.stringify(n.name)}]`;
|
|
924
|
+
if (names.has(n.name)) throw new ValidationError(`${row} is listed twice`);
|
|
925
|
+
names.add(n.name);
|
|
926
|
+
if (!(requireFiniteNonNeg(n.memoryGiB, `${row}.memoryGiB`) > 0)) {
|
|
927
|
+
throw new ValidationError(`${row}.memoryGiB must be above 0`);
|
|
928
|
+
}
|
|
929
|
+
requireFiniteNonNeg(n.ssdGiB ?? 0, `${row}.ssdGiB`);
|
|
930
|
+
requireFiniteNonNeg(n.hourlyUSD, `${row}.hourlyUSD`);
|
|
931
|
+
if (n.maxShards !== void 0 && (!Number.isInteger(n.maxShards) || n.maxShards < 1)) {
|
|
932
|
+
throw new ValidationError(`${row}.maxShards must be an integer >= 1; got ${n.maxShards}`);
|
|
933
|
+
}
|
|
934
|
+
if (!(usableGiB(sizing, n) > 0)) {
|
|
935
|
+
throw new ValidationError(
|
|
936
|
+
`${row} leaves no memory for data once ${reserved} of it is reserved`
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
function cheapestCluster(sizing, dataGiB) {
|
|
942
|
+
const fits = [];
|
|
943
|
+
for (const nodeType of sizing.nodeTypes) {
|
|
944
|
+
const usable = usableGiB(sizing, nodeType);
|
|
945
|
+
let shards = Math.max(1, Math.ceil(dataGiB / usable));
|
|
946
|
+
if (shards > 1 && (shards - 1) * usable >= dataGiB) shards -= 1;
|
|
947
|
+
if (nodeType.maxShards !== void 0 && shards > nodeType.maxShards) continue;
|
|
948
|
+
const nodes = shards * (1 + sizing.replicasPerShard);
|
|
949
|
+
fits.push({
|
|
950
|
+
nodeType,
|
|
951
|
+
shards,
|
|
952
|
+
nodes,
|
|
953
|
+
monthlyUSD: nodes * nodeType.hourlyUSD * HOURS_PER_MONTH
|
|
954
|
+
});
|
|
955
|
+
}
|
|
956
|
+
const size = bytesWords(dataGiB * GIB);
|
|
957
|
+
if (fits.length === 0) {
|
|
958
|
+
throw new ValidationError(
|
|
959
|
+
`no node type in pricing.redis.sizedToData.nodeTypes holds ${size} within its maxShards`
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
const cheapest = fits.reduce((m, c) => Math.min(m, c.monthlyUSD), Infinity);
|
|
963
|
+
const [best] = fits.filter((c) => c.monthlyUSD <= cheapest + TIE_USD).sort(
|
|
964
|
+
(a, b) => a.nodes - b.nodes || a.monthlyUSD - b.monthlyUSD || (a.nodeType.name < b.nodeType.name ? -1 : a.nodeType.name > b.nodeType.name ? 1 : 0)
|
|
965
|
+
);
|
|
966
|
+
if (best === void 0 || !Number.isFinite(best.monthlyUSD)) {
|
|
967
|
+
throw new ValidationError(
|
|
968
|
+
`pricing.redis.sizedToData prices ${size} at no finite cost; check its node types`
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
return best;
|
|
972
|
+
}
|
|
973
|
+
function snapshotSizing(sizing) {
|
|
974
|
+
if (sizing === null || typeof sizing !== "object") return sizing;
|
|
975
|
+
const { source, nodeTypes, replicasPerShard, reservedMemoryFraction } = sizing;
|
|
976
|
+
const row = (n) => {
|
|
977
|
+
if (n === null || typeof n !== "object") return n;
|
|
978
|
+
const { name, memoryGiB, ssdGiB, hourlyUSD, maxShards } = n;
|
|
979
|
+
return {
|
|
980
|
+
name,
|
|
981
|
+
memoryGiB,
|
|
982
|
+
hourlyUSD,
|
|
983
|
+
...ssdGiB === void 0 ? {} : { ssdGiB },
|
|
984
|
+
...maxShards === void 0 ? {} : { maxShards }
|
|
985
|
+
};
|
|
986
|
+
};
|
|
987
|
+
return {
|
|
988
|
+
source,
|
|
989
|
+
replicasPerShard,
|
|
990
|
+
reservedMemoryFraction,
|
|
991
|
+
nodeTypes: Array.isArray(nodeTypes) ? Array.from(nodeTypes, row) : nodeTypes
|
|
992
|
+
};
|
|
993
|
+
}
|
|
994
|
+
function resolveBaseline(redis, storedGiB) {
|
|
995
|
+
if (redis === null || typeof redis !== "object") {
|
|
996
|
+
throw new ValidationError("pricing.redis must be { monthlyUSD } or { sizedToData }");
|
|
997
|
+
}
|
|
998
|
+
const { monthlyUSD, sizedToData } = redis;
|
|
999
|
+
const fixed = monthlyUSD !== void 0;
|
|
1000
|
+
const sized = sizedToData !== void 0;
|
|
1001
|
+
if (fixed === sized) {
|
|
1002
|
+
throw new ValidationError(
|
|
1003
|
+
"pricing.redis must be exactly one of { monthlyUSD } and { sizedToData }" + (fixed ? ", not both: spreading AWS_US_EAST_1_ONDEMAND.redis keeps its sizedToData, so pass redis: { monthlyUSD } alone" : "")
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
if (sized) {
|
|
1007
|
+
const sizing = snapshotSizing(sizedToData);
|
|
1008
|
+
validateSizing(sizing);
|
|
1009
|
+
const cluster = cheapestCluster(sizing, storedGiB);
|
|
1010
|
+
return { monthlyUSD: cluster.monthlyUSD, sized: { sizing, cluster } };
|
|
1011
|
+
}
|
|
1012
|
+
return { monthlyUSD: requireFiniteNonNeg(monthlyUSD, "pricing.redis.monthlyUSD") };
|
|
1013
|
+
}
|
|
1014
|
+
function bytesWords(bytes) {
|
|
1015
|
+
const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
|
|
1016
|
+
let value = bytes;
|
|
1017
|
+
let unit = 0;
|
|
1018
|
+
const shown = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
|
|
1019
|
+
while (unit < units.length - 1 && shown(value, unit) >= 1024) {
|
|
1020
|
+
value /= 1024;
|
|
1021
|
+
unit += 1;
|
|
1022
|
+
}
|
|
1023
|
+
if (unit > 0) return `${value.toFixed(2)} ${units[unit]}`;
|
|
1024
|
+
const n = Math.round(value);
|
|
1025
|
+
return `${n} byte${n === 1 ? "" : "s"}`;
|
|
1026
|
+
}
|
|
1027
|
+
function clusterWords(c) {
|
|
1028
|
+
const count = (n) => n.toLocaleString("en-US");
|
|
1029
|
+
const nodes = `${count(c.nodes)} ${c.nodeType.name} node${c.nodes === 1 ? "" : "s"}`;
|
|
1030
|
+
return c.shards === 1 ? `1 shard of ${nodes}` : `${count(c.shards)} shards, ${nodes}`;
|
|
1031
|
+
}
|
|
743
1032
|
function sizingBytes(spec) {
|
|
744
1033
|
if (spec.sizeBytes !== void 0) return requireFiniteNonNeg(spec.sizeBytes, "segment.sizeBytes");
|
|
745
1034
|
if (spec.cardinality !== void 0) {
|
|
@@ -754,8 +1043,14 @@ function buildReport(input) {
|
|
|
754
1043
|
requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
|
|
755
1044
|
requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
|
|
756
1045
|
requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
|
|
757
|
-
requireFiniteNonNeg(
|
|
1046
|
+
const sizedRead = requireFiniteNonNeg(
|
|
1047
|
+
storage.requestsPerSizedRead ?? 1,
|
|
1048
|
+
"pricing.storage.requestsPerSizedRead"
|
|
1049
|
+
);
|
|
758
1050
|
const storageBytes = requireFiniteNonNeg(input.storageBytes, "storageBytes");
|
|
1051
|
+
const storedGiB = storageBytes / GIB;
|
|
1052
|
+
const baseline = resolveBaseline(redis, storedGiB);
|
|
1053
|
+
const baselineUSD = baseline.monthlyUSD;
|
|
759
1054
|
const cacheHitRate = clamp01(
|
|
760
1055
|
requireFiniteNonNeg(input.workload.cacheHitRate ?? 0, "cacheHitRate")
|
|
761
1056
|
);
|
|
@@ -773,36 +1068,92 @@ function buildReport(input) {
|
|
|
773
1068
|
input.workload.requestsPerLoad ?? 1,
|
|
774
1069
|
"requestsPerLoad"
|
|
775
1070
|
);
|
|
1071
|
+
const operandsPerIntersect = requireFiniteNonNeg(
|
|
1072
|
+
input.workload.operandsPerIntersect ?? 2,
|
|
1073
|
+
"operandsPerIntersect"
|
|
1074
|
+
);
|
|
1075
|
+
if (operandsPerIntersect < 1) {
|
|
1076
|
+
throw new ValidationError(
|
|
1077
|
+
`operandsPerIntersect must be at least 1, since an intersection reads its operands; got ${operandsPerIntersect}`
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
const hotSegments = requireFiniteNonNeg(input.workload.hotSegments ?? 0, "hotSegments");
|
|
1081
|
+
const readerProcesses = requireFiniteNonNeg(
|
|
1082
|
+
input.workload.readerProcesses ?? 1,
|
|
1083
|
+
"readerProcesses"
|
|
1084
|
+
);
|
|
1085
|
+
if (readerProcesses < 1) {
|
|
1086
|
+
throw new ValidationError(`readerProcesses must be at least 1; got ${readerProcesses}`);
|
|
1087
|
+
}
|
|
1088
|
+
const genTtlMs = requireFiniteNonNeg(
|
|
1089
|
+
input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
|
|
1090
|
+
"genTtlMs"
|
|
1091
|
+
);
|
|
776
1092
|
const storageGetUSD = storage.getPerMillion / 1e6;
|
|
777
1093
|
const putUSD = storage.putPerMillion / 1e6;
|
|
778
1094
|
const missFraction = 1 - cacheHitRate;
|
|
779
1095
|
const readMisses = readsPerSec * S * missFraction;
|
|
780
1096
|
const intersects = intersectsPerSec * S;
|
|
781
|
-
const
|
|
1097
|
+
const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
|
|
1098
|
+
const storageUSD = storedGiB * storage.storagePerGiBMonth;
|
|
782
1099
|
const readsUSD = readMisses * storageGetUSD;
|
|
783
|
-
const
|
|
784
|
-
const
|
|
785
|
-
const
|
|
786
|
-
const
|
|
1100
|
+
const intersectGets = chunksPerIntersect + SIZED_READS_PER_COLD_OPERAND * operandsPerIntersect * sizedRead;
|
|
1101
|
+
const intersectsUSD = intersects * intersectGets * storageGetUSD;
|
|
1102
|
+
const loadGets = STORE_LOAD_SIZED_READS * sizedRead;
|
|
1103
|
+
const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
|
|
1104
|
+
const refreshUSD = refreshes * sizedRead * storageGetUSD;
|
|
1105
|
+
const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD;
|
|
1106
|
+
const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
|
|
787
1107
|
const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
|
|
788
|
-
const verdict = total <=
|
|
789
|
-
const dominant = Math.max(readsUSD, intersectsUSD, storageUSD, loadsUSD);
|
|
1108
|
+
const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
|
|
1109
|
+
const dominant = Math.max(readsUSD, intersectsUSD, storageUSD, loadsUSD, refreshUSD);
|
|
790
1110
|
let driver = "storage";
|
|
791
1111
|
if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
|
|
792
|
-
else if (dominant === intersectsUSD && intersectsUSD > 0)
|
|
793
|
-
|
|
794
|
-
|
|
1112
|
+
else if (dominant === intersectsUSD && intersectsUSD > 0) {
|
|
1113
|
+
driver = "intersections (pointer, index and chunk GETs)";
|
|
1114
|
+
} else if (dominant === loadsUSD && loadsUSD > 0)
|
|
1115
|
+
driver = "loads (object, listing and pointer requests)";
|
|
1116
|
+
else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
|
|
1117
|
+
const sized = baseline.sized;
|
|
1118
|
+
const noBytes = Math.round(storageBytes) === 0;
|
|
1119
|
+
const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
|
|
1120
|
+
const rationale = verdict === "lose-zone" ? `pay-per-use total $${total.toFixed(2)}/mo exceeds the ${redisWords ?? `$${baselineUSD}/mo flat baseline`}; dominated by ${driver}` : verdict === "win-big" ? `$${total.toFixed(2)}/mo \u2014 \u226410% of the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}` : `$${total.toFixed(2)}/mo, under the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}`;
|
|
1121
|
+
const dataTiering = sized !== void 0 && (sized.cluster.nodeType.ssdGiB ?? 0) > 0;
|
|
1122
|
+
const redisNote = sized === void 0 ? `Redis priced at the fixed $${baselineUSD}/mo given, whatever the data size.` : (noBytes ? "Redis priced at the cheapest cluster in the catalogue, as no bytes are counted, because nothing is stored or nothing was measured" : `Redis priced at the cheapest cluster in the catalogue that holds the ${bytesWords(storageBytes)} stored`) + `: ${clusterWords(sized.cluster)}, each shard a primary and ${sized.sizing.replicasPerShard} replica(s), ${Number((sized.sizing.reservedMemoryFraction * 100).toFixed(1))}% of memory reserved (${sized.sizing.source}).` + (noBytes ? " Pass pricing.redis: { monthlyUSD } to compare against a Redis of your own." : " Compressed bytes are a floor on the memory Redis needs: a native Redis bitmap is sized by its highest id, so sparse ids need more.") + " Redis bought another way \u2014 fewer replicas, another engine, reserved nodes \u2014 can cost less than the catalogue prices it." + (dataTiering ? ` ${sized.cluster.nodeType.name} is a data-tiering node, which keeps the values read least recently on its SSD. It is priced as if the data read often fits in its memory: AWS recommends data tiering for workloads that regularly read up to 20% of their data.` : "") + (input.sizesFromCardinality ? " Some segment sizes came from cardinality, at 2 bytes an id, which is more than a dense bitmap needs, so this Redis may be larger than the data calls for: pass sizeBytes to size it to the compressed bytes." : "");
|
|
795
1123
|
const notes = [
|
|
796
1124
|
"Same-region egress treated as free; internet egress is not modeled.",
|
|
797
1125
|
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
798
|
-
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo
|
|
799
|
-
...
|
|
800
|
-
|
|
1126
|
+
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${STORE_LOAD_PUT_CLASS} PUT-class and ${loadGets} GETs that store.load() adds (listings, pointer, index).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
|
|
1127
|
+
...intersects > 0 ? [
|
|
1128
|
+
`Intersections priced cold: ${intersectGets} GETs each, ${SIZED_READS_PER_COLD_OPERAND * sizedRead} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk read(s); cacheHitRate does not apply.`
|
|
1129
|
+
] : [],
|
|
1130
|
+
hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
|
|
1131
|
+
hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
|
|
1132
|
+
...input.extraNotes ?? [],
|
|
1133
|
+
// Last, so the notes a report already carried keep their places.
|
|
1134
|
+
redisNote
|
|
1135
|
+
].filter((n) => n !== null);
|
|
801
1136
|
return {
|
|
802
1137
|
monthlyUSD: {
|
|
803
|
-
byOp: {
|
|
1138
|
+
byOp: {
|
|
1139
|
+
reads: readsUSD,
|
|
1140
|
+
intersects: intersectsUSD,
|
|
1141
|
+
storage: storageUSD,
|
|
1142
|
+
loads: loadsUSD,
|
|
1143
|
+
pointerRefresh: refreshUSD
|
|
1144
|
+
},
|
|
804
1145
|
total
|
|
805
1146
|
},
|
|
1147
|
+
redisBaseline: sized === void 0 ? { basis: "fixed", monthlyUSD: baselineUSD } : {
|
|
1148
|
+
basis: "sized-to-data",
|
|
1149
|
+
monthlyUSD: baselineUSD,
|
|
1150
|
+
cluster: {
|
|
1151
|
+
nodeType: sized.cluster.nodeType.name,
|
|
1152
|
+
shards: sized.cluster.shards,
|
|
1153
|
+
nodes: sized.cluster.nodes,
|
|
1154
|
+
dataTiering
|
|
1155
|
+
}
|
|
1156
|
+
},
|
|
806
1157
|
redisCrossover: { readsPerSec: readsCross },
|
|
807
1158
|
verdict,
|
|
808
1159
|
rationale,
|
|
@@ -810,36 +1161,39 @@ function buildReport(input) {
|
|
|
810
1161
|
};
|
|
811
1162
|
}
|
|
812
1163
|
function estimateCost(input) {
|
|
813
|
-
const pricing = input.pricing ??
|
|
1164
|
+
const pricing = input.pricing ?? AWS_US_EAST_1_ONDEMAND;
|
|
814
1165
|
const workload = input.workload ?? {};
|
|
815
1166
|
let storageBytes = 0;
|
|
1167
|
+
let sizesFromCardinality = false;
|
|
816
1168
|
for (const spec of input.segments) {
|
|
817
1169
|
const count = Math.floor(requireFiniteNonNeg(spec.count ?? 1, "segment.count"));
|
|
818
1170
|
storageBytes += sizingBytes(spec) * count;
|
|
1171
|
+
if (spec.sizeBytes === void 0 && spec.cardinality !== void 0 && sizingBytes(spec) * count > 0) {
|
|
1172
|
+
sizesFromCardinality = true;
|
|
1173
|
+
}
|
|
819
1174
|
}
|
|
820
|
-
return buildReport({ storageBytes, workload, pricing, grounded: false });
|
|
1175
|
+
return buildReport({ storageBytes, workload, pricing, grounded: false, sizesFromCardinality });
|
|
821
1176
|
}
|
|
822
1177
|
function groundedReport(input) {
|
|
823
1178
|
return buildReport({
|
|
824
1179
|
storageBytes: input.storageBytes,
|
|
825
1180
|
workload: input.workload ?? {},
|
|
826
|
-
pricing: input.pricing ??
|
|
1181
|
+
pricing: input.pricing ?? AWS_US_EAST_1_ONDEMAND,
|
|
827
1182
|
grounded: input.grounded ?? true,
|
|
828
1183
|
extraNotes: input.extraNotes
|
|
829
1184
|
});
|
|
830
1185
|
}
|
|
831
1186
|
|
|
832
1187
|
// src/core/due-index.ts
|
|
833
|
-
var DUE_NAMESPACE_PREFIX = "cbm.due.";
|
|
834
1188
|
var DUE_BUCKET_MS = 864e5;
|
|
835
1189
|
function dueBucket(expiresAt) {
|
|
836
1190
|
return Math.floor(expiresAt / DUE_BUCKET_MS);
|
|
837
1191
|
}
|
|
838
1192
|
function dueNamespace(bucket) {
|
|
839
|
-
return `${
|
|
1193
|
+
return `${RESERVED_NAMESPACE_PREFIX}${bucket}`;
|
|
840
1194
|
}
|
|
841
1195
|
function isDueIndexRow(record) {
|
|
842
|
-
return record.namespace
|
|
1196
|
+
return isReservedNamespace(record.namespace);
|
|
843
1197
|
}
|
|
844
1198
|
function encodeDueName(ref) {
|
|
845
1199
|
const ns = ref.namespace ?? "";
|
|
@@ -891,12 +1245,14 @@ function validateMaxScanSegments(value, op) {
|
|
|
891
1245
|
async function drainRegistry(registry, options) {
|
|
892
1246
|
const { maxScanSegments, op } = options;
|
|
893
1247
|
validateMaxScanSegments(maxScanSegments, op);
|
|
1248
|
+
const raise = options.raise ?? "raise `maxScanSegments`";
|
|
1249
|
+
const advice = options.narrowable === false ? `${raise[0].toUpperCase()}${raise.slice(1)} if the namespace really is that large and the memory is ` : `Narrow it with \`namespace\`, or ${raise} if the fleet really is that large and the memory is `;
|
|
894
1250
|
const rows = [];
|
|
895
1251
|
for await (const rec of registry.list(options.namespace)) {
|
|
896
1252
|
if (options.namespace === void 0 && isReservedRow(rec)) continue;
|
|
897
1253
|
if (rows.length >= maxScanSegments) {
|
|
898
1254
|
throw new BudgetExceededError(
|
|
899
|
-
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed.
|
|
1255
|
+
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident).`
|
|
900
1256
|
);
|
|
901
1257
|
}
|
|
902
1258
|
rows.push(rec);
|
|
@@ -916,7 +1272,7 @@ async function* excludingReservedRows(source) {
|
|
|
916
1272
|
function requireCodec(codec, api) {
|
|
917
1273
|
if (codec === void 0) {
|
|
918
1274
|
throw new ValidationError(
|
|
919
|
-
`${api} needs a bitmap codec: pass \`codec
|
|
1275
|
+
`${api} needs a bitmap codec: pass \`codec\`. @cloudbitmaps/core is codec-agnostic and has no default. A flavor's store passes its own codec, so an application calls the store's method instead of this function.`
|
|
920
1276
|
);
|
|
921
1277
|
}
|
|
922
1278
|
return codec;
|
|
@@ -926,6 +1282,7 @@ function requireCodec(codec, api) {
|
|
|
926
1282
|
var DEFAULT_NDJSON_BATCH_BYTES = 64 * 1024;
|
|
927
1283
|
var errMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
928
1284
|
async function runExport(reader, registry, sink, options = {}) {
|
|
1285
|
+
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
929
1286
|
const format = options.format ?? "roaring";
|
|
930
1287
|
const ext = format === "roaring" ? ".roaring" : ".ndjson";
|
|
931
1288
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
@@ -997,13 +1354,12 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
997
1354
|
|
|
998
1355
|
// src/core/discover.ts
|
|
999
1356
|
async function segmentExists(ref, registry) {
|
|
1000
|
-
|
|
1357
|
+
validateUserRef(ref);
|
|
1001
1358
|
const record = await registry.get(ref);
|
|
1002
1359
|
return record !== null && record.status !== "destroyed" && record.currentGen !== null;
|
|
1003
1360
|
}
|
|
1004
1361
|
async function* listSegments(registry, options = {}) {
|
|
1005
|
-
if (options.namespace !== void 0)
|
|
1006
|
-
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
1362
|
+
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
1007
1363
|
const raw = registry.list(options.namespace);
|
|
1008
1364
|
const source = options.namespace === void 0 ? excludingReservedRows(raw) : raw;
|
|
1009
1365
|
for await (const record of source) {
|
|
@@ -1016,6 +1372,9 @@ async function* listSegments(registry, options = {}) {
|
|
|
1016
1372
|
}
|
|
1017
1373
|
}
|
|
1018
1374
|
|
|
1375
|
+
// src/drivers/backends.ts
|
|
1376
|
+
import { join as join3 } from "node:path";
|
|
1377
|
+
|
|
1019
1378
|
// src/drivers/memory.ts
|
|
1020
1379
|
import { createHash } from "node:crypto";
|
|
1021
1380
|
|
|
@@ -1059,39 +1418,6 @@ var BufferReader = class {
|
|
|
1059
1418
|
};
|
|
1060
1419
|
|
|
1061
1420
|
// src/drivers/memory.ts
|
|
1062
|
-
var MemoryStorageChunkSource = class {
|
|
1063
|
-
chunks = /* @__PURE__ */ new Map();
|
|
1064
|
-
async getChunk(ref) {
|
|
1065
|
-
validateChunkRef(ref);
|
|
1066
|
-
return this.chunks.get(chunkRefKey(ref)) ?? null;
|
|
1067
|
-
}
|
|
1068
|
-
async listChunkKeys(ref) {
|
|
1069
|
-
validateSegmentRef(ref);
|
|
1070
|
-
const prefix = segmentPrefix(ref);
|
|
1071
|
-
const keys = [];
|
|
1072
|
-
for (const key of this.chunks.keys()) {
|
|
1073
|
-
if (key.startsWith(prefix)) keys.push(Number(key.slice(prefix.length)));
|
|
1074
|
-
}
|
|
1075
|
-
return keys;
|
|
1076
|
-
}
|
|
1077
|
-
async sizeOf(ref) {
|
|
1078
|
-
validateSegmentRef(ref);
|
|
1079
|
-
const prefix = segmentPrefix(ref);
|
|
1080
|
-
let sizeBytes = 0;
|
|
1081
|
-
let found = false;
|
|
1082
|
-
for (const [key, bytes] of this.chunks) {
|
|
1083
|
-
if (!key.startsWith(prefix)) continue;
|
|
1084
|
-
found = true;
|
|
1085
|
-
sizeBytes += bytes.length;
|
|
1086
|
-
}
|
|
1087
|
-
return found ? { sizeBytes } : null;
|
|
1088
|
-
}
|
|
1089
|
-
/** Test/seed helper — populate immutable Storage bytes for a chunk directly (bypassing the `.crbm` format). */
|
|
1090
|
-
seed(ref, bytes) {
|
|
1091
|
-
validateChunkRef(ref);
|
|
1092
|
-
this.chunks.set(chunkRefKey(ref), bytes);
|
|
1093
|
-
}
|
|
1094
|
-
};
|
|
1095
1421
|
var MemoryRegistryDriver = class {
|
|
1096
1422
|
rows = /* @__PURE__ */ new Map();
|
|
1097
1423
|
now;
|
|
@@ -1139,9 +1465,13 @@ var MemoryRegistryDriver = class {
|
|
|
1139
1465
|
if (namespace === void 0 || row.namespace === namespace) yield structuredClone(row);
|
|
1140
1466
|
}
|
|
1141
1467
|
}
|
|
1142
|
-
async delete(ref) {
|
|
1468
|
+
async delete(ref, expected) {
|
|
1143
1469
|
validateSegmentRef(ref);
|
|
1144
|
-
|
|
1470
|
+
const key = segmentKey(ref);
|
|
1471
|
+
if (expected !== void 0 && this.rows.get(key)?.token !== expected) {
|
|
1472
|
+
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1473
|
+
}
|
|
1474
|
+
this.rows.delete(key);
|
|
1145
1475
|
}
|
|
1146
1476
|
};
|
|
1147
1477
|
var MemoryStorageDriver = class {
|
|
@@ -1421,17 +1751,31 @@ var LocalFsStorageDriver = class {
|
|
|
1421
1751
|
// src/drivers/localfs/registry.ts
|
|
1422
1752
|
import { constants as FS3 } from "node:fs";
|
|
1423
1753
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1424
|
-
import { mkdir as mkdir2, open as open3, readdir as readdir2, rename, unlink as unlink2 } from "node:fs/promises";
|
|
1425
|
-
import { dirname as dirname2 } from "node:path";
|
|
1754
|
+
import { mkdir as mkdir2, open as open3, readdir as readdir2, realpath, rename, unlink as unlink2 } from "node:fs/promises";
|
|
1755
|
+
import { basename, dirname as dirname2, join as join2, resolve } from "node:path";
|
|
1426
1756
|
var DEFAULT_MAX_ROW_BYTES = 1 * 1024 * 1024;
|
|
1757
|
+
var rowChains = /* @__PURE__ */ new Map();
|
|
1758
|
+
async function rowLockKey(path) {
|
|
1759
|
+
const abs = resolve(path);
|
|
1760
|
+
const tail = [basename(abs)];
|
|
1761
|
+
let dir = dirname2(abs);
|
|
1762
|
+
for (; ; ) {
|
|
1763
|
+
try {
|
|
1764
|
+
return join2(await realpath(dir), ...tail);
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
const parent = dirname2(dir);
|
|
1767
|
+
if (!isCode(err, "ENOENT") || parent === dir) throw mapFsError(err);
|
|
1768
|
+
tail.unshift(basename(dir));
|
|
1769
|
+
dir = parent;
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1427
1773
|
var LocalFsRegistryDriver = class {
|
|
1428
1774
|
constructor(root, options = {}) {
|
|
1429
1775
|
this.root = root;
|
|
1430
1776
|
this.now = options.now ?? (() => Date.now());
|
|
1431
1777
|
}
|
|
1432
1778
|
root;
|
|
1433
|
-
/** Per-row promise chain — serializes read-modify-write so an in-process CAS never loses an update. */
|
|
1434
|
-
chain = /* @__PURE__ */ new Map();
|
|
1435
1779
|
now;
|
|
1436
1780
|
capabilities() {
|
|
1437
1781
|
return { strongRead: true };
|
|
@@ -1489,11 +1833,17 @@ var LocalFsRegistryDriver = class {
|
|
|
1489
1833
|
}
|
|
1490
1834
|
}
|
|
1491
1835
|
}
|
|
1492
|
-
async delete(ref) {
|
|
1836
|
+
async delete(ref, expected) {
|
|
1493
1837
|
const path = registryRowPath(this.root, ref);
|
|
1494
1838
|
return this.withRowLock(path, async () => {
|
|
1495
1839
|
const current = await this.readRow(path);
|
|
1496
|
-
if (
|
|
1840
|
+
if (expected !== void 0) {
|
|
1841
|
+
if (current === null || current.deleted || current.record.token !== expected) {
|
|
1842
|
+
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1843
|
+
}
|
|
1844
|
+
} else if (current === null || current.deleted) {
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1497
1847
|
const token = String(registryCounterOf(current.record) + 1);
|
|
1498
1848
|
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
1499
1849
|
});
|
|
@@ -1562,25 +1912,24 @@ var LocalFsRegistryDriver = class {
|
|
|
1562
1912
|
});
|
|
1563
1913
|
await fsyncDir(dirname2(path));
|
|
1564
1914
|
}
|
|
1565
|
-
/** Serialize callbacks for a row
|
|
1566
|
-
withRowLock(
|
|
1567
|
-
const
|
|
1915
|
+
/** Serialize callbacks for a row so read-modify-write is atomic across every instance in the process. */
|
|
1916
|
+
async withRowLock(path, fn) {
|
|
1917
|
+
const key = await rowLockKey(path);
|
|
1918
|
+
const prev = rowChains.get(key) ?? Promise.resolve();
|
|
1568
1919
|
const result = prev.then(fn, fn);
|
|
1569
1920
|
const tail = result.then(
|
|
1570
1921
|
() => void 0,
|
|
1571
1922
|
() => void 0
|
|
1572
1923
|
);
|
|
1573
|
-
|
|
1924
|
+
rowChains.set(key, tail);
|
|
1574
1925
|
void tail.then(() => {
|
|
1575
|
-
if (
|
|
1926
|
+
if (rowChains.get(key) === tail) rowChains.delete(key);
|
|
1576
1927
|
});
|
|
1577
1928
|
return result;
|
|
1578
1929
|
}
|
|
1579
1930
|
};
|
|
1580
1931
|
|
|
1581
1932
|
// src/drivers/backends.ts
|
|
1582
|
-
import { existsSync } from "node:fs";
|
|
1583
|
-
import { join as join2 } from "node:path";
|
|
1584
1933
|
var MemoryStorage = class {
|
|
1585
1934
|
storage;
|
|
1586
1935
|
registry;
|
|
@@ -1593,14 +1942,9 @@ var MemoryStorage = class {
|
|
|
1593
1942
|
var LocalFsStorage = class {
|
|
1594
1943
|
constructor(root, options = {}) {
|
|
1595
1944
|
this.root = root;
|
|
1596
|
-
|
|
1597
|
-
throw new ValidationError(
|
|
1598
|
-
`${root} holds a "cold/" directory but no "storage/" \u2014 this store was written before the tier was renamed. Rename it (\`mv ${root}/cold ${root}/storage\`) and re-run; the objects inside are unchanged. Do NOT treat the "missing-storage-generation" you would otherwise see as a torn restore \u2014 the data is intact, it is the path that moved.`
|
|
1599
|
-
);
|
|
1600
|
-
}
|
|
1601
|
-
this.storage = new LocalFsStorageDriver(join2(root, "storage"));
|
|
1945
|
+
this.storage = new LocalFsStorageDriver(join3(root, "storage"));
|
|
1602
1946
|
this.registry = new LocalFsRegistryDriver(
|
|
1603
|
-
|
|
1947
|
+
join3(root, "registry"),
|
|
1604
1948
|
options.now === void 0 ? {} : { now: options.now }
|
|
1605
1949
|
);
|
|
1606
1950
|
brandAsBackend(this);
|
|
@@ -1779,7 +2123,7 @@ var FOOTER = {
|
|
|
1779
2123
|
flags: 20,
|
|
1780
2124
|
// u32
|
|
1781
2125
|
payloadCodecId: 24,
|
|
1782
|
-
// u16 —
|
|
2126
|
+
// u16 — which codec wrote the chunk payloads
|
|
1783
2127
|
elementWidth: 26,
|
|
1784
2128
|
// u8
|
|
1785
2129
|
containerCodec: 27,
|
|
@@ -1831,6 +2175,29 @@ var RETAINED_BYTES_PER_INDEX_ENTRY = 160;
|
|
|
1831
2175
|
function magicMatches(bytes, offset) {
|
|
1832
2176
|
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
1833
2177
|
}
|
|
2178
|
+
var sizePart = (size) => `${size}:`;
|
|
2179
|
+
var fingerprintFor = (size, footerCrc) => `${sizePart(size)}${footerCrc}`;
|
|
2180
|
+
function checkSize(tail, size) {
|
|
2181
|
+
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
2182
|
+
throw new IntegrityError(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
function checkedFooter(tail, size) {
|
|
2186
|
+
checkSize(tail, size);
|
|
2187
|
+
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
2188
|
+
throw new IntegrityError(`.crbm too small: ${size}B`);
|
|
2189
|
+
}
|
|
2190
|
+
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
2191
|
+
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
2192
|
+
throw new IntegrityError(".crbm end magic mismatch");
|
|
2193
|
+
}
|
|
2194
|
+
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
2195
|
+
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
2196
|
+
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
2197
|
+
throw new IntegrityError(".crbm footer CRC mismatch");
|
|
2198
|
+
}
|
|
2199
|
+
return { footer, fview, storedFooterCrc };
|
|
2200
|
+
}
|
|
1834
2201
|
function readU64(view, offset, field) {
|
|
1835
2202
|
const big = view.getBigUint64(offset, true);
|
|
1836
2203
|
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
@@ -1839,7 +2206,7 @@ function readU64(view, offset, field) {
|
|
|
1839
2206
|
return Number(big);
|
|
1840
2207
|
}
|
|
1841
2208
|
var CrbmReader = class _CrbmReader {
|
|
1842
|
-
constructor(blob, objectSize, generation, lineage, totalCardinality, entries, orderedKeys, servedFromTail, crypto) {
|
|
2209
|
+
constructor(blob, objectSize, generation, lineage, totalCardinality, entries, orderedKeys, servedFromTail, crypto, footerCrc) {
|
|
1843
2210
|
this.blob = blob;
|
|
1844
2211
|
this.objectSize = objectSize;
|
|
1845
2212
|
this.generation = generation;
|
|
@@ -1849,6 +2216,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
1849
2216
|
this.orderedKeys = orderedKeys;
|
|
1850
2217
|
this.servedFromTail = servedFromTail;
|
|
1851
2218
|
this.crypto = crypto;
|
|
2219
|
+
this.footerCrc = footerCrc;
|
|
1852
2220
|
}
|
|
1853
2221
|
blob;
|
|
1854
2222
|
objectSize;
|
|
@@ -1859,6 +2227,15 @@ var CrbmReader = class _CrbmReader {
|
|
|
1859
2227
|
orderedKeys;
|
|
1860
2228
|
servedFromTail;
|
|
1861
2229
|
crypto;
|
|
2230
|
+
footerCrc;
|
|
2231
|
+
/**
|
|
2232
|
+
* Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
|
|
2233
|
+
* objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
|
|
2234
|
+
* the object it reopens is the one it pinned, since a purged and reloaded name reuses the generation number.
|
|
2235
|
+
*/
|
|
2236
|
+
get fingerprint() {
|
|
2237
|
+
return fingerprintFor(this.objectSize, this.footerCrc);
|
|
2238
|
+
}
|
|
1862
2239
|
/** Total object bytes (from the one-GET tail read) — for grounded storage cost. */
|
|
1863
2240
|
get sizeBytes() {
|
|
1864
2241
|
return this.objectSize;
|
|
@@ -1877,23 +2254,26 @@ var CrbmReader = class _CrbmReader {
|
|
|
1877
2254
|
for (const [chunkKey, entry] of this.entries) out.set(chunkKey, entry.cardinality);
|
|
1878
2255
|
return out;
|
|
1879
2256
|
}
|
|
2257
|
+
/**
|
|
2258
|
+
* Whether the object behind `blob` is the one `fingerprint` names ({@link fingerprint}), from one tail read of a
|
|
2259
|
+
* footer's worth, with no key and no index, since the footer is stored in the clear and checked by its own CRC.
|
|
2260
|
+
* Another size is another object, whatever its bytes hold: a short one that is no `.crbm` at all is not the one
|
|
2261
|
+
* pinned. At the same size the footer's CRC says, and a footer that fails its own checks says nothing about which
|
|
2262
|
+
* object is there, so this throws, as an open would. What a pin asks, to tell the object it opened from one that
|
|
2263
|
+
* has since been stored under its key, whoever wrote it.
|
|
2264
|
+
*/
|
|
2265
|
+
static async sameObject(blob, fingerprint) {
|
|
2266
|
+
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2267
|
+
checkSize(tail, size);
|
|
2268
|
+
if (!fingerprint.startsWith(sizePart(size))) return false;
|
|
2269
|
+
return fingerprintFor(size, checkedFooter(tail, size).storedFooterCrc) === fingerprint;
|
|
2270
|
+
}
|
|
1880
2271
|
static async open(blob, options = {}) {
|
|
1881
2272
|
const tailBytes = Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
1882
2273
|
const maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
|
|
1883
2274
|
const maxIndexBytes = options.maxIndexBytes ?? DEFAULT_MAX_INDEX_BYTES;
|
|
1884
2275
|
const { bytes: tail, size } = await blob.getTail(tailBytes);
|
|
1885
|
-
|
|
1886
|
-
throw new IntegrityError(`.crbm too small: ${size}B`);
|
|
1887
|
-
}
|
|
1888
|
-
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
1889
|
-
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
1890
|
-
throw new IntegrityError(".crbm end magic mismatch");
|
|
1891
|
-
}
|
|
1892
|
-
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
1893
|
-
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
1894
|
-
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
1895
|
-
throw new IntegrityError(".crbm footer CRC mismatch");
|
|
1896
|
-
}
|
|
2276
|
+
const { footer, fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
1897
2277
|
const versionMajor = footer[FOOTER.versionMajor];
|
|
1898
2278
|
if (versionMajor !== VERSION_MAJOR) {
|
|
1899
2279
|
throw new UnsupportedError(`.crbm major version ${versionMajor} not supported`);
|
|
@@ -1970,8 +2350,9 @@ var CrbmReader = class _CrbmReader {
|
|
|
1970
2350
|
) : indexBytes;
|
|
1971
2351
|
const { entries, orderedKeys, cardinalitySum } = parseIndex(
|
|
1972
2352
|
indexForParse,
|
|
1973
|
-
|
|
1974
|
-
maxPayloadBytes
|
|
2353
|
+
indexOffset,
|
|
2354
|
+
maxPayloadBytes,
|
|
2355
|
+
encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
|
|
1975
2356
|
);
|
|
1976
2357
|
if (encrypted) {
|
|
1977
2358
|
if (chunkCount !== 0 || totalCardinality !== 0) {
|
|
@@ -2000,7 +2381,8 @@ var CrbmReader = class _CrbmReader {
|
|
|
2000
2381
|
entries,
|
|
2001
2382
|
orderedKeys,
|
|
2002
2383
|
servedFromTail,
|
|
2003
|
-
options.crypto
|
|
2384
|
+
options.crypto,
|
|
2385
|
+
storedFooterCrc
|
|
2004
2386
|
);
|
|
2005
2387
|
}
|
|
2006
2388
|
/** Chunk keys present in this generation, ascending. */
|
|
@@ -2044,7 +2426,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
2044
2426
|
);
|
|
2045
2427
|
}
|
|
2046
2428
|
};
|
|
2047
|
-
function parseIndex(indexBytes,
|
|
2429
|
+
function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1) {
|
|
2048
2430
|
const entries = /* @__PURE__ */ new Map();
|
|
2049
2431
|
const orderedKeys = [];
|
|
2050
2432
|
let pos = 0;
|
|
@@ -2068,13 +2450,13 @@ function parseIndex(indexBytes, objectSize, maxPayloadBytes) {
|
|
|
2068
2450
|
throw new IntegrityError(`.crbm index has a duplicate chunkKey ${chunkKey}`);
|
|
2069
2451
|
}
|
|
2070
2452
|
if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
|
|
2071
|
-
if (len.value
|
|
2453
|
+
if (len.value < minPayloadBytes || len.value > maxPayloadBytes) {
|
|
2072
2454
|
throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len.value} invalid`);
|
|
2073
2455
|
}
|
|
2074
2456
|
if (card.value < 1 || card.value > MAX_CHUNK_CARDINALITY) {
|
|
2075
2457
|
throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card.value} invalid`);
|
|
2076
2458
|
}
|
|
2077
|
-
if (offset
|
|
2459
|
+
if (offset + len.value > payloadEnd) {
|
|
2078
2460
|
throw new IntegrityError(`.crbm chunk ${chunkKey} payload out of bounds`);
|
|
2079
2461
|
}
|
|
2080
2462
|
entries.set(chunkKey, {
|
|
@@ -2244,11 +2626,14 @@ function storageBlobReader(driver, key) {
|
|
|
2244
2626
|
}
|
|
2245
2627
|
};
|
|
2246
2628
|
}
|
|
2247
|
-
var
|
|
2629
|
+
var notThePinned = (ref, generation) => new NotFoundError(
|
|
2630
|
+
`segment "${ref.segment}" generation ${generation} is no longer the object this handle pinned`
|
|
2631
|
+
);
|
|
2632
|
+
function versionOf(generation, lineage) {
|
|
2633
|
+
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
2634
|
+
}
|
|
2248
2635
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
2249
2636
|
var YIELD_EVERY_IDS = 1 << 14;
|
|
2250
|
-
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
2251
|
-
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
2252
2637
|
var CrbmStorageChunkSource = class {
|
|
2253
2638
|
constructor(driver, options = {}) {
|
|
2254
2639
|
this.driver = driver;
|
|
@@ -2284,21 +2669,47 @@ var CrbmStorageChunkSource = class {
|
|
|
2284
2669
|
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
2285
2670
|
clock: clock ?? { now: () => 0 }
|
|
2286
2671
|
});
|
|
2672
|
+
this.replacedPins = new BoundedLru({
|
|
2673
|
+
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
2674
|
+
clock: { now: () => 0 }
|
|
2675
|
+
});
|
|
2287
2676
|
}
|
|
2288
2677
|
driver;
|
|
2289
2678
|
/**
|
|
2290
2679
|
* One resolved reader per segment, re-resolved on a short TTL ({@link CrbmStorageChunkSourceOptions.currentGenTtlMs},
|
|
2291
2680
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
2292
2681
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
2293
|
-
* opens the new generation — so a long-lived source observes new generations within the TTL
|
|
2294
|
-
*
|
|
2295
|
-
*
|
|
2296
|
-
*
|
|
2297
|
-
* one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
2682
|
+
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
2683
|
+
* this with a **generation-keyed** cache so a bump never serves a stale decoded chunk. Without a clock or a
|
|
2684
|
+
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
2685
|
+
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
2686
|
+
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
2298
2687
|
* 1024): past the ceiling the least-recently-used segment's reader (and its parsed index) is evicted — the
|
|
2299
2688
|
* steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
|
|
2300
2689
|
*/
|
|
2301
2690
|
snapshots;
|
|
2691
|
+
/**
|
|
2692
|
+
* Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
|
|
2693
|
+
* and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
|
|
2694
|
+
* the check again. A replacement can be undone, by a restore that puts the pinned object back, so
|
|
2695
|
+
* {@link invalidate} forgets the segment's, and a pin of the same version that opens the object forgets the one
|
|
2696
|
+
* against it. An object found gone is not kept: that is the open's own `NotFoundError`, and a 404 can pass. A
|
|
2697
|
+
* reader still memoised keeps answering from the index it holds, so the pin's `count()` does, until a later pin of
|
|
2698
|
+
* that version opens the object now under the key, which takes its place.
|
|
2699
|
+
*/
|
|
2700
|
+
replacedPins;
|
|
2701
|
+
/**
|
|
2702
|
+
* A check of a pinned object still under way, by {@link heldKey}, so the reads that ask at once share one footer
|
|
2703
|
+
* read. One forgotten before it ends, by {@link invalidate} or a pin, keeps nothing it finds.
|
|
2704
|
+
*/
|
|
2705
|
+
checking = /* @__PURE__ */ new Map();
|
|
2706
|
+
/**
|
|
2707
|
+
* How many times {@link invalidate} has run, so a read that waited can tell whether the store was told something
|
|
2708
|
+
* meanwhile. An eviction from the reader cache is not an invalidation, and what a read found stands through one. It
|
|
2709
|
+
* counts every segment's invalidations: a read that waited through another segment's only costs a later read the
|
|
2710
|
+
* check again.
|
|
2711
|
+
*/
|
|
2712
|
+
invalidations = 0;
|
|
2302
2713
|
registry;
|
|
2303
2714
|
keystore;
|
|
2304
2715
|
requireEncryption;
|
|
@@ -2311,18 +2722,35 @@ var CrbmStorageChunkSource = class {
|
|
|
2311
2722
|
const existing = this.snapshots.get(key);
|
|
2312
2723
|
if (existing === void 0) return this.install(key, this.resolveLatest(ref)).reader;
|
|
2313
2724
|
if (!this.expired(existing.installedAtMs)) return existing.reader;
|
|
2314
|
-
|
|
2725
|
+
const snap = this.install(
|
|
2726
|
+
key,
|
|
2727
|
+
this.computeRefreshedReader(ref, existing, () => this.retrySoon(snap))
|
|
2728
|
+
);
|
|
2729
|
+
return snap.reader;
|
|
2730
|
+
}
|
|
2731
|
+
/**
|
|
2732
|
+
* A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
|
|
2733
|
+
* {@link REFRESH_RETRY_MS} (never longer than the TTL) rather than after a whole TTL, so the next read after
|
|
2734
|
+
* that asks the registry again. It costs no read of its own; it only changes when the next refresh happens.
|
|
2735
|
+
*/
|
|
2736
|
+
retrySoon(snap) {
|
|
2737
|
+
snap.installedAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
2315
2738
|
}
|
|
2316
2739
|
/** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
|
|
2317
|
-
async computeRefreshedReader(ref, existing) {
|
|
2740
|
+
async computeRefreshedReader(ref, existing, onTransientFault) {
|
|
2318
2741
|
const current = await existing.reader.catch(() => null);
|
|
2319
2742
|
let target;
|
|
2320
2743
|
try {
|
|
2321
2744
|
target = await this.resolveTarget(ref);
|
|
2322
|
-
} catch {
|
|
2745
|
+
} catch (err) {
|
|
2746
|
+
if (!isTransientError(err)) throw err;
|
|
2323
2747
|
target = void 0;
|
|
2324
2748
|
}
|
|
2325
|
-
if (target === void 0)
|
|
2749
|
+
if (target === void 0) {
|
|
2750
|
+
if (current === null) return this.resolveLatest(ref);
|
|
2751
|
+
onTransientFault();
|
|
2752
|
+
return current;
|
|
2753
|
+
}
|
|
2326
2754
|
if (target === null) return null;
|
|
2327
2755
|
if (current !== null && target.generation === current.generation && target.lineage === current.lineage) {
|
|
2328
2756
|
return current;
|
|
@@ -2352,6 +2780,14 @@ var CrbmStorageChunkSource = class {
|
|
|
2352
2780
|
now() {
|
|
2353
2781
|
return this.clock ? this.clock.now() : 0;
|
|
2354
2782
|
}
|
|
2783
|
+
/**
|
|
2784
|
+
* How often this source re-reads a segment's pointer while the segment is being read, in ms: its TTL, or 0 when
|
|
2785
|
+
* it has no timed refresh (no clock, no registry, or a TTL of 0). The grounded cost report prices the refresh at
|
|
2786
|
+
* this.
|
|
2787
|
+
*/
|
|
2788
|
+
get pointerRefreshMs() {
|
|
2789
|
+
return this.clock !== void 0 && this.registry !== void 0 ? this.currentGenTtlMs : 0;
|
|
2790
|
+
}
|
|
2355
2791
|
expired(installedAtMs) {
|
|
2356
2792
|
return this.clock !== void 0 && this.registry !== void 0 && this.currentGenTtlMs > 0 && this.clock.now() - installedAtMs >= this.currentGenTtlMs;
|
|
2357
2793
|
}
|
|
@@ -2360,11 +2796,12 @@ var CrbmStorageChunkSource = class {
|
|
|
2360
2796
|
* is observed instead of serving a stale decoded chunk. Served from the (TTL-refreshed)
|
|
2361
2797
|
* snapshot, so no extra backend read within the TTL window. `null` if the segment has no committed generation.
|
|
2362
2798
|
*
|
|
2363
|
-
* Heals a swept generation exactly like {@link withFreshSnapshot}
|
|
2364
|
-
* before any chunk fetch,
|
|
2365
|
-
*
|
|
2366
|
-
*
|
|
2367
|
-
*
|
|
2799
|
+
* Heals a swept generation exactly like {@link withFreshSnapshot}: a caller that resolves the generation once
|
|
2800
|
+
* per op, before any chunk fetch, would otherwise fail the whole operation rather than one chunk. The engine
|
|
2801
|
+
* keys by {@link currentVersion} instead, which heals the same way, and falls back to this for a source that
|
|
2802
|
+
* cannot report a version. It is spelled out rather than delegated because a lookup like this runs once per
|
|
2803
|
+
* operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
|
|
2804
|
+
* no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
|
|
2368
2805
|
* frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
|
|
2369
2806
|
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
2370
2807
|
*/
|
|
@@ -2380,50 +2817,126 @@ var CrbmStorageChunkSource = class {
|
|
|
2380
2817
|
}
|
|
2381
2818
|
/**
|
|
2382
2819
|
* Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with the
|
|
2383
|
-
* reader, the DEK it unwrapped at open time)
|
|
2820
|
+
* reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
2821
|
+
* resolves the segment again.
|
|
2384
2822
|
*
|
|
2385
2823
|
* Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
|
|
2386
2824
|
* an erasure that deletes the generation holding the bit, a `dropSegment`, a crypto-shred, a retirement —
|
|
2387
2825
|
* leaves the pointer's answer unchanged from this source's point of view while making the snapshot wrong.
|
|
2388
2826
|
* Until this is called the source keeps serving that snapshot with no backend read at all, so nothing on the
|
|
2389
|
-
* storage side can close the window; with `cache.genTtlMs: 0
|
|
2827
|
+
* storage side can close the window; with no registry, no clock or `cache.genTtlMs: 0`, nothing bounds it at all.
|
|
2390
2828
|
*/
|
|
2391
2829
|
invalidate(ref) {
|
|
2830
|
+
this.invalidations += 1;
|
|
2392
2831
|
const key = segmentKey(ref);
|
|
2393
2832
|
this.snapshots.delete(key);
|
|
2394
2833
|
const pinnedPrefix = `${key}@`;
|
|
2395
2834
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2835
|
+
this.replacedPins.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2836
|
+
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
2396
2837
|
}
|
|
2397
2838
|
/**
|
|
2398
|
-
*
|
|
2399
|
-
*
|
|
2400
|
-
*
|
|
2401
|
-
*
|
|
2839
|
+
* `<generation>` for a registry-less source, or `<generation>:<row token>` with one. The token moves on every row
|
|
2840
|
+
* write, so an unrelated write (a `setRetention`) costs the segment's decoded chunks once — bounded, and on an
|
|
2841
|
+
* admin path. Exactness in the direction that matters: with a registry, two incarnations can never share a version
|
|
2842
|
+
* string. Without one they can: a name purged and loaded again out of band restarts at the same bare number, and
|
|
2843
|
+
* a live read that reopens it after an eviction reads the new object's chunks beside the old one's cached ones. A
|
|
2844
|
+
* store with no registry cannot write, so that takes a change made from outside it; a pin is kept apart from it by
|
|
2845
|
+
* the fingerprint it records.
|
|
2402
2846
|
*
|
|
2403
|
-
*
|
|
2404
|
-
*
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
*
|
|
2408
|
-
* `<generation>:<row token>` with one. The token moves on every row write, so an unrelated write (a
|
|
2409
|
-
* `setRetention`) costs the segment's decoded chunks once — bounded, and on an admin path. Exactness in the
|
|
2410
|
-
* direction that matters: two incarnations can never share a version string.
|
|
2847
|
+
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
2848
|
+
* chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
|
|
2849
|
+
* racing a publish and a `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race
|
|
2850
|
+
* on a chunk fetch heals. Spelled out rather than delegated for the reason `currentGeneration` gives: it is
|
|
2851
|
+
* almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
|
|
2411
2852
|
*/
|
|
2412
2853
|
async currentVersion(ref) {
|
|
2413
|
-
const
|
|
2414
|
-
|
|
2415
|
-
|
|
2854
|
+
const pending = this.resolvedReader(ref);
|
|
2855
|
+
let reader;
|
|
2856
|
+
try {
|
|
2857
|
+
reader = await pending;
|
|
2858
|
+
} catch (err) {
|
|
2859
|
+
if (!isNotFoundError(err)) throw err;
|
|
2860
|
+
this.dropStale(segmentKey(ref), pending);
|
|
2861
|
+
reader = await this.resolvedReader(ref);
|
|
2862
|
+
}
|
|
2863
|
+
return reader === null ? null : versionOf(reader.generation, reader.lineage);
|
|
2416
2864
|
}
|
|
2417
2865
|
/**
|
|
2418
2866
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
2419
2867
|
* identifies those exact bytes. `null` when the segment resolves to no generation — there is nothing to pin,
|
|
2420
2868
|
* and a caller must treat that as "this handle reads empty", not "pinning is unsupported here".
|
|
2869
|
+
*
|
|
2870
|
+
* Heals a generation swept between the registry read and the open, as {@link currentVersion} does: a publish
|
|
2871
|
+
* and a `keep: 0` sweep can land in that gap, and nothing has been handed out yet, so the generation current
|
|
2872
|
+
* once the swept one is gone is exactly what a pin taken now should hold. One retry, then propagate.
|
|
2421
2873
|
*/
|
|
2422
2874
|
async pinGeneration(ref) {
|
|
2875
|
+
try {
|
|
2876
|
+
return await this.pinOnce(ref);
|
|
2877
|
+
} catch (err) {
|
|
2878
|
+
if (!isNotFoundError(err)) throw err;
|
|
2879
|
+
return this.pinOnce(ref);
|
|
2880
|
+
}
|
|
2881
|
+
}
|
|
2882
|
+
async pinOnce(ref) {
|
|
2423
2883
|
const target = await this.resolveTarget(ref);
|
|
2424
2884
|
if (target === null) return null;
|
|
2425
|
-
const version =
|
|
2426
|
-
|
|
2885
|
+
const version = versionOf(target.generation, target.lineage);
|
|
2886
|
+
let reader;
|
|
2887
|
+
let opened = true;
|
|
2888
|
+
if (target.lineage === void 0) {
|
|
2889
|
+
reader = await this.openForTarget(ref, target);
|
|
2890
|
+
this.install(this.pinnedKey(ref, version, reader.fingerprint), Promise.resolve(reader));
|
|
2891
|
+
} else {
|
|
2892
|
+
const epoch = this.invalidations;
|
|
2893
|
+
const key = this.pinnedKey(ref, version);
|
|
2894
|
+
let entry = this.snapshots.get(key);
|
|
2895
|
+
opened = entry === void 0;
|
|
2896
|
+
entry ??= this.install(key, this.openForTarget(ref, target));
|
|
2897
|
+
reader = await entry.reader;
|
|
2898
|
+
if (reader === null) return null;
|
|
2899
|
+
const replaced = (r) => this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0;
|
|
2900
|
+
const known = !opened && replaced(reader);
|
|
2901
|
+
if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
|
|
2902
|
+
this.install(key, Promise.resolve(reader));
|
|
2903
|
+
}
|
|
2904
|
+
if (known) {
|
|
2905
|
+
const current = this.snapshots.peek(key);
|
|
2906
|
+
let fresh = current !== void 0 && current !== entry ? current : this.install(key, this.openForTarget(ref, target));
|
|
2907
|
+
reader = await fresh.reader;
|
|
2908
|
+
if (reader === null) return null;
|
|
2909
|
+
if (fresh === current && replaced(reader)) {
|
|
2910
|
+
fresh = this.install(key, this.openForTarget(ref, target));
|
|
2911
|
+
reader = await fresh.reader;
|
|
2912
|
+
if (reader === null) return null;
|
|
2913
|
+
}
|
|
2914
|
+
opened = fresh !== current;
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
if (opened) {
|
|
2918
|
+
const held = this.heldKey(ref, version, reader.fingerprint);
|
|
2919
|
+
this.replacedPins.delete(held);
|
|
2920
|
+
this.checking.delete(held);
|
|
2921
|
+
}
|
|
2922
|
+
return { generation: target.generation, version, fingerprint: reader.fingerprint };
|
|
2923
|
+
}
|
|
2924
|
+
/**
|
|
2925
|
+
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
2926
|
+
* a name purged and loaded again gets a new row, and with it a new token. Without one the version is the bare
|
|
2927
|
+
* generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
|
|
2928
|
+
*/
|
|
2929
|
+
pinnedKey(ref, version, fingerprint) {
|
|
2930
|
+
return this.registry === void 0 && fingerprint !== void 0 ? this.heldKey(ref, version, fingerprint) : `${segmentKey(ref)}@${version}`;
|
|
2931
|
+
}
|
|
2932
|
+
/**
|
|
2933
|
+
* One pin's hold on one segment: its version, and the fingerprint of the object it pinned. A check of that object
|
|
2934
|
+
* is shared under it, and what the check finds is kept under it, with a registry or without one: pins of one
|
|
2935
|
+
* version can hold different objects, since a pin built by hand can name any, and an object replaced from outside
|
|
2936
|
+
* the store leaves the row, and so the version, as it was.
|
|
2937
|
+
*/
|
|
2938
|
+
heldKey(ref, version, fingerprint) {
|
|
2939
|
+
return `${segmentKey(ref)}@${version}#${fingerprint}`;
|
|
2427
2940
|
}
|
|
2428
2941
|
/**
|
|
2429
2942
|
* A reader for one **specific** generation, memoized in the same bounded LRU as the live snapshots under a
|
|
@@ -2431,60 +2944,161 @@ var CrbmStorageChunkSource = class {
|
|
|
2431
2944
|
*
|
|
2432
2945
|
* Sharing the LRU is the point, not an implementation detail. A pinned reader held in a private field is
|
|
2433
2946
|
* outside the memory ceiling the library advertises — measured at 10.34 MiB per live pin, 32 pins holding
|
|
2434
|
-
* 111.7 MiB against an 8 MiB configured bound. A pinned *generation number* has no such problem:
|
|
2435
|
-
*
|
|
2436
|
-
*
|
|
2947
|
+
* 111.7 MiB against an 8 MiB configured bound. A pinned *generation number* has no such problem: an object is
|
|
2948
|
+
* immutable, so eviction is harmless, and re-opening at the same number reopens the same bytes unless the name
|
|
2949
|
+
* was purged and loaded again, which the pin's fingerprint then says. It also removes a memoized-rejection bug by
|
|
2950
|
+
* construction — `install` forgets a promise that rejects,
|
|
2437
2951
|
* where a hand-rolled `this.reader ??= open()` cached the rejection for the life of the handle and made a pin
|
|
2438
2952
|
* the one read path with no resilience.
|
|
2439
2953
|
*
|
|
2440
2954
|
* **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
|
|
2441
2955
|
* crypto-shred must not keep unwrapping a DEK the shred destroyed; a destroyed segment resolves no generation
|
|
2442
|
-
* for anyone,
|
|
2956
|
+
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
2957
|
+
* the key it unwrapped until it is evicted or invalidated.
|
|
2958
|
+
*
|
|
2959
|
+
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
2960
|
+
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
2961
|
+
* so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
|
|
2962
|
+
* already cached from the old one. The row's token cannot tell them apart, since it moves on every write, a
|
|
2963
|
+
* publish included; the object's fingerprint can, and a pin whose object has been replaced fails, as one whose
|
|
2964
|
+
* generation has been swept does. The memo is keyed so that two pins of one generation number in two
|
|
2965
|
+
* incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
|
|
2443
2966
|
*/
|
|
2444
|
-
async readerAt(ref, generation) {
|
|
2445
|
-
const
|
|
2446
|
-
const
|
|
2447
|
-
|
|
2448
|
-
|
|
2967
|
+
async readerAt(ref, generation, version, fingerprint) {
|
|
2968
|
+
const at = version ?? String(generation);
|
|
2969
|
+
const key = this.pinnedKey(ref, at, fingerprint);
|
|
2970
|
+
const epoch = this.invalidations;
|
|
2971
|
+
let entry = this.snapshots.get(key);
|
|
2972
|
+
const opening = entry === void 0;
|
|
2973
|
+
if (entry === void 0) {
|
|
2974
|
+
if (fingerprint !== void 0 && this.replacedPins.get(this.heldKey(ref, at, fingerprint)) !== void 0)
|
|
2975
|
+
throw notThePinned(ref, generation);
|
|
2976
|
+
entry = this.install(key, this.openAt(ref, generation));
|
|
2977
|
+
}
|
|
2978
|
+
let reader;
|
|
2979
|
+
try {
|
|
2980
|
+
reader = await entry.reader;
|
|
2981
|
+
} catch (err) {
|
|
2982
|
+
if (fingerprint === void 0 || isTransientError(err) || isNotFoundError(err)) throw err;
|
|
2983
|
+
await this.throwIfReplaced(ref, generation, at, fingerprint);
|
|
2984
|
+
throw err;
|
|
2985
|
+
}
|
|
2986
|
+
if (reader === null) {
|
|
2987
|
+
if (version === void 0) return null;
|
|
2988
|
+
throw new NotFoundError(
|
|
2989
|
+
`segment "${ref.segment}" generation ${generation}, which this handle pinned, can no longer be read: its registry row is gone or destroyed`
|
|
2990
|
+
);
|
|
2991
|
+
}
|
|
2992
|
+
if (fingerprint !== void 0 && reader.fingerprint !== fingerprint) {
|
|
2993
|
+
if (this.invalidations === epoch)
|
|
2994
|
+
this.replacedPins.set(this.heldKey(ref, at, fingerprint), true);
|
|
2995
|
+
if (opening && this.snapshots.peek(key) === entry) this.snapshots.delete(key);
|
|
2996
|
+
throw notThePinned(ref, generation);
|
|
2997
|
+
}
|
|
2998
|
+
return reader;
|
|
2449
2999
|
}
|
|
2450
3000
|
async openAt(ref, generation) {
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
3001
|
+
if (this.registry === void 0) return this.openForTarget(ref, { generation });
|
|
3002
|
+
const record = await this.registry.get(ref);
|
|
3003
|
+
if (record === null || record.status === "destroyed") return null;
|
|
3004
|
+
return this.openForTarget(ref, {
|
|
3005
|
+
generation,
|
|
3006
|
+
lineage: record.token,
|
|
3007
|
+
wrappedDeks: record.wrappedDeks
|
|
3008
|
+
});
|
|
2458
3009
|
}
|
|
2459
|
-
/**
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
3010
|
+
/**
|
|
3011
|
+
* Read one chunk of a specific generation — the pinned read path. `held`, when given, is what a pin holds of the
|
|
3012
|
+
* object, and the read fails if the generation is now another object.
|
|
3013
|
+
*/
|
|
3014
|
+
async getChunkAt(ref, generation, held) {
|
|
3015
|
+
validateUserRef(ref);
|
|
3016
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
3017
|
+
if (reader === null) return null;
|
|
3018
|
+
const fingerprint = held?.fingerprint;
|
|
3019
|
+
if (held !== void 0 && fingerprint !== void 0 && this.replacedPins.get(this.heldKey(ref, held.version, fingerprint)) !== void 0)
|
|
3020
|
+
throw notThePinned(ref, generation);
|
|
3021
|
+
try {
|
|
3022
|
+
return await reader.getChunk(ref.chunkKey);
|
|
3023
|
+
} catch (err) {
|
|
3024
|
+
if (held === void 0 || fingerprint === void 0) throw err;
|
|
3025
|
+
if (!(isIntegrityError(err) || isValidationError(err))) throw err;
|
|
3026
|
+
await this.throwIfReplaced(ref, generation, held.version, fingerprint);
|
|
3027
|
+
throw err;
|
|
3028
|
+
}
|
|
2463
3029
|
}
|
|
2464
|
-
/**
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
3030
|
+
/**
|
|
3031
|
+
* Throws the pin's NotFoundError if the object under `generation`'s key is no longer the one the pin holds. Its
|
|
3032
|
+
* footer says, read with no key and no row ({@link CrbmReader.sameObject}), so an object written under a key this
|
|
3033
|
+
* store lacks answers the same way, and so does one gone from its key. A replacement is kept
|
|
3034
|
+
* ({@link replacedPins}), and a check under way is shared by the reads that ask at once; neither "gone" nor "the
|
|
3035
|
+
* same" is kept, since either can change. A transient fault is rethrown, for the store's retries to see for what it
|
|
3036
|
+
* is. A footer that cannot be read otherwise says nothing about which object is there, so this returns, and the
|
|
3037
|
+
* caller's own error stands.
|
|
3038
|
+
*/
|
|
3039
|
+
async throwIfReplaced(ref, generation, version, fingerprint) {
|
|
3040
|
+
const held = this.heldKey(ref, version, fingerprint);
|
|
3041
|
+
if (this.replacedPins.get(held) !== void 0) throw notThePinned(ref, generation);
|
|
3042
|
+
let check = this.checking.get(held);
|
|
3043
|
+
if (check === void 0) {
|
|
3044
|
+
const at = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3045
|
+
const ended = () => this.checking.get(held) === started && this.checking.delete(held);
|
|
3046
|
+
const started = CrbmReader.sameObject(
|
|
3047
|
+
storageBlobReader(this.driver, at),
|
|
3048
|
+
fingerprint
|
|
3049
|
+
).then(
|
|
3050
|
+
(same) => {
|
|
3051
|
+
if (ended() && !same) this.replacedPins.set(held, true);
|
|
3052
|
+
return !same;
|
|
3053
|
+
},
|
|
3054
|
+
(err) => {
|
|
3055
|
+
ended();
|
|
3056
|
+
if (isNotFoundError(err)) return true;
|
|
3057
|
+
throw err;
|
|
3058
|
+
}
|
|
3059
|
+
);
|
|
3060
|
+
this.checking.set(held, started);
|
|
3061
|
+
check = started;
|
|
3062
|
+
}
|
|
3063
|
+
let replaced;
|
|
3064
|
+
try {
|
|
3065
|
+
replaced = await check;
|
|
3066
|
+
} catch (err) {
|
|
3067
|
+
if (isTransientError(err)) throw err;
|
|
3068
|
+
return;
|
|
3069
|
+
}
|
|
3070
|
+
if (replaced) throw notThePinned(ref, generation);
|
|
2469
3071
|
}
|
|
2470
|
-
/** Chunk keys of a specific generation — the pinned shape read. */
|
|
2471
|
-
async listChunkKeysAt(ref, generation) {
|
|
2472
|
-
|
|
2473
|
-
const reader = await this.readerAt(ref, generation);
|
|
3072
|
+
/** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
|
|
3073
|
+
async listChunkKeysAt(ref, generation, held) {
|
|
3074
|
+
validateUserRef(ref);
|
|
3075
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
2474
3076
|
return reader === null ? [] : reader.chunkKeys();
|
|
2475
3077
|
}
|
|
2476
|
-
/**
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
3078
|
+
/**
|
|
3079
|
+
* Per-chunk cardinalities of a specific generation — powers a pinned `count()` with no payload reads. `held` as
|
|
3080
|
+
* for {@link getChunkAt}.
|
|
3081
|
+
*/
|
|
3082
|
+
async cardinalitiesAt(ref, generation, held) {
|
|
3083
|
+
validateUserRef(ref);
|
|
3084
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
2480
3085
|
return reader === null ? null : reader.cardinalities();
|
|
2481
3086
|
}
|
|
2482
|
-
/** Grounded size of a specific generation. */
|
|
2483
|
-
async sizeOfAt(ref, generation) {
|
|
2484
|
-
|
|
2485
|
-
const reader = await this.readerAt(ref, generation);
|
|
3087
|
+
/** Grounded size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
3088
|
+
async sizeOfAt(ref, generation, held) {
|
|
3089
|
+
validateUserRef(ref);
|
|
3090
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
2486
3091
|
return reader === null ? null : { sizeBytes: reader.sizeBytes };
|
|
2487
3092
|
}
|
|
3093
|
+
/**
|
|
3094
|
+
* Whether the segment is a registered name at all. With a registry that is exactly "a row exists" — including
|
|
3095
|
+
* a row with no generation yet (minted by `setRetention` before the first load) and a `destroyed` tombstone,
|
|
3096
|
+
* both of which are names somebody deliberately created. Without a registry there is no registration to
|
|
3097
|
+
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
3098
|
+
*
|
|
3099
|
+
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
3100
|
+
* tell apart.
|
|
3101
|
+
*/
|
|
2488
3102
|
async exists(ref) {
|
|
2489
3103
|
if (this.registry !== void 0) return await this.registry.get(ref) !== null;
|
|
2490
3104
|
for await (const key of this.driver.list(ref)) {
|
|
@@ -2515,7 +3129,7 @@ var CrbmStorageChunkSource = class {
|
|
|
2515
3129
|
generation: target.generation
|
|
2516
3130
|
};
|
|
2517
3131
|
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
2518
|
-
return
|
|
3132
|
+
return openChecked(this.driver, genKey, {
|
|
2519
3133
|
...this.readerOptions,
|
|
2520
3134
|
crypto,
|
|
2521
3135
|
lineage: target.lineage
|
|
@@ -2567,11 +3181,11 @@ var CrbmStorageChunkSource = class {
|
|
|
2567
3181
|
return this.withFreshSnapshot(ref, (reader) => reader.getChunk(ref.chunkKey), null);
|
|
2568
3182
|
}
|
|
2569
3183
|
async listChunkKeys(ref) {
|
|
2570
|
-
|
|
3184
|
+
validateUserRef(ref);
|
|
2571
3185
|
return this.withFreshSnapshot(ref, (reader) => reader.chunkKeys(), []);
|
|
2572
3186
|
}
|
|
2573
3187
|
async sizeOf(ref) {
|
|
2574
|
-
|
|
3188
|
+
validateUserRef(ref);
|
|
2575
3189
|
return this.withFreshSnapshot(
|
|
2576
3190
|
ref,
|
|
2577
3191
|
(reader) => ({ sizeBytes: reader.sizeBytes }),
|
|
@@ -2579,7 +3193,7 @@ var CrbmStorageChunkSource = class {
|
|
|
2579
3193
|
);
|
|
2580
3194
|
}
|
|
2581
3195
|
async cardinalities(ref) {
|
|
2582
|
-
|
|
3196
|
+
validateUserRef(ref);
|
|
2583
3197
|
return this.withFreshSnapshot(
|
|
2584
3198
|
ref,
|
|
2585
3199
|
(reader) => reader.cardinalities(),
|
|
@@ -2715,7 +3329,7 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
2715
3329
|
}
|
|
2716
3330
|
async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
2717
3331
|
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
2718
|
-
throw new ValidationError("requireEncryption:
|
|
3332
|
+
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
2719
3333
|
}
|
|
2720
3334
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
2721
3335
|
const byChunk = /* @__PURE__ */ new Map();
|
|
@@ -2768,7 +3382,7 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
2768
3382
|
if (pause !== null) await pause;
|
|
2769
3383
|
}
|
|
2770
3384
|
if (options.keystore !== void 0 && options.registry === void 0) {
|
|
2771
|
-
throw new ValidationError("
|
|
3385
|
+
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
2772
3386
|
}
|
|
2773
3387
|
const existing = options.registry !== void 0 ? await options.registry.get(key) : null;
|
|
2774
3388
|
if (existing?.status === "destroyed") {
|
|
@@ -2778,7 +3392,7 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
2778
3392
|
}
|
|
2779
3393
|
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
2780
3394
|
throw new KeyUnavailableError(
|
|
2781
|
-
`segment "${key.segment}" is encrypted but this
|
|
3395
|
+
`segment "${key.segment}" is encrypted but this load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
2782
3396
|
);
|
|
2783
3397
|
}
|
|
2784
3398
|
let crypto;
|
|
@@ -2827,7 +3441,16 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
2827
3441
|
return { size, sha256, chunkCount: chunks.length, cardinality, wrappedDeks: newWrapped };
|
|
2828
3442
|
}
|
|
2829
3443
|
function openGenerationReader(storage, key, crypto, options = {}) {
|
|
2830
|
-
return
|
|
3444
|
+
return openChecked(storage, key, { ...options, crypto });
|
|
3445
|
+
}
|
|
3446
|
+
async function openChecked(storage, key, options) {
|
|
3447
|
+
const reader = await CrbmReader.open(storageBlobReader(storage, key), options);
|
|
3448
|
+
if (reader.generation !== key.generation) {
|
|
3449
|
+
throw new IntegrityError(
|
|
3450
|
+
`segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
|
|
3451
|
+
);
|
|
3452
|
+
}
|
|
3453
|
+
return reader;
|
|
2831
3454
|
}
|
|
2832
3455
|
async function verifyGeneration(storage, key, expected, crypto) {
|
|
2833
3456
|
const expectedKeys = [...expected.chunkKeys].sort((a, b) => a - b);
|
|
@@ -2847,6 +3470,10 @@ async function verifyGeneration(storage, key, expected, crypto) {
|
|
|
2847
3470
|
}
|
|
2848
3471
|
|
|
2849
3472
|
// src/core/pinned-storage-source.ts
|
|
3473
|
+
var heldBy = (pin) => {
|
|
3474
|
+
if (pin.version === null) return void 0;
|
|
3475
|
+
return pin.fingerprint == null ? { version: pin.version } : { version: pin.version, fingerprint: pin.fingerprint };
|
|
3476
|
+
};
|
|
2850
3477
|
var PinnedStorageChunkSource = class {
|
|
2851
3478
|
constructor(inner, pins) {
|
|
2852
3479
|
this.inner = inner;
|
|
@@ -2854,6 +3481,10 @@ var PinnedStorageChunkSource = class {
|
|
|
2854
3481
|
}
|
|
2855
3482
|
inner;
|
|
2856
3483
|
pins;
|
|
3484
|
+
/** The unpinned segments refresh as the store's own source does; a pinned one never does. */
|
|
3485
|
+
get pointerRefreshMs() {
|
|
3486
|
+
return this.inner.pointerRefreshMs;
|
|
3487
|
+
}
|
|
2857
3488
|
/**
|
|
2858
3489
|
* The pin for this segment, or `undefined` if it is not pinned here.
|
|
2859
3490
|
*
|
|
@@ -2867,36 +3498,50 @@ var PinnedStorageChunkSource = class {
|
|
|
2867
3498
|
getChunk(ref) {
|
|
2868
3499
|
const pin = this.pinFor(ref);
|
|
2869
3500
|
if (pin === void 0) return this.inner.getChunk(ref);
|
|
2870
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.getChunkAt(ref, pin.generation);
|
|
3501
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.getChunkAt(ref, pin.generation, heldBy(pin));
|
|
2871
3502
|
}
|
|
2872
3503
|
listChunkKeys(ref) {
|
|
2873
3504
|
const pin = this.pinFor(ref);
|
|
2874
3505
|
if (pin === void 0) return this.inner.listChunkKeys(ref);
|
|
2875
|
-
return pin.generation === null ? Promise.resolve([]) : this.inner.listChunkKeysAt(ref, pin.generation);
|
|
3506
|
+
return pin.generation === null ? Promise.resolve([]) : this.inner.listChunkKeysAt(ref, pin.generation, heldBy(pin));
|
|
2876
3507
|
}
|
|
2877
3508
|
sizeOf(ref) {
|
|
2878
3509
|
const pin = this.pinFor(ref);
|
|
2879
3510
|
if (pin === void 0) return this.inner.sizeOf(ref);
|
|
2880
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.sizeOfAt(ref, pin.generation);
|
|
3511
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.sizeOfAt(ref, pin.generation, heldBy(pin));
|
|
2881
3512
|
}
|
|
2882
3513
|
cardinalities(ref) {
|
|
2883
3514
|
const pin = this.pinFor(ref);
|
|
2884
3515
|
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
2885
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation);
|
|
3516
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
2886
3517
|
}
|
|
2887
3518
|
currentGeneration(ref) {
|
|
2888
3519
|
const pin = this.pinFor(ref);
|
|
2889
3520
|
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
2890
3521
|
}
|
|
2891
3522
|
/**
|
|
2892
|
-
* A pinned segment reports the version captured when it was pinned, so its decoded chunks
|
|
2893
|
-
*
|
|
2894
|
-
*
|
|
2895
|
-
* `
|
|
3523
|
+
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
3524
|
+
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
3525
|
+
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
3526
|
+
* and a lapsed `cache.genTtlMs`, a reader-cache eviction, a sweep that heals the read forward, or an
|
|
3527
|
+
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
3528
|
+
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
3529
|
+
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
3530
|
+
* read fetches exactly its own generation, so the entries it fills are always that generation's.
|
|
3531
|
+
*
|
|
3532
|
+
* The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
|
|
3533
|
+
* live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
|
|
3534
|
+
* though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
|
|
3535
|
+
* the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
|
|
3536
|
+
* key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
|
|
2896
3537
|
*/
|
|
2897
3538
|
currentVersion(ref) {
|
|
2898
3539
|
const pin = this.pinFor(ref);
|
|
2899
|
-
|
|
3540
|
+
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
3541
|
+
if (pin.version === null) return Promise.resolve(null);
|
|
3542
|
+
return Promise.resolve(
|
|
3543
|
+
`pin ${pin.version}${pin.fingerprint == null ? "" : `#${pin.fingerprint}`}`
|
|
3544
|
+
);
|
|
2900
3545
|
}
|
|
2901
3546
|
exists(ref) {
|
|
2902
3547
|
return this.inner.exists(ref);
|
|
@@ -2916,7 +3561,10 @@ async function nextGeneration(ref, deps) {
|
|
|
2916
3561
|
return highest + 1;
|
|
2917
3562
|
}
|
|
2918
3563
|
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
2919
|
-
const keep =
|
|
3564
|
+
const keep = options.keep ?? 1;
|
|
3565
|
+
if (!Number.isInteger(keep) || keep < 0) {
|
|
3566
|
+
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
3567
|
+
}
|
|
2920
3568
|
const record = await deps.registry.get(ref);
|
|
2921
3569
|
if (record === null) return [];
|
|
2922
3570
|
const current = record.currentGen;
|
|
@@ -2987,7 +3635,8 @@ async function currentCardinality(ref, deps, record) {
|
|
|
2987
3635
|
return total;
|
|
2988
3636
|
}
|
|
2989
3637
|
async function loadSegment(ref, ids, deps, options = {}) {
|
|
2990
|
-
|
|
3638
|
+
validateUserRef(ref);
|
|
3639
|
+
requireCodec(deps.codec, "loadSegment");
|
|
2991
3640
|
const keep = options.keep ?? 1;
|
|
2992
3641
|
if (!Number.isInteger(keep) || keep < 0) {
|
|
2993
3642
|
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
@@ -3027,6 +3676,14 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3027
3676
|
});
|
|
3028
3677
|
} catch (err) {
|
|
3029
3678
|
if (!isWriteConflictError(err)) throw err;
|
|
3679
|
+
audit.onEvent({
|
|
3680
|
+
kind: "segment.load-refused",
|
|
3681
|
+
segment: ref.segment,
|
|
3682
|
+
namespace: ref.namespace,
|
|
3683
|
+
generation,
|
|
3684
|
+
reason: "superseded",
|
|
3685
|
+
cardinality: 0
|
|
3686
|
+
});
|
|
3030
3687
|
return {
|
|
3031
3688
|
generation,
|
|
3032
3689
|
published: false,
|
|
@@ -3106,7 +3763,7 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3106
3763
|
|
|
3107
3764
|
// src/core/rollback.ts
|
|
3108
3765
|
async function listGenerations(ref, deps) {
|
|
3109
|
-
|
|
3766
|
+
validateUserRef(ref);
|
|
3110
3767
|
const record = await deps.registry.get(ref);
|
|
3111
3768
|
const current = record?.currentGen ?? null;
|
|
3112
3769
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -3114,7 +3771,7 @@ async function listGenerations(ref, deps) {
|
|
|
3114
3771
|
return [...seen].sort((a, b) => a - b).map((generation) => ({ generation, current: generation === current }));
|
|
3115
3772
|
}
|
|
3116
3773
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
3117
|
-
|
|
3774
|
+
validateUserRef(ref);
|
|
3118
3775
|
if (!Number.isInteger(toGeneration) || toGeneration < 0) {
|
|
3119
3776
|
throw new ValidationError(
|
|
3120
3777
|
`rollback: generation must be a non-negative integer; got ${String(toGeneration)}`
|
|
@@ -3163,7 +3820,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3163
3820
|
} catch {
|
|
3164
3821
|
}
|
|
3165
3822
|
throw new NotFoundError(
|
|
3166
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" was collected while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the
|
|
3823
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" was collected while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
3167
3824
|
);
|
|
3168
3825
|
}
|
|
3169
3826
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -3178,17 +3835,8 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3178
3835
|
|
|
3179
3836
|
// src/core/erase-id.ts
|
|
3180
3837
|
var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
|
|
3181
|
-
async function assertCollected(ref, holder, collected, storage) {
|
|
3182
|
-
if (collected.includes(holder)) return;
|
|
3183
|
-
for await (const key of storage.list(ref)) {
|
|
3184
|
-
if (key.generation !== holder) continue;
|
|
3185
|
-
throw new WriteConflictError(
|
|
3186
|
-
`erasure of segment ${ref.segment} could not remove generation ${holder} \u2014 the one holding the id \u2014 which is still in the bucket; re-run`
|
|
3187
|
-
);
|
|
3188
|
-
}
|
|
3189
|
-
}
|
|
3190
3838
|
async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
3191
|
-
|
|
3839
|
+
validateUserRef(ref);
|
|
3192
3840
|
const { chunkKey, remainder } = splitId(id);
|
|
3193
3841
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
3194
3842
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
|
|
@@ -3228,50 +3876,89 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3228
3876
|
if (row.currentGen === null) return "no-generation";
|
|
3229
3877
|
return row.currentGen === from && row.token === fromToken ? null : "superseded";
|
|
3230
3878
|
};
|
|
3231
|
-
const
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3879
|
+
const holds = async (generation2) => {
|
|
3880
|
+
try {
|
|
3881
|
+
const reader = await openGenerationReader(
|
|
3882
|
+
deps.storage,
|
|
3883
|
+
{ ...base, generation: generation2 },
|
|
3884
|
+
cryptoAt(generation2)
|
|
3885
|
+
);
|
|
3886
|
+
const bytes = await reader.getChunk(chunkKey);
|
|
3887
|
+
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
3888
|
+
} catch (err) {
|
|
3889
|
+
if (isNotFoundError(err)) return null;
|
|
3890
|
+
throw err;
|
|
3235
3891
|
}
|
|
3236
|
-
|
|
3237
|
-
|
|
3892
|
+
};
|
|
3893
|
+
const holderLeft = async (clean) => {
|
|
3894
|
+
const present = /* @__PURE__ */ new Set();
|
|
3895
|
+
for await (const key2 of deps.storage.list(ref)) present.add(key2.generation);
|
|
3896
|
+
for (const generation2 of [...present].sort((a, b) => b - a)) {
|
|
3897
|
+
if (!clean.has(generation2) && await holds(generation2) === true) return generation2;
|
|
3238
3898
|
}
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
if (
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3899
|
+
return void 0;
|
|
3900
|
+
};
|
|
3901
|
+
const cannotRemove = (generation2) => new WriteConflictError(
|
|
3902
|
+
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which still holds the id and is still in the bucket; re-run`
|
|
3903
|
+
);
|
|
3904
|
+
const notInCurrent = async () => {
|
|
3905
|
+
const notMember = {
|
|
3906
|
+
...base,
|
|
3907
|
+
erased: false,
|
|
3908
|
+
reason: "not-member",
|
|
3909
|
+
fromGeneration: from,
|
|
3910
|
+
collected: []
|
|
3911
|
+
};
|
|
3912
|
+
const others = /* @__PURE__ */ new Set();
|
|
3913
|
+
for await (const key2 of deps.storage.list(ref)) {
|
|
3914
|
+
if (key2.generation !== from) others.add(key2.generation);
|
|
3915
|
+
}
|
|
3916
|
+
if (others.size === 0) return notMember;
|
|
3917
|
+
const newestFirst = [...others].sort((a, b) => b - a);
|
|
3918
|
+
const clean = /* @__PURE__ */ new Set([from]);
|
|
3919
|
+
const holdersAbove = [];
|
|
3920
|
+
for (const generation2 of newestFirst.filter((g) => g > from)) {
|
|
3921
|
+
const held = await holds(generation2);
|
|
3922
|
+
if (held === true) holdersAbove.push(generation2);
|
|
3923
|
+
else if (held === false) clean.add(generation2);
|
|
3924
|
+
}
|
|
3925
|
+
let holderBelow;
|
|
3926
|
+
if (holdersAbove.length === 0) {
|
|
3927
|
+
for (const generation2 of newestFirst.filter((g) => g < from)) {
|
|
3928
|
+
const held = await holds(generation2);
|
|
3929
|
+
if (held === true) {
|
|
3930
|
+
holderBelow = generation2;
|
|
3931
|
+
break;
|
|
3932
|
+
}
|
|
3933
|
+
if (held === false) clean.add(generation2);
|
|
3260
3934
|
}
|
|
3261
|
-
await assertCollected(ref, generation2, alsoCollected, deps.storage);
|
|
3262
|
-
return { ...base, erased: true, fromGeneration: generation2, collected: alsoCollected };
|
|
3263
3935
|
}
|
|
3264
|
-
|
|
3936
|
+
const newest = holdersAbove[0] ?? holderBelow;
|
|
3937
|
+
if (newest === void 0) return notMember;
|
|
3938
|
+
const collected2 = [...await gcOrphanGenerations(ref, deps, { keep: 0 })];
|
|
3939
|
+
let moved = null;
|
|
3940
|
+
for (const generation2 of holdersAbove) {
|
|
3941
|
+
moved = rowVerdict(await deps.registry.get(ref));
|
|
3942
|
+
if (moved !== null) break;
|
|
3943
|
+
await deps.storage.delete({ ...base, generation: generation2 });
|
|
3944
|
+
collected2.push(generation2);
|
|
3945
|
+
}
|
|
3946
|
+
const left2 = await holderLeft(clean);
|
|
3947
|
+
if (left2 === void 0) return { ...base, erased: true, fromGeneration: newest, collected: collected2 };
|
|
3948
|
+
if (moved !== null) {
|
|
3949
|
+
return { ...base, erased: false, reason: moved, fromGeneration: newest, collected: collected2 };
|
|
3950
|
+
}
|
|
3951
|
+
throw cannotRemove(left2);
|
|
3265
3952
|
};
|
|
3266
3953
|
const stage = async () => {
|
|
3267
3954
|
let written;
|
|
3268
3955
|
try {
|
|
3269
3956
|
const reader = await openGenerationReader(deps.storage, fromKey, cryptoAt(from));
|
|
3270
3957
|
const bytes = await reader.getChunk(chunkKey);
|
|
3271
|
-
if (bytes === null) return notInCurrent();
|
|
3958
|
+
if (bytes === null) return await notInCurrent();
|
|
3272
3959
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
3273
3960
|
assertRemaindersInRange(target, chunkKey);
|
|
3274
|
-
if (!target.has(remainder)) return notInCurrent();
|
|
3961
|
+
if (!target.has(remainder)) return await notInCurrent();
|
|
3275
3962
|
target.remove(remainder);
|
|
3276
3963
|
const generation2 = await nextGeneration(ref, deps);
|
|
3277
3964
|
const key2 = { ...base, generation: generation2 };
|
|
@@ -3300,14 +3987,24 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3300
3987
|
throw err;
|
|
3301
3988
|
}
|
|
3302
3989
|
};
|
|
3990
|
+
const discardRefused = async (written) => {
|
|
3991
|
+
const row = await deps.registry.get(ref);
|
|
3992
|
+
if (row === null || row.status !== "active" || row.currentGen === null) return;
|
|
3993
|
+
if (row.currentGen <= from || row.currentGen >= written) return;
|
|
3994
|
+
await deps.storage.delete({ ...base, generation: written });
|
|
3995
|
+
};
|
|
3303
3996
|
const staged = await stage();
|
|
3304
|
-
if ("erased" in staged)
|
|
3997
|
+
if ("erased" in staged) {
|
|
3998
|
+
if (!staged.erased && staged.generation !== void 0) await discardRefused(staged.generation);
|
|
3999
|
+
return staged;
|
|
4000
|
+
}
|
|
3305
4001
|
const { generation, key } = staged;
|
|
3306
4002
|
const published = await publishGeneration(deps.registry, key, {
|
|
3307
4003
|
expectFrom: from,
|
|
3308
4004
|
expectToken: fromToken
|
|
3309
4005
|
});
|
|
3310
4006
|
if (!published) {
|
|
4007
|
+
await discardRefused(generation);
|
|
3311
4008
|
return {
|
|
3312
4009
|
...base,
|
|
3313
4010
|
erased: false,
|
|
@@ -3325,7 +4022,8 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3325
4022
|
generation
|
|
3326
4023
|
});
|
|
3327
4024
|
const collected = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
3328
|
-
await
|
|
4025
|
+
const left = await holderLeft(/* @__PURE__ */ new Set([generation]));
|
|
4026
|
+
if (left !== void 0) throw cannotRemove(left);
|
|
3329
4027
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
3330
4028
|
}
|
|
3331
4029
|
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes) {
|
|
@@ -3456,6 +4154,7 @@ var MAX_CAS_ATTEMPTS = 8;
|
|
|
3456
4154
|
var ERASE_CONCURRENCY = 8;
|
|
3457
4155
|
var MAX_STORAGE_SWEEPS = 3;
|
|
3458
4156
|
async function destroySegment(ref, deps, options) {
|
|
4157
|
+
validateUserRef(ref);
|
|
3459
4158
|
if (options.confirmSegment !== ref.segment) {
|
|
3460
4159
|
throw new ValidationError(
|
|
3461
4160
|
`destroySegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental crypto-shred)`
|
|
@@ -3475,15 +4174,23 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
3475
4174
|
if (typeof namespace !== "string" || namespace.length === 0) {
|
|
3476
4175
|
throw new ValidationError("eraseNamespace: namespace must be a non-empty string");
|
|
3477
4176
|
}
|
|
4177
|
+
validateUserNamespace(namespace);
|
|
3478
4178
|
if (options.confirmNamespace !== namespace) {
|
|
3479
4179
|
throw new ValidationError(
|
|
3480
4180
|
`eraseNamespace: confirmNamespace must equal the namespace "${namespace}" (guard against accidental erasure)`
|
|
3481
4181
|
);
|
|
3482
4182
|
}
|
|
3483
|
-
const
|
|
3484
|
-
|
|
3485
|
-
|
|
3486
|
-
|
|
4183
|
+
const rows = await drainRegistry(deps.registry, {
|
|
4184
|
+
namespace,
|
|
4185
|
+
maxScanSegments: options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS,
|
|
4186
|
+
op: "eraseNamespace",
|
|
4187
|
+
narrowable: false
|
|
4188
|
+
// already one namespace: the only remedy is a higher ceiling
|
|
4189
|
+
});
|
|
4190
|
+
const refs = rows.map((rec) => ({
|
|
4191
|
+
namespace: rec.namespace,
|
|
4192
|
+
segment: rec.segment
|
|
4193
|
+
}));
|
|
3487
4194
|
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
3488
4195
|
const destroyed = [];
|
|
3489
4196
|
let segmentsShredded = 0;
|
|
@@ -3510,6 +4217,7 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
3510
4217
|
return { destroyed };
|
|
3511
4218
|
}
|
|
3512
4219
|
async function dropSegment(ref, deps, options) {
|
|
4220
|
+
validateUserRef(ref);
|
|
3513
4221
|
if (options.confirmSegment !== ref.segment) {
|
|
3514
4222
|
throw new ValidationError(
|
|
3515
4223
|
`dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
|
|
@@ -3663,6 +4371,7 @@ function readRetentionPolicy(meta) {
|
|
|
3663
4371
|
return { expiresAt: raw };
|
|
3664
4372
|
}
|
|
3665
4373
|
async function getSegmentRetention(ref, deps) {
|
|
4374
|
+
validateUserRef(ref);
|
|
3666
4375
|
const record = await deps.registry.get(ref);
|
|
3667
4376
|
if (record === null || record.status === "destroyed") return null;
|
|
3668
4377
|
return readRetentionPolicy(record.retention);
|
|
@@ -3693,6 +4402,7 @@ async function reindex(ref, deps, expiresAt, previousExpiresAt) {
|
|
|
3693
4402
|
return indexed;
|
|
3694
4403
|
}
|
|
3695
4404
|
async function setSegmentRetention(ref, deps, policy) {
|
|
4405
|
+
validateUserRef(ref);
|
|
3696
4406
|
validateRetentionPolicy(policy);
|
|
3697
4407
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
3698
4408
|
let previousExpiresAt;
|
|
@@ -3731,6 +4441,7 @@ async function setSegmentRetention(ref, deps, policy) {
|
|
|
3731
4441
|
);
|
|
3732
4442
|
}
|
|
3733
4443
|
async function clearSegmentRetention(ref, deps) {
|
|
4444
|
+
validateUserRef(ref);
|
|
3734
4445
|
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
3735
4446
|
const record = await deps.registry.get(ref);
|
|
3736
4447
|
if (record === null) return false;
|
|
@@ -3771,6 +4482,7 @@ async function rowsFromDueIndex(registry, options) {
|
|
|
3771
4482
|
for await (const pointer of registry.list(dueNamespace(bucket))) {
|
|
3772
4483
|
const ref = decodeDueName(pointer.segment);
|
|
3773
4484
|
if (ref === null) continue;
|
|
4485
|
+
if (isReservedNamespace(ref.namespace)) continue;
|
|
3774
4486
|
if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
|
|
3775
4487
|
const key = `${ref.namespace ?? ""}\0${ref.segment}`;
|
|
3776
4488
|
if (seen.has(key)) continue;
|
|
@@ -3795,6 +4507,7 @@ async function forgetDuePointer(registry, ref, expiresAt) {
|
|
|
3795
4507
|
}
|
|
3796
4508
|
}
|
|
3797
4509
|
async function retireExpired(deps, options) {
|
|
4510
|
+
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
3798
4511
|
const now = options.now;
|
|
3799
4512
|
if (!Number.isFinite(now)) {
|
|
3800
4513
|
throw new ValidationError(
|
|
@@ -3867,7 +4580,7 @@ async function retireExpired(deps, options) {
|
|
|
3867
4580
|
continue;
|
|
3868
4581
|
}
|
|
3869
4582
|
}
|
|
3870
|
-
if (!dryRun) await deps.registry.delete(ref);
|
|
4583
|
+
if (!dryRun) await deps.registry.delete(ref, rec.token);
|
|
3871
4584
|
if (!dryRun) tombstonesPurged += 1;
|
|
3872
4585
|
entries.push({
|
|
3873
4586
|
...base,
|
|
@@ -3918,9 +4631,15 @@ async function retireExpired(deps, options) {
|
|
|
3918
4631
|
retired += 1;
|
|
3919
4632
|
entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
3920
4633
|
await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt);
|
|
3921
|
-
if (result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
3922
|
-
await deps.registry.
|
|
3923
|
-
|
|
4634
|
+
if (purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
4635
|
+
const tombstone = await deps.registry.get(ref).catch(() => null);
|
|
4636
|
+
if (tombstone?.status === "destroyed") {
|
|
4637
|
+
const deleted = await deps.registry.delete(ref, tombstone.token).then(
|
|
4638
|
+
() => true,
|
|
4639
|
+
() => false
|
|
4640
|
+
);
|
|
4641
|
+
if (deleted) continue;
|
|
4642
|
+
}
|
|
3924
4643
|
}
|
|
3925
4644
|
await stampRetirement(deps.registry, ref, now).catch(() => void 0);
|
|
3926
4645
|
} catch (err) {
|
|
@@ -4000,6 +4719,7 @@ async function generationsPresent(storage, ref) {
|
|
|
4000
4719
|
return present;
|
|
4001
4720
|
}
|
|
4002
4721
|
async function runConsistencyCheck(deps, options = {}) {
|
|
4722
|
+
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
4003
4723
|
const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
|
|
4004
4724
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
4005
4725
|
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
@@ -4008,7 +4728,10 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4008
4728
|
const recs = await drainRegistry(deps.registry, {
|
|
4009
4729
|
namespace: options.namespace,
|
|
4010
4730
|
maxScanSegments,
|
|
4011
|
-
op: "checkConsistency"
|
|
4731
|
+
op: "checkConsistency",
|
|
4732
|
+
// Two callers reach this refusal, and only one of them can raise the ceiling: `store.checkConsistency` takes
|
|
4733
|
+
// no `maxScanSegments`, so the advice names the call that does.
|
|
4734
|
+
raise: "raise `maxScanSegments` on `runConsistencyCheck` (`store.checkConsistency` takes no ceiling, so run `runConsistencyCheck({ storage: backend.storage, registry: backend.registry }, { maxScanSegments })` over the same backend instead)"
|
|
4012
4735
|
});
|
|
4013
4736
|
const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
|
|
4014
4737
|
if (rec.status === "destroyed") return { kind: "ok" };
|
|
@@ -4080,14 +4803,17 @@ async function withRetry(op, policy, deps) {
|
|
|
4080
4803
|
lastErr = err;
|
|
4081
4804
|
if (attempt >= attempts || !retryable(err)) throw err;
|
|
4082
4805
|
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
4083
|
-
|
|
4806
|
+
try {
|
|
4807
|
+
deps.onRetry?.({ attempt, delayMs, err });
|
|
4808
|
+
} catch {
|
|
4809
|
+
}
|
|
4084
4810
|
await deps.clock.sleep(delayMs);
|
|
4085
4811
|
}
|
|
4086
4812
|
}
|
|
4087
4813
|
throw lastErr;
|
|
4088
4814
|
}
|
|
4089
4815
|
|
|
4090
|
-
// src/drivers/retry/retrying-
|
|
4816
|
+
// src/drivers/retry/retrying-chunk-source.ts
|
|
4091
4817
|
function toRetry(opts) {
|
|
4092
4818
|
return {
|
|
4093
4819
|
policy: opts.policy ?? DEFAULT_RETRY_POLICY,
|
|
@@ -4110,6 +4836,13 @@ var RetryingStorageChunkSource = class {
|
|
|
4110
4836
|
invalidate;
|
|
4111
4837
|
exists;
|
|
4112
4838
|
currentVersion;
|
|
4839
|
+
/**
|
|
4840
|
+
* Forwarded, not retried: a property, with no request behind it. Without it the grounded cost report priced a
|
|
4841
|
+
* retrying store's refresh at the default TTL whatever the store was configured with.
|
|
4842
|
+
*/
|
|
4843
|
+
get pointerRefreshMs() {
|
|
4844
|
+
return this.inner.pointerRefreshMs;
|
|
4845
|
+
}
|
|
4113
4846
|
constructor(inner, opts) {
|
|
4114
4847
|
this.inner = inner;
|
|
4115
4848
|
const r = toRetry(opts);
|
|
@@ -4149,82 +4882,6 @@ var RetryingStorageChunkSource = class {
|
|
|
4149
4882
|
return withRetry(() => this.inner.listChunkKeys(ref), this.policy, this.deps);
|
|
4150
4883
|
}
|
|
4151
4884
|
};
|
|
4152
|
-
var RetryingStorageDriver = class {
|
|
4153
|
-
inner;
|
|
4154
|
-
policy;
|
|
4155
|
-
deps;
|
|
4156
|
-
constructor(inner, opts) {
|
|
4157
|
-
this.inner = inner;
|
|
4158
|
-
const r = toRetry(opts);
|
|
4159
|
-
this.policy = r.policy;
|
|
4160
|
-
this.deps = r.deps;
|
|
4161
|
-
}
|
|
4162
|
-
capabilities() {
|
|
4163
|
-
return this.inner.capabilities();
|
|
4164
|
-
}
|
|
4165
|
-
putImmutable(key, write) {
|
|
4166
|
-
return withRetry(() => this.inner.putImmutable(key, write), this.policy, this.deps);
|
|
4167
|
-
}
|
|
4168
|
-
getRange(key, offset, length) {
|
|
4169
|
-
return withRetry(() => this.inner.getRange(key, offset, length), this.policy, this.deps);
|
|
4170
|
-
}
|
|
4171
|
-
getTail(key, maxBytes) {
|
|
4172
|
-
return withRetry(() => this.inner.getTail(key, maxBytes), this.policy, this.deps);
|
|
4173
|
-
}
|
|
4174
|
-
delete(key) {
|
|
4175
|
-
return withRetry(() => this.inner.delete(key), this.policy, this.deps);
|
|
4176
|
-
}
|
|
4177
|
-
async *list(ref) {
|
|
4178
|
-
const keys = await withRetry(
|
|
4179
|
-
async () => {
|
|
4180
|
-
const out = [];
|
|
4181
|
-
for await (const k of this.inner.list(ref)) out.push(k);
|
|
4182
|
-
return out;
|
|
4183
|
-
},
|
|
4184
|
-
this.policy,
|
|
4185
|
-
this.deps
|
|
4186
|
-
);
|
|
4187
|
-
yield* keys;
|
|
4188
|
-
}
|
|
4189
|
-
};
|
|
4190
|
-
var RetryingRegistryDriver = class {
|
|
4191
|
-
inner;
|
|
4192
|
-
policy;
|
|
4193
|
-
deps;
|
|
4194
|
-
constructor(inner, opts) {
|
|
4195
|
-
this.inner = inner;
|
|
4196
|
-
const r = toRetry(opts);
|
|
4197
|
-
this.policy = r.policy;
|
|
4198
|
-
this.deps = r.deps;
|
|
4199
|
-
}
|
|
4200
|
-
capabilities() {
|
|
4201
|
-
return this.inner.capabilities();
|
|
4202
|
-
}
|
|
4203
|
-
get(ref) {
|
|
4204
|
-
return withRetry(() => this.inner.get(ref), this.policy, this.deps);
|
|
4205
|
-
}
|
|
4206
|
-
create(ref, record) {
|
|
4207
|
-
return withRetry(() => this.inner.create(ref, record), this.policy, this.deps);
|
|
4208
|
-
}
|
|
4209
|
-
compareAndSwap(ref, expected, patch) {
|
|
4210
|
-
return withRetry(() => this.inner.compareAndSwap(ref, expected, patch), this.policy, this.deps);
|
|
4211
|
-
}
|
|
4212
|
-
async *list(namespace) {
|
|
4213
|
-
const records = await withRetry(
|
|
4214
|
-
async () => {
|
|
4215
|
-
const out = [];
|
|
4216
|
-
for await (const r of this.inner.list(namespace)) out.push(r);
|
|
4217
|
-
return out;
|
|
4218
|
-
},
|
|
4219
|
-
this.policy,
|
|
4220
|
-
this.deps
|
|
4221
|
-
);
|
|
4222
|
-
yield* records;
|
|
4223
|
-
}
|
|
4224
|
-
delete(ref) {
|
|
4225
|
-
return withRetry(() => this.inner.delete(ref), this.policy, this.deps);
|
|
4226
|
-
}
|
|
4227
|
-
};
|
|
4228
4885
|
export {
|
|
4229
4886
|
AWS_US_EAST_1_ONDEMAND,
|
|
4230
4887
|
BoundedLru,
|
|
@@ -4236,47 +4893,36 @@ export {
|
|
|
4236
4893
|
CrbmReader,
|
|
4237
4894
|
CrbmStorageChunkSource,
|
|
4238
4895
|
DEFAULT_BUDGET,
|
|
4239
|
-
DEFAULT_PRICING,
|
|
4240
4896
|
DEFAULT_RETRY_POLICY,
|
|
4897
|
+
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
4241
4898
|
InProcessKeystore,
|
|
4242
4899
|
IntegrityError,
|
|
4243
4900
|
KeyUnavailableError,
|
|
4244
|
-
LocalFsRegistryDriver,
|
|
4245
4901
|
LocalFsStorage,
|
|
4246
|
-
LocalFsStorageDriver,
|
|
4247
4902
|
MIN_EXPIRES_AT_MS,
|
|
4248
|
-
MemoryRegistryDriver,
|
|
4249
4903
|
MemoryStorage,
|
|
4250
|
-
MemoryStorageChunkSource,
|
|
4251
|
-
MemoryStorageDriver,
|
|
4252
4904
|
NOOP_METRICS,
|
|
4253
4905
|
NodeAead,
|
|
4254
4906
|
NotFoundError,
|
|
4907
|
+
ONE_REDIS_HA_CLUSTER,
|
|
4255
4908
|
PinnedStorageChunkSource,
|
|
4256
4909
|
RecordingAuditSink,
|
|
4257
|
-
RetryingRegistryDriver,
|
|
4258
4910
|
RetryingStorageChunkSource,
|
|
4259
|
-
RetryingStorageDriver,
|
|
4260
4911
|
SegmentEngine,
|
|
4261
|
-
TimeoutError,
|
|
4262
4912
|
TransientError,
|
|
4263
4913
|
UnsupportedError,
|
|
4264
4914
|
ValidationError,
|
|
4265
4915
|
WriteConflictError,
|
|
4266
4916
|
aadFor,
|
|
4267
|
-
bulkLoadCrbmGeneration,
|
|
4268
4917
|
checkBudget,
|
|
4269
4918
|
clearSegmentRetention,
|
|
4270
4919
|
collectWithinBudget,
|
|
4271
|
-
createBackend,
|
|
4272
4920
|
destroySegment,
|
|
4273
4921
|
dropSegment,
|
|
4274
|
-
encodeNameForPath,
|
|
4275
4922
|
eraseIdFromSegment,
|
|
4276
4923
|
eraseNamespace,
|
|
4277
4924
|
estimateCost,
|
|
4278
4925
|
excludingReservedRows,
|
|
4279
|
-
gcOrphanGenerations,
|
|
4280
4926
|
getSegmentRetention,
|
|
4281
4927
|
groundedReport,
|
|
4282
4928
|
isCloudRoaringError,
|
|
@@ -4290,9 +4936,6 @@ export {
|
|
|
4290
4936
|
listSegments,
|
|
4291
4937
|
loadSegment,
|
|
4292
4938
|
mapWithConcurrency,
|
|
4293
|
-
namespacePathPart,
|
|
4294
|
-
nextGeneration,
|
|
4295
|
-
publishGeneration,
|
|
4296
4939
|
readRetentionPolicy,
|
|
4297
4940
|
resolveBudget,
|
|
4298
4941
|
resolvePerOpBudget,
|
|
@@ -4305,8 +4948,6 @@ export {
|
|
|
4305
4948
|
segmentKey,
|
|
4306
4949
|
setSegmentRetention,
|
|
4307
4950
|
splitId,
|
|
4308
|
-
|
|
4309
|
-
withRetry,
|
|
4310
|
-
writeCrbmGeneration
|
|
4951
|
+
withRetry
|
|
4311
4952
|
};
|
|
4312
4953
|
//# sourceMappingURL=index.js.map
|