@cloudbitmaps/core 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-TNWTIAON.js → chunk-YSC7NJWV.js} +314 -216
- package/dist/chunk-YSC7NJWV.js.map +7 -0
- package/dist/core/audit.d.ts +56 -3
- package/dist/core/combine-many.d.ts +13 -4
- package/dist/core/consistency.d.ts +10 -4
- package/dist/core/crbm/fingerprint.d.ts +18 -0
- package/dist/core/crbm/format.d.ts +2 -4
- package/dist/core/crbm/frame.d.ts +10 -0
- package/dist/core/crbm/reader.d.ts +1 -3
- package/dist/core/crbm-storage-source.d.ts +249 -96
- package/dist/core/engine.d.ts +22 -14
- package/dist/core/erase-id.d.ts +62 -26
- package/dist/core/generation-gc.d.ts +3 -2
- package/dist/core/keys.d.ts +28 -0
- package/dist/core/leases.d.ts +7 -0
- package/dist/core/load.d.ts +11 -7
- package/dist/core/metrics.d.ts +5 -5
- package/dist/core/pinned-storage-source.d.ts +3 -3
- package/dist/core/pointer-id.d.ts +19 -0
- package/dist/core/ports.d.ts +131 -35
- package/dist/core/reader-defaults.d.ts +14 -3
- package/dist/core/rollback.d.ts +2 -1
- package/dist/core/summary.d.ts +38 -21
- package/dist/core/token.d.ts +13 -8
- package/dist/driver-kit.d.ts +2 -1
- package/dist/driver-kit.js +11 -4
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +1 -1
- package/dist/drivers/_shared/registry.d.ts +26 -17
- package/dist/drivers/localfs/storage.d.ts +7 -6
- package/dist/drivers/memory.d.ts +8 -6
- package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
- package/dist/export/index.d.ts +16 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.js +1288 -1080
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-TNWTIAON.js.map +0 -7
- package/dist/core/cost.d.ts +0 -351
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
CapabilityError,
|
|
4
4
|
CloudRoaringError,
|
|
5
5
|
DEFAULT_NAMESPACE,
|
|
6
|
+
FOOTER_BYTES,
|
|
6
7
|
IntegrityError,
|
|
7
8
|
KeyUnavailableError,
|
|
8
9
|
LEASE_SKEW_MS,
|
|
@@ -14,9 +15,12 @@ import {
|
|
|
14
15
|
MAX_METADATA_BYTES,
|
|
15
16
|
MAX_NAME_LENGTH,
|
|
16
17
|
NotFoundError,
|
|
18
|
+
PREAMBLE_BYTES,
|
|
17
19
|
RESERVED_NAMESPACE_PREFIX,
|
|
18
20
|
StaleOperandError,
|
|
19
21
|
TransientError,
|
|
22
|
+
UNANSWERED_BASE_MS,
|
|
23
|
+
UNANSWERED_RESENDS,
|
|
20
24
|
UnsupportedError,
|
|
21
25
|
ValidationError,
|
|
22
26
|
WriteConflictError,
|
|
@@ -24,6 +28,7 @@ import {
|
|
|
24
28
|
assertRegistryCanWrite,
|
|
25
29
|
brandAsBackend,
|
|
26
30
|
canonicalMetadataJson,
|
|
31
|
+
compactFingerprint,
|
|
27
32
|
copiedMetadata,
|
|
28
33
|
decodeNameFromPath,
|
|
29
34
|
drawIncarnation,
|
|
@@ -32,7 +37,10 @@ import {
|
|
|
32
37
|
encodeNameForPath,
|
|
33
38
|
encodedNameLength,
|
|
34
39
|
entropyIsAvailable,
|
|
40
|
+
fingerprintFor,
|
|
41
|
+
fingerprintParts,
|
|
35
42
|
heldGenerations,
|
|
43
|
+
incarnationField,
|
|
36
44
|
incarnationOf,
|
|
37
45
|
isBudgetExceededError,
|
|
38
46
|
isCapabilityError,
|
|
@@ -60,10 +68,13 @@ import {
|
|
|
60
68
|
nextRegistryToken,
|
|
61
69
|
onlyLeasesDiffer,
|
|
62
70
|
parseRegistryEnvelope,
|
|
71
|
+
pointerIdOf,
|
|
63
72
|
recordFromNew,
|
|
64
73
|
releaseLease,
|
|
74
|
+
renewPointer,
|
|
65
75
|
sameIncarnation,
|
|
66
76
|
serializeRegistryEnvelope,
|
|
77
|
+
sizePart,
|
|
67
78
|
takeLease,
|
|
68
79
|
usableKeptGens,
|
|
69
80
|
validateChunkKeyOrder,
|
|
@@ -74,7 +85,7 @@ import {
|
|
|
74
85
|
validateUserNamespace,
|
|
75
86
|
validateUserRef,
|
|
76
87
|
webCryptoEntropy
|
|
77
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-YSC7NJWV.js";
|
|
78
89
|
|
|
79
90
|
// src/core/bit-route.ts
|
|
80
91
|
var U32_MAX = 4294967295;
|
|
@@ -252,9 +263,7 @@ var ChunkWindow = class {
|
|
|
252
263
|
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
253
264
|
var VERSION_MAJOR = 1;
|
|
254
265
|
var VERSION_MINOR = 0;
|
|
255
|
-
var PREAMBLE_BYTES = 8;
|
|
256
266
|
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
257
|
-
var FOOTER_BYTES = 104;
|
|
258
267
|
var FOOTER = {
|
|
259
268
|
indexOffset: 0,
|
|
260
269
|
// u64
|
|
@@ -329,11 +338,42 @@ function segmentKey(ref) {
|
|
|
329
338
|
function segmentPrefix(ref) {
|
|
330
339
|
return `${segmentKey(ref)}${FIELD}`;
|
|
331
340
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
341
|
+
var KEPT_SEGMENT_KEYS = 1024;
|
|
342
|
+
var KeptSegmentKeys = class {
|
|
343
|
+
constructor(encode, max = KEPT_SEGMENT_KEYS) {
|
|
344
|
+
this.encode = encode;
|
|
345
|
+
this.max = max;
|
|
346
|
+
}
|
|
347
|
+
encode;
|
|
348
|
+
max;
|
|
349
|
+
byNamespace = /* @__PURE__ */ new Map();
|
|
350
|
+
kept = 0;
|
|
351
|
+
/** The segment's encoded key. */
|
|
352
|
+
of(ref) {
|
|
353
|
+
let names = this.byNamespace.get(ref.namespace);
|
|
354
|
+
const found = names?.get(ref.segment);
|
|
355
|
+
if (found !== void 0) return found;
|
|
356
|
+
const encoded = this.encode(ref);
|
|
357
|
+
if (this.kept >= this.max) {
|
|
358
|
+
this.byNamespace.clear();
|
|
359
|
+
this.kept = 0;
|
|
360
|
+
names = void 0;
|
|
361
|
+
}
|
|
362
|
+
if (names === void 0) {
|
|
363
|
+
names = /* @__PURE__ */ new Map();
|
|
364
|
+
this.byNamespace.set(ref.namespace, names);
|
|
365
|
+
}
|
|
366
|
+
names.set(ref.segment, encoded);
|
|
367
|
+
this.kept += 1;
|
|
368
|
+
return encoded;
|
|
369
|
+
}
|
|
370
|
+
/** How many encoded keys are kept. */
|
|
371
|
+
get size() {
|
|
372
|
+
return this.kept;
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
function chunkKeyUnder(prefix, chunkKey, version) {
|
|
376
|
+
return version === void 0 ? `${prefix}${chunkKey}` : `${prefix}${chunkKey}${FIELD}${version}`;
|
|
337
377
|
}
|
|
338
378
|
function shardOf(key, totalShards) {
|
|
339
379
|
let h = 2166136261;
|
|
@@ -381,7 +421,9 @@ var RecordingAuditSink = class {
|
|
|
381
421
|
}
|
|
382
422
|
/** An independent copy of the recorded events, in emission order. */
|
|
383
423
|
snapshot() {
|
|
384
|
-
return this.recorded.map(
|
|
424
|
+
return this.recorded.map(
|
|
425
|
+
(e) => e.kind === "segment.collect" ? { ...e, collected: [...e.collected] } : { ...e }
|
|
426
|
+
);
|
|
385
427
|
}
|
|
386
428
|
reset() {
|
|
387
429
|
this.recorded.length = 0;
|
|
@@ -619,6 +661,11 @@ var SegmentEngine = class {
|
|
|
619
661
|
openStreams = /* @__PURE__ */ new Set();
|
|
620
662
|
/** How many times {@link invalidate} has been called: a read compares it with the count it began under. */
|
|
621
663
|
invalidations = 0;
|
|
664
|
+
/**
|
|
665
|
+
* The cache-key prefixes ({@link segmentPrefix}) of the segments this engine reads, kept rather than encoded for each
|
|
666
|
+
* chunk. A kept prefix is the one {@link invalidate} finds the segment's chunks by.
|
|
667
|
+
*/
|
|
668
|
+
prefixes = new KeptSegmentKeys(segmentPrefix);
|
|
622
669
|
constructor(deps) {
|
|
623
670
|
this.storage = deps.storage;
|
|
624
671
|
this.cache = deps.cache;
|
|
@@ -668,27 +715,30 @@ var SegmentEngine = class {
|
|
|
668
715
|
return total;
|
|
669
716
|
}
|
|
670
717
|
/**
|
|
671
|
-
* What the segment's current generation is: its number, its id count
|
|
672
|
-
*
|
|
718
|
+
* What the segment's current generation is: its number, its id count, its metadata and the size of its object in
|
|
719
|
+
* bytes. A source with a `stat` (the `.crbm` source) answers all four from one resolution, so they cannot straddle
|
|
720
|
+
* a publish: from the registry row's summary when it can use it, else from the opened generation's footer and
|
|
721
|
+
* index, with no payload read. A source without one answers from its other methods, each its own resolution: its
|
|
722
|
+
* summary when it has one, else `count()` and the generation it resolves, and its `sizeOf`, or `sizeBytes: null`
|
|
723
|
+
* when it has none. A segment with no generation answers `{ generation: null, cardinality: 0, sizeBytes: null }`.
|
|
673
724
|
*/
|
|
674
725
|
async stat(seg) {
|
|
726
|
+
if (this.storage.stat) {
|
|
727
|
+
const found = await this.storage.stat(seg);
|
|
728
|
+
if (found === null) return { generation: null, cardinality: 0, sizeBytes: null };
|
|
729
|
+
return found;
|
|
730
|
+
}
|
|
675
731
|
if (this.storage.summary) {
|
|
676
|
-
|
|
732
|
+
const summary = await this.storage.summary(seg);
|
|
733
|
+
if (summary === null) return { generation: null, cardinality: 0, sizeBytes: null };
|
|
734
|
+
return { ...summary, sizeBytes: await this.sizeOf(seg) };
|
|
677
735
|
}
|
|
678
736
|
const generation = this.storage.currentGeneration ? await this.storage.currentGeneration(seg) : null;
|
|
679
|
-
return { generation, cardinality: await this.count(seg) };
|
|
680
|
-
}
|
|
681
|
-
/** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
682
|
-
get supportsStorageSize() {
|
|
683
|
-
return typeof this.storage.sizeOf === "function";
|
|
737
|
+
return { generation, cardinality: await this.count(seg), sizeBytes: await this.sizeOf(seg) };
|
|
684
738
|
}
|
|
685
|
-
/**
|
|
686
|
-
|
|
687
|
-
return this.storage.
|
|
688
|
-
}
|
|
689
|
-
/** Grounded Storage size of a segment's current generation (cost reporting), or null if it has no generation. */
|
|
690
|
-
segmentSize(seg) {
|
|
691
|
-
return this.storage.sizeOf ? this.storage.sizeOf(seg) : Promise.resolve(null);
|
|
739
|
+
/** The size of the segment's current generation from the source's `sizeOf`, or `null` when it cannot say. */
|
|
740
|
+
async sizeOf(seg) {
|
|
741
|
+
return this.storage.sizeOf ? (await this.storage.sizeOf(seg))?.sizeBytes ?? null : null;
|
|
692
742
|
}
|
|
693
743
|
/** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
|
|
694
744
|
chunkWindow(seg, chunkKeys, gen, ramp) {
|
|
@@ -920,11 +970,13 @@ var SegmentEngine = class {
|
|
|
920
970
|
* the result — every chunk read is a whole, checksum-verified, immutable generation. The edge a *long* call can
|
|
921
971
|
* hit is invariant 3's: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
|
|
922
972
|
* operand's not-yet-requested chunks may re-resolve forward to the newer generation (a generation hop within one long
|
|
923
|
-
* call) — the call never crashes or returns a torn object, but may mix generations.
|
|
924
|
-
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
|
|
925
|
-
* (`maxOpenSegments`), whose re-read re-resolves fresh
|
|
926
|
-
*
|
|
927
|
-
*
|
|
973
|
+
* call) — the call never crashes or returns a torn object, but may mix generations. Some things hop it without
|
|
974
|
+
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call** on a
|
|
975
|
+
* store with no timed refresh (`maxOpenSegments`), whose re-read re-resolves fresh, and on a store with one the
|
|
976
|
+
* resolution cache letting the operand's resolution go; a sweep deleting the generation it was reading, or another
|
|
977
|
+
* object under its number, met by this read or another, which heals the read forward, as another read's failed open
|
|
978
|
+
* of the operand does; and an invalidation, which this store's own `load`, `rollback` and `eraseSubject` make and
|
|
979
|
+
* `invalidate()` makes on request. Still whole/immutable per read, never torn. An operand's stream resolves its
|
|
928
980
|
* segment again before it serves each chunk, so each of those moves a running read as it would a read of one chunk, and
|
|
929
981
|
* the ranges it had requested of the earlier generation are dropped, not served.
|
|
930
982
|
*/
|
|
@@ -1306,7 +1358,7 @@ var SegmentEngine = class {
|
|
|
1306
1358
|
}
|
|
1307
1359
|
/** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
|
|
1308
1360
|
chunkCacheKey(ref, version) {
|
|
1309
|
-
return
|
|
1361
|
+
return chunkKeyUnder(this.prefixes.of(ref), ref.chunkKey, version);
|
|
1310
1362
|
}
|
|
1311
1363
|
/** The segment's chunk keys, ascending, checked as untrusted tier data. */
|
|
1312
1364
|
async chunkKeys(seg) {
|
|
@@ -1620,599 +1672,6 @@ var BoundedLru = class {
|
|
|
1620
1672
|
}
|
|
1621
1673
|
};
|
|
1622
1674
|
|
|
1623
|
-
// src/core/generation-gc.ts
|
|
1624
|
-
async function nextGeneration(ref, deps) {
|
|
1625
|
-
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
1626
|
-
}
|
|
1627
|
-
async function aboveEverything(ref, deps, record) {
|
|
1628
|
-
let highest = record?.currentGen ?? -1;
|
|
1629
|
-
for await (const key of deps.storage.list(ref)) {
|
|
1630
|
-
if (key.generation > highest) highest = key.generation;
|
|
1631
|
-
}
|
|
1632
|
-
return numberAbove(highest);
|
|
1633
|
-
}
|
|
1634
|
-
function numberAbove(highest) {
|
|
1635
|
-
if (highest >= Number.MAX_SAFE_INTEGER) {
|
|
1636
|
-
throw new IntegrityError("the segment has no generation number left above its highest");
|
|
1637
|
-
}
|
|
1638
|
-
return highest + 1;
|
|
1639
|
-
}
|
|
1640
|
-
async function nextLoadGeneration(ref, deps, row) {
|
|
1641
|
-
const generation = numberAbove(row?.currentGen ?? -1);
|
|
1642
|
-
try {
|
|
1643
|
-
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
1644
|
-
} catch (err) {
|
|
1645
|
-
if (isNotFoundError(err)) return { generation, checked: true };
|
|
1646
|
-
}
|
|
1647
|
-
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
1648
|
-
}
|
|
1649
|
-
var LIST_COLLECTION_CADENCE = 16;
|
|
1650
|
-
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
1651
|
-
return (await listingPass(ref, deps, { keep: options.keep })).deleted;
|
|
1652
|
-
}
|
|
1653
|
-
async function listingPass(ref, deps, options) {
|
|
1654
|
-
const keep = options.keep ?? 1;
|
|
1655
|
-
const quiet = options.quiet === true;
|
|
1656
|
-
if (!Number.isInteger(keep) || keep < 0) {
|
|
1657
|
-
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
1658
|
-
}
|
|
1659
|
-
const record = await deps.registry.get(ref);
|
|
1660
|
-
if (record === null) return { deleted: [], kept: [], complete: true };
|
|
1661
|
-
const current = record.currentGen;
|
|
1662
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1663
|
-
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
1664
|
-
const gens = [...seen];
|
|
1665
|
-
const after = await deps.registry.get(ref);
|
|
1666
|
-
if (after === null) {
|
|
1667
|
-
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
1668
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
1669
|
-
}
|
|
1670
|
-
if (record.status === "destroyed" && after.token !== record.token) {
|
|
1671
|
-
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
1672
|
-
throw new WriteConflictError(
|
|
1673
|
-
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
1674
|
-
);
|
|
1675
|
-
}
|
|
1676
|
-
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
1677
|
-
const named = record.status === "destroyed" ? void 0 : usableKeptGens(after);
|
|
1678
|
-
const protect = options.protect === void 0 || named === void 0 ? void 0 : /* @__PURE__ */ new Set([...options.protect, ...named]);
|
|
1679
|
-
const stillCollectable = async (generation) => {
|
|
1680
|
-
const still = await deps.registry.get(ref);
|
|
1681
|
-
if (still === null) {
|
|
1682
|
-
if (quiet) return "stop";
|
|
1683
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
1684
|
-
}
|
|
1685
|
-
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
1686
|
-
const now = protect === void 0 ? void 0 : usableKeptGens(still);
|
|
1687
|
-
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
1688
|
-
if (!ok || !unnamed) {
|
|
1689
|
-
if (quiet) return "stop";
|
|
1690
|
-
throw new WriteConflictError(
|
|
1691
|
-
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
1692
|
-
);
|
|
1693
|
-
}
|
|
1694
|
-
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
1695
|
-
return "delete";
|
|
1696
|
-
};
|
|
1697
|
-
const guard = record.status === "destroyed" ? void 0 : options.leases;
|
|
1698
|
-
const held = guard === void 0 ? void 0 : heldGenerations(after, guard.now());
|
|
1699
|
-
const below = record.status === "destroyed" || cutoff === null ? [] : gens.filter((g) => g < cutoff).sort((a, b) => b - a);
|
|
1700
|
-
const candidates = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
1701
|
-
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
1702
|
-
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
1703
|
-
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
1704
|
-
[]
|
|
1705
|
-
) : protect !== void 0 ? (
|
|
1706
|
-
// The row's list is the window: everything below the pointer it does not name is garbage.
|
|
1707
|
-
below.filter((g) => !protect.has(g))
|
|
1708
|
-
) : (
|
|
1709
|
-
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
1710
|
-
below.slice(keep)
|
|
1711
|
-
);
|
|
1712
|
-
const leasedSpared = new Set(candidates.filter((g) => held?.has(g) === true));
|
|
1713
|
-
const toDelete = candidates.filter((g) => !leasedSpared.has(g));
|
|
1714
|
-
const done = [];
|
|
1715
|
-
for (const generation of toDelete) {
|
|
1716
|
-
const verdict = await stillCollectable(generation);
|
|
1717
|
-
if (verdict === "stop") return { deleted: done, kept: [], complete: false };
|
|
1718
|
-
if (verdict === "skip") {
|
|
1719
|
-
leasedSpared.add(generation);
|
|
1720
|
-
continue;
|
|
1721
|
-
}
|
|
1722
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
1723
|
-
done.push(generation);
|
|
1724
|
-
}
|
|
1725
|
-
const deleted = new Set(done);
|
|
1726
|
-
return {
|
|
1727
|
-
deleted: done,
|
|
1728
|
-
kept: below.filter((g) => !deleted.has(g) && !leasedSpared.has(g)).reverse(),
|
|
1729
|
-
complete: true
|
|
1730
|
-
};
|
|
1731
|
-
}
|
|
1732
|
-
async function collectAfterLoad(ref, deps, options) {
|
|
1733
|
-
const { generation, keep, byName, currentGone, kept, proveCurrent, leases } = options;
|
|
1734
|
-
if (keep >= generation) return [];
|
|
1735
|
-
const list = kept?.list;
|
|
1736
|
-
if (list === void 0 || currentGone === true)
|
|
1737
|
-
return reconcile(ref, deps, { keep, kept, leases });
|
|
1738
|
-
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
1739
|
-
if (byName && !periodic) {
|
|
1740
|
-
if (keep >= 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
1741
|
-
return reconcile(ref, deps, { keep, kept, leases });
|
|
1742
|
-
}
|
|
1743
|
-
return deleteEvicted(ref, deps, { generation, evict: kept.evict, leases });
|
|
1744
|
-
}
|
|
1745
|
-
return (await listingPass(ref, deps, { keep, protect: list, quiet: true, leases })).deleted;
|
|
1746
|
-
}
|
|
1747
|
-
async function reconcile(ref, deps, options) {
|
|
1748
|
-
const { keep, kept, leases } = options;
|
|
1749
|
-
const {
|
|
1750
|
-
deleted,
|
|
1751
|
-
kept: present,
|
|
1752
|
-
complete
|
|
1753
|
-
} = await listingPass(ref, deps, { keep, quiet: true, leases });
|
|
1754
|
-
const token = kept?.token;
|
|
1755
|
-
if (complete && token !== void 0 && keep <= MAX_KEPT_GENERATIONS && !sameList(kept?.list, present)) {
|
|
1756
|
-
try {
|
|
1757
|
-
await deps.registry.compareAndSwap(ref, token, { keptGens: present });
|
|
1758
|
-
} catch (err) {
|
|
1759
|
-
if (!isWriteConflictError(err)) throw err;
|
|
1760
|
-
}
|
|
1761
|
-
}
|
|
1762
|
-
return deleted;
|
|
1763
|
-
}
|
|
1764
|
-
function sameList(a, b) {
|
|
1765
|
-
return a !== void 0 && a.length === b.length && a.every((g, i) => g === b[i]);
|
|
1766
|
-
}
|
|
1767
|
-
async function objectIsThere(storage, key) {
|
|
1768
|
-
try {
|
|
1769
|
-
await storage.getTail(key, 0);
|
|
1770
|
-
return true;
|
|
1771
|
-
} catch {
|
|
1772
|
-
return false;
|
|
1773
|
-
}
|
|
1774
|
-
}
|
|
1775
|
-
async function deleteEvicted(ref, deps, options) {
|
|
1776
|
-
const deleted = [];
|
|
1777
|
-
for (const generation of options.evict) {
|
|
1778
|
-
const still = await deps.registry.get(ref);
|
|
1779
|
-
if (still === null || still.currentGen === null || still.currentGen < options.generation) break;
|
|
1780
|
-
const named = usableKeptGens(still);
|
|
1781
|
-
if (named === void 0 || generation >= still.currentGen || named.includes(generation)) break;
|
|
1782
|
-
if (options.leases !== void 0 && heldGenerations(still, options.leases.now()).has(generation)) {
|
|
1783
|
-
continue;
|
|
1784
|
-
}
|
|
1785
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
1786
|
-
deleted.push(generation);
|
|
1787
|
-
}
|
|
1788
|
-
return deleted;
|
|
1789
|
-
}
|
|
1790
|
-
|
|
1791
|
-
// src/core/reader-defaults.ts
|
|
1792
|
-
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
1793
|
-
var REFRESH_RETRY_MS = 500;
|
|
1794
|
-
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
1795
|
-
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
1796
|
-
|
|
1797
|
-
// src/core/cost.ts
|
|
1798
|
-
function deepFreeze(value) {
|
|
1799
|
-
if (value !== null && typeof value === "object" && !Object.isFrozen(value)) {
|
|
1800
|
-
Object.freeze(value);
|
|
1801
|
-
for (const inner of Object.values(value)) deepFreeze(inner);
|
|
1802
|
-
}
|
|
1803
|
-
return value;
|
|
1804
|
-
}
|
|
1805
|
-
var ELASTICACHE_REDIS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
1806
|
-
source: "ElastiCache for Redis OSS, us-east-1 on-demand, AWS price list 20260914063714",
|
|
1807
|
-
replicasPerShard: 2,
|
|
1808
|
-
reservedMemoryFraction: 0.25,
|
|
1809
|
-
nodeTypes: [
|
|
1810
|
-
{ name: "cache.t4g.micro", memoryGiB: 0.5, hourlyUSD: 0.016, maxShards: 1 },
|
|
1811
|
-
{ name: "cache.t4g.small", memoryGiB: 1.37, hourlyUSD: 0.032, maxShards: 1 },
|
|
1812
|
-
{ name: "cache.t4g.medium", memoryGiB: 3.09, hourlyUSD: 0.065, maxShards: 1 },
|
|
1813
|
-
{ name: "cache.m6g.large", memoryGiB: 6.38, hourlyUSD: 0.149 },
|
|
1814
|
-
{ name: "cache.r6g.large", memoryGiB: 13.07, hourlyUSD: 0.206 },
|
|
1815
|
-
{ name: "cache.r6g.xlarge", memoryGiB: 26.32, hourlyUSD: 0.411 },
|
|
1816
|
-
{ name: "cache.r6g.2xlarge", memoryGiB: 52.82, hourlyUSD: 0.821 },
|
|
1817
|
-
{ name: "cache.r6g.4xlarge", memoryGiB: 105.81, hourlyUSD: 1.642 },
|
|
1818
|
-
{ name: "cache.r6g.8xlarge", memoryGiB: 209.55, hourlyUSD: 3.284 },
|
|
1819
|
-
{ name: "cache.r6g.12xlarge", memoryGiB: 317.77, hourlyUSD: 4.925 },
|
|
1820
|
-
{ name: "cache.r6g.16xlarge", memoryGiB: 419.09, hourlyUSD: 6.567 },
|
|
1821
|
-
{ name: "cache.r6gd.xlarge", memoryGiB: 26.32, ssdGiB: 99.33, hourlyUSD: 0.781 },
|
|
1822
|
-
{ name: "cache.r6gd.2xlarge", memoryGiB: 52.82, ssdGiB: 199.07, hourlyUSD: 1.56 },
|
|
1823
|
-
{ name: "cache.r6gd.4xlarge", memoryGiB: 105.81, ssdGiB: 398.14, hourlyUSD: 3.12 },
|
|
1824
|
-
{ name: "cache.r6gd.8xlarge", memoryGiB: 209.55, ssdGiB: 796.28, hourlyUSD: 6.24 },
|
|
1825
|
-
{ name: "cache.r6gd.12xlarge", memoryGiB: 317.77, ssdGiB: 1194.42, hourlyUSD: 9.358 },
|
|
1826
|
-
{ name: "cache.r6gd.16xlarge", memoryGiB: 419.09, ssdGiB: 1592.56, hourlyUSD: 12.477 }
|
|
1827
|
-
]
|
|
1828
|
-
});
|
|
1829
|
-
var ONE_REDIS_HA_CLUSTER = deepFreeze({ monthlyUSD: 346 });
|
|
1830
|
-
var AWS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
1831
|
-
name: "aws-us-east-1-ondemand",
|
|
1832
|
-
storage: { getPerMillion: 0.4, putPerMillion: 5, storagePerGiBMonth: 0.023 },
|
|
1833
|
-
redis: { sizedToData: ELASTICACHE_REDIS_US_EAST_1_ONDEMAND }
|
|
1834
|
-
});
|
|
1835
|
-
var HOURS_PER_MONTH = 730;
|
|
1836
|
-
var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
|
|
1837
|
-
var GIB = 1024 ** 3;
|
|
1838
|
-
var STORE_LOAD_PUT_CLASS = 1 + 1 / LIST_COLLECTION_CADENCE;
|
|
1839
|
-
var STORE_LOAD_POINTER_READS = 2 + 2 / LIST_COLLECTION_CADENCE;
|
|
1840
|
-
var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
|
|
1841
|
-
var RETENTION_SWEEP_REQUESTS = deepFreeze({
|
|
1842
|
-
conditionalDelete: {
|
|
1843
|
-
retirement: { reads: 7, writes: 2, deletes: 1 },
|
|
1844
|
-
purge: { reads: 4, writes: 0, deletes: 2 }
|
|
1845
|
-
},
|
|
1846
|
-
tombstoning: {
|
|
1847
|
-
retirement: { reads: 6, writes: 2, deletes: 0 },
|
|
1848
|
-
purge: { reads: 3, writes: 1, deletes: 0 }
|
|
1849
|
-
}
|
|
1850
|
-
});
|
|
1851
|
-
var shown = (n) => String(Number(n.toFixed(2)));
|
|
1852
|
-
function requireFiniteNonNeg(n, field) {
|
|
1853
|
-
if (n === void 0 || !Number.isFinite(n) || n < 0) {
|
|
1854
|
-
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
1855
|
-
}
|
|
1856
|
-
return n;
|
|
1857
|
-
}
|
|
1858
|
-
function clamp01(n) {
|
|
1859
|
-
return n < 0 ? 0 : n > 1 ? 1 : n;
|
|
1860
|
-
}
|
|
1861
|
-
var TIE_USD = 1e-6;
|
|
1862
|
-
function usableGiB(sizing, nodeType) {
|
|
1863
|
-
return nodeType.memoryGiB * (1 - sizing.reservedMemoryFraction) + (nodeType.ssdGiB ?? 0);
|
|
1864
|
-
}
|
|
1865
|
-
function validateSizing(sizing) {
|
|
1866
|
-
const at = "pricing.redis.sizedToData";
|
|
1867
|
-
if (sizing === null || typeof sizing !== "object") {
|
|
1868
|
-
throw new ValidationError(`${at} must be a RedisSizing`);
|
|
1869
|
-
}
|
|
1870
|
-
if (typeof sizing.source !== "string" || sizing.source === "") {
|
|
1871
|
-
throw new ValidationError(`${at}.source must say where the prices come from`);
|
|
1872
|
-
}
|
|
1873
|
-
if (!Array.isArray(sizing.nodeTypes) || sizing.nodeTypes.length === 0) {
|
|
1874
|
-
throw new ValidationError(`${at}.nodeTypes must list at least one node type`);
|
|
1875
|
-
}
|
|
1876
|
-
if (!Number.isInteger(sizing.replicasPerShard) || sizing.replicasPerShard < 0) {
|
|
1877
|
-
throw new ValidationError(
|
|
1878
|
-
`${at}.replicasPerShard must be an integer >= 0; got ${sizing.replicasPerShard}`
|
|
1879
|
-
);
|
|
1880
|
-
}
|
|
1881
|
-
const reserved = requireFiniteNonNeg(
|
|
1882
|
-
sizing.reservedMemoryFraction,
|
|
1883
|
-
`${at}.reservedMemoryFraction`
|
|
1884
|
-
);
|
|
1885
|
-
if (reserved >= 1) {
|
|
1886
|
-
throw new ValidationError(`${at}.reservedMemoryFraction must be below 1; got ${reserved}`);
|
|
1887
|
-
}
|
|
1888
|
-
const names = /* @__PURE__ */ new Set();
|
|
1889
|
-
for (const n of sizing.nodeTypes) {
|
|
1890
|
-
if (n === null || typeof n !== "object" || typeof n.name !== "string" || n.name === "") {
|
|
1891
|
-
throw new ValidationError(`${at}.nodeTypes: every node type needs a name`);
|
|
1892
|
-
}
|
|
1893
|
-
const row = `${at}.nodeTypes[${JSON.stringify(n.name)}]`;
|
|
1894
|
-
if (names.has(n.name)) throw new ValidationError(`${row} is listed twice`);
|
|
1895
|
-
names.add(n.name);
|
|
1896
|
-
if (!(requireFiniteNonNeg(n.memoryGiB, `${row}.memoryGiB`) > 0)) {
|
|
1897
|
-
throw new ValidationError(`${row}.memoryGiB must be above 0`);
|
|
1898
|
-
}
|
|
1899
|
-
requireFiniteNonNeg(n.ssdGiB ?? 0, `${row}.ssdGiB`);
|
|
1900
|
-
requireFiniteNonNeg(n.hourlyUSD, `${row}.hourlyUSD`);
|
|
1901
|
-
if (n.maxShards !== void 0 && (!Number.isInteger(n.maxShards) || n.maxShards < 1)) {
|
|
1902
|
-
throw new ValidationError(`${row}.maxShards must be an integer >= 1; got ${n.maxShards}`);
|
|
1903
|
-
}
|
|
1904
|
-
if (!(usableGiB(sizing, n) > 0)) {
|
|
1905
|
-
throw new ValidationError(
|
|
1906
|
-
`${row} leaves no memory for data once ${reserved} of it is reserved`
|
|
1907
|
-
);
|
|
1908
|
-
}
|
|
1909
|
-
}
|
|
1910
|
-
}
|
|
1911
|
-
function cheapestCluster(sizing, dataGiB) {
|
|
1912
|
-
const fits = [];
|
|
1913
|
-
for (const nodeType of sizing.nodeTypes) {
|
|
1914
|
-
const usable = usableGiB(sizing, nodeType);
|
|
1915
|
-
let shards = Math.max(1, Math.ceil(dataGiB / usable));
|
|
1916
|
-
if (shards > 1 && (shards - 1) * usable >= dataGiB) shards -= 1;
|
|
1917
|
-
if (nodeType.maxShards !== void 0 && shards > nodeType.maxShards) continue;
|
|
1918
|
-
const nodes = shards * (1 + sizing.replicasPerShard);
|
|
1919
|
-
fits.push({
|
|
1920
|
-
nodeType,
|
|
1921
|
-
shards,
|
|
1922
|
-
nodes,
|
|
1923
|
-
monthlyUSD: nodes * nodeType.hourlyUSD * HOURS_PER_MONTH
|
|
1924
|
-
});
|
|
1925
|
-
}
|
|
1926
|
-
const size = bytesWords(dataGiB * GIB);
|
|
1927
|
-
if (fits.length === 0) {
|
|
1928
|
-
throw new ValidationError(
|
|
1929
|
-
`no node type in pricing.redis.sizedToData.nodeTypes holds ${size} within its maxShards`
|
|
1930
|
-
);
|
|
1931
|
-
}
|
|
1932
|
-
const cheapest = fits.reduce((m, c) => Math.min(m, c.monthlyUSD), Infinity);
|
|
1933
|
-
const [best] = fits.filter((c) => c.monthlyUSD <= cheapest + TIE_USD).sort(
|
|
1934
|
-
(a, b) => a.nodes - b.nodes || a.monthlyUSD - b.monthlyUSD || (a.nodeType.name < b.nodeType.name ? -1 : a.nodeType.name > b.nodeType.name ? 1 : 0)
|
|
1935
|
-
);
|
|
1936
|
-
if (best === void 0 || !Number.isFinite(best.monthlyUSD)) {
|
|
1937
|
-
throw new ValidationError(
|
|
1938
|
-
`pricing.redis.sizedToData prices ${size} at no finite cost; check its node types`
|
|
1939
|
-
);
|
|
1940
|
-
}
|
|
1941
|
-
return best;
|
|
1942
|
-
}
|
|
1943
|
-
function snapshotSizing(sizing) {
|
|
1944
|
-
if (sizing === null || typeof sizing !== "object") return sizing;
|
|
1945
|
-
const { source, nodeTypes, replicasPerShard, reservedMemoryFraction } = sizing;
|
|
1946
|
-
const row = (n) => {
|
|
1947
|
-
if (n === null || typeof n !== "object") return n;
|
|
1948
|
-
const { name, memoryGiB, ssdGiB, hourlyUSD, maxShards } = n;
|
|
1949
|
-
return {
|
|
1950
|
-
name,
|
|
1951
|
-
memoryGiB,
|
|
1952
|
-
hourlyUSD,
|
|
1953
|
-
...ssdGiB === void 0 ? {} : { ssdGiB },
|
|
1954
|
-
...maxShards === void 0 ? {} : { maxShards }
|
|
1955
|
-
};
|
|
1956
|
-
};
|
|
1957
|
-
return {
|
|
1958
|
-
source,
|
|
1959
|
-
replicasPerShard,
|
|
1960
|
-
reservedMemoryFraction,
|
|
1961
|
-
nodeTypes: Array.isArray(nodeTypes) ? Array.from(nodeTypes, row) : nodeTypes
|
|
1962
|
-
};
|
|
1963
|
-
}
|
|
1964
|
-
function resolveBaseline(redis, storedGiB) {
|
|
1965
|
-
if (redis === null || typeof redis !== "object") {
|
|
1966
|
-
throw new ValidationError("pricing.redis must be { monthlyUSD } or { sizedToData }");
|
|
1967
|
-
}
|
|
1968
|
-
const { monthlyUSD, sizedToData } = redis;
|
|
1969
|
-
const fixed = monthlyUSD !== void 0;
|
|
1970
|
-
const sized = sizedToData !== void 0;
|
|
1971
|
-
if (fixed === sized) {
|
|
1972
|
-
throw new ValidationError(
|
|
1973
|
-
"pricing.redis must be exactly one of { monthlyUSD } and { sizedToData }" + (fixed ? ", not both: spreading AWS_US_EAST_1_ONDEMAND.redis keeps its sizedToData, so pass redis: { monthlyUSD } alone" : "")
|
|
1974
|
-
);
|
|
1975
|
-
}
|
|
1976
|
-
if (sized) {
|
|
1977
|
-
const sizing = snapshotSizing(sizedToData);
|
|
1978
|
-
validateSizing(sizing);
|
|
1979
|
-
const cluster = cheapestCluster(sizing, storedGiB);
|
|
1980
|
-
return { monthlyUSD: cluster.monthlyUSD, sized: { sizing, cluster } };
|
|
1981
|
-
}
|
|
1982
|
-
return { monthlyUSD: requireFiniteNonNeg(monthlyUSD, "pricing.redis.monthlyUSD") };
|
|
1983
|
-
}
|
|
1984
|
-
function bytesWords(bytes) {
|
|
1985
|
-
const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
|
|
1986
|
-
let value = bytes;
|
|
1987
|
-
let unit = 0;
|
|
1988
|
-
const shown2 = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
|
|
1989
|
-
while (unit < units.length - 1 && shown2(value, unit) >= 1024) {
|
|
1990
|
-
value /= 1024;
|
|
1991
|
-
unit += 1;
|
|
1992
|
-
}
|
|
1993
|
-
if (unit > 0) return `${value.toFixed(2)} ${units[unit]}`;
|
|
1994
|
-
const n = Math.round(value);
|
|
1995
|
-
return `${n} byte${n === 1 ? "" : "s"}`;
|
|
1996
|
-
}
|
|
1997
|
-
function clusterWords(c) {
|
|
1998
|
-
const count = (n) => n.toLocaleString("en-US");
|
|
1999
|
-
const nodes = `${count(c.nodes)} ${c.nodeType.name} node${c.nodes === 1 ? "" : "s"}`;
|
|
2000
|
-
return c.shards === 1 ? `1 shard of ${nodes}` : `${count(c.shards)} shards, ${nodes}`;
|
|
2001
|
-
}
|
|
2002
|
-
function sizingBytes(spec) {
|
|
2003
|
-
if (spec.sizeBytes !== void 0) return requireFiniteNonNeg(spec.sizeBytes, "segment.sizeBytes");
|
|
2004
|
-
if (spec.cardinality !== void 0) {
|
|
2005
|
-
return requireFiniteNonNeg(spec.cardinality, "segment.cardinality") * 2;
|
|
2006
|
-
}
|
|
2007
|
-
return 0;
|
|
2008
|
-
}
|
|
2009
|
-
function buildReport(input) {
|
|
2010
|
-
const { pricing, grounded } = input;
|
|
2011
|
-
const { storage, redis } = pricing;
|
|
2012
|
-
const S = SECONDS_PER_MONTH;
|
|
2013
|
-
requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
|
|
2014
|
-
requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
|
|
2015
|
-
requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
|
|
2016
|
-
const pointerRead = requireFiniteNonNeg(
|
|
2017
|
-
storage.requestsPerPointerRead ?? 1,
|
|
2018
|
-
"pricing.storage.requestsPerPointerRead"
|
|
2019
|
-
);
|
|
2020
|
-
const sizedRead = requireFiniteNonNeg(
|
|
2021
|
-
storage.requestsPerSizedRead ?? 1,
|
|
2022
|
-
"pricing.storage.requestsPerSizedRead"
|
|
2023
|
-
);
|
|
2024
|
-
const storageBytes = requireFiniteNonNeg(input.storageBytes, "storageBytes");
|
|
2025
|
-
const storedGiB = storageBytes / GIB;
|
|
2026
|
-
const baseline = resolveBaseline(redis, storedGiB);
|
|
2027
|
-
const baselineUSD = baseline.monthlyUSD;
|
|
2028
|
-
if ((input.workload.cacheHitRate ?? 0) > 1) {
|
|
2029
|
-
throw new ValidationError(
|
|
2030
|
-
`cacheHitRate must be a fraction from 0 to 1; got ${String(input.workload.cacheHitRate)}`
|
|
2031
|
-
);
|
|
2032
|
-
}
|
|
2033
|
-
const cacheHitRate = clamp01(
|
|
2034
|
-
requireFiniteNonNeg(input.workload.cacheHitRate ?? 0, "cacheHitRate")
|
|
2035
|
-
);
|
|
2036
|
-
const readsPerSec = requireFiniteNonNeg(input.workload.readsPerSec ?? 0, "readsPerSec");
|
|
2037
|
-
const intersectsPerSec = requireFiniteNonNeg(
|
|
2038
|
-
input.workload.intersectsPerSec ?? 0,
|
|
2039
|
-
"intersectsPerSec"
|
|
2040
|
-
);
|
|
2041
|
-
const chunksPerIntersect = requireFiniteNonNeg(
|
|
2042
|
-
input.workload.chunksPerIntersect ?? 1,
|
|
2043
|
-
"chunksPerIntersect"
|
|
2044
|
-
);
|
|
2045
|
-
const loadsPerMonth = requireFiniteNonNeg(input.workload.loadsPerMonth ?? 0, "loadsPerMonth");
|
|
2046
|
-
const requestsPerLoad = requireFiniteNonNeg(
|
|
2047
|
-
input.workload.requestsPerLoad ?? 1,
|
|
2048
|
-
"requestsPerLoad"
|
|
2049
|
-
);
|
|
2050
|
-
const operandsPerIntersect = requireFiniteNonNeg(
|
|
2051
|
-
input.workload.operandsPerIntersect ?? 2,
|
|
2052
|
-
"operandsPerIntersect"
|
|
2053
|
-
);
|
|
2054
|
-
if (operandsPerIntersect < 1) {
|
|
2055
|
-
throw new ValidationError(
|
|
2056
|
-
`operandsPerIntersect must be at least 1, since an intersection reads its operands; got ${operandsPerIntersect}`
|
|
2057
|
-
);
|
|
2058
|
-
}
|
|
2059
|
-
const hotSegments = requireFiniteNonNeg(input.workload.hotSegments ?? 0, "hotSegments");
|
|
2060
|
-
const readerProcesses = requireFiniteNonNeg(
|
|
2061
|
-
input.workload.readerProcesses ?? 1,
|
|
2062
|
-
"readerProcesses"
|
|
2063
|
-
);
|
|
2064
|
-
if (readerProcesses < 1) {
|
|
2065
|
-
throw new ValidationError(`readerProcesses must be at least 1; got ${readerProcesses}`);
|
|
2066
|
-
}
|
|
2067
|
-
const genTtlMs = requireFiniteNonNeg(
|
|
2068
|
-
input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
|
|
2069
|
-
"genTtlMs"
|
|
2070
|
-
);
|
|
2071
|
-
const retirementsPerMonth = requireFiniteNonNeg(
|
|
2072
|
-
input.workload.retirementsPerMonth ?? 0,
|
|
2073
|
-
"retirementsPerMonth"
|
|
2074
|
-
);
|
|
2075
|
-
const purgesPerMonth = requireFiniteNonNeg(input.workload.purgesPerMonth ?? 0, "purgesPerMonth");
|
|
2076
|
-
const conditionalDelete = input.workload.conditionalDelete ?? true;
|
|
2077
|
-
if (typeof conditionalDelete !== "boolean") {
|
|
2078
|
-
throw new ValidationError(`conditionalDelete must be a boolean; got ${conditionalDelete}`);
|
|
2079
|
-
}
|
|
2080
|
-
const storageGetUSD = storage.getPerMillion / 1e6;
|
|
2081
|
-
const putUSD = storage.putPerMillion / 1e6;
|
|
2082
|
-
const missFraction = 1 - cacheHitRate;
|
|
2083
|
-
const readMisses = readsPerSec * S * missFraction;
|
|
2084
|
-
const intersects = intersectsPerSec * S;
|
|
2085
|
-
const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
|
|
2086
|
-
const storageUSD = storedGiB * storage.storagePerGiBMonth;
|
|
2087
|
-
const readsUSD = readMisses * storageGetUSD;
|
|
2088
|
-
const getsPerColdOperand = pointerRead + sizedRead;
|
|
2089
|
-
const intersectGets = chunksPerIntersect + getsPerColdOperand * operandsPerIntersect;
|
|
2090
|
-
const intersectsUSD = intersects * intersectGets * storageGetUSD;
|
|
2091
|
-
const loadGets = STORE_LOAD_POINTER_READS * pointerRead + STORE_LOAD_EXISTENCE_CHECKS;
|
|
2092
|
-
const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
|
|
2093
|
-
const refreshUSD = refreshes * pointerRead * storageGetUSD;
|
|
2094
|
-
const sweep = RETENTION_SWEEP_REQUESTS[conditionalDelete ? "conditionalDelete" : "tombstoning"];
|
|
2095
|
-
const retentionUSD = retirementsPerMonth * (sweep.retirement.reads * storageGetUSD + sweep.retirement.writes * putUSD) + purgesPerMonth * (sweep.purge.reads * storageGetUSD + sweep.purge.writes * putUSD);
|
|
2096
|
-
const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD + retentionUSD;
|
|
2097
|
-
const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
|
|
2098
|
-
const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
|
|
2099
|
-
const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
|
|
2100
|
-
const dominant = Math.max(
|
|
2101
|
-
readsUSD,
|
|
2102
|
-
intersectsUSD,
|
|
2103
|
-
storageUSD,
|
|
2104
|
-
loadsUSD,
|
|
2105
|
-
refreshUSD,
|
|
2106
|
-
retentionUSD
|
|
2107
|
-
);
|
|
2108
|
-
let driver = "storage";
|
|
2109
|
-
if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
|
|
2110
|
-
else if (dominant === intersectsUSD && intersectsUSD > 0) {
|
|
2111
|
-
driver = "intersections (pointer, index and chunk GETs)";
|
|
2112
|
-
} else if (dominant === loadsUSD && loadsUSD > 0)
|
|
2113
|
-
driver = "loads (object, listing and pointer requests)";
|
|
2114
|
-
else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
|
|
2115
|
-
else if (dominant === retentionUSD && retentionUSD > 0)
|
|
2116
|
-
driver = "the retention sweep (registry reads and writes)";
|
|
2117
|
-
const sized = baseline.sized;
|
|
2118
|
-
const noBytes = Math.round(storageBytes) === 0;
|
|
2119
|
-
const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
|
|
2120
|
-
const rationale = verdict === "lose-zone" ? `pay-per-use total $${total.toFixed(2)}/mo exceeds the ${redisWords ?? `$${baselineUSD}/mo flat baseline`}; dominated by ${driver}` : verdict === "win-big" ? `$${total.toFixed(2)}/mo \u2014 \u226410% of the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}` : `$${total.toFixed(2)}/mo, under the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}`;
|
|
2121
|
-
const dataTiering = sized !== void 0 && (sized.cluster.nodeType.ssdGiB ?? 0) > 0;
|
|
2122
|
-
const redisNote = sized === void 0 ? `Redis priced at the fixed $${baselineUSD}/mo given, whatever the data size.` : (noBytes ? "Redis priced at the cheapest cluster in the catalogue, as no bytes are counted, because nothing is stored or nothing was measured" : `Redis priced at the cheapest cluster in the catalogue that holds the ${bytesWords(storageBytes)} stored`) + `: ${clusterWords(sized.cluster)}, each shard a primary and ${sized.sizing.replicasPerShard} replica(s), ${Number((sized.sizing.reservedMemoryFraction * 100).toFixed(1))}% of memory reserved (${sized.sizing.source}).` + (noBytes ? " Pass pricing.redis: { monthlyUSD } to compare against a Redis of your own." : " Compressed bytes are a floor on the memory Redis needs: a native Redis bitmap is sized by its highest id, so sparse ids need more.") + " Redis bought another way \u2014 fewer replicas, another engine, reserved nodes \u2014 can cost less than the catalogue prices it." + (dataTiering ? ` ${sized.cluster.nodeType.name} is a data-tiering node, which keeps the values read least recently on its SSD. It is priced as if the data read often fits in its memory: AWS recommends data tiering for workloads that regularly read up to 20% of their data.` : "") + (input.sizesFromCardinality ? " Some segment sizes came from cardinality, at 2 bytes an id, which is more than a dense bitmap needs, so this Redis may be larger than the data calls for: pass sizeBytes to size it to the compressed bytes." : "");
|
|
2123
|
-
const notes = [
|
|
2124
|
-
"Same-region egress treated as free; internet egress is not modeled.",
|
|
2125
|
-
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
2126
|
-
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${shown(STORE_LOAD_PUT_CLASS)} PUT-class and ${shown(loadGets)} GETs that store.load() adds (the pointer, a check that the next generation number is free, a check that the current generation is there, and a listing every ${LIST_COLLECTION_CADENCE}th load).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
|
|
2127
|
-
...intersects > 0 ? [
|
|
2128
|
-
`Intersections priced cold: ${intersectGets} GETs each, ${getsPerColdOperand} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk range request(s); cacheHitRate does not apply.`
|
|
2129
|
-
] : [],
|
|
2130
|
-
hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
|
|
2131
|
-
hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
|
|
2132
|
-
...retirementsPerMonth > 0 || purgesPerMonth > 0 ? [
|
|
2133
|
-
`Retention sweep modeled: ${retirementsPerMonth}/mo retirements at ${sweep.retirement.reads} reads, ${sweep.retirement.writes} writes and ${sweep.retirement.deletes} delete(s) each, ${purgesPerMonth}/mo purges at ${sweep.purge.reads} reads, ${sweep.purge.writes} writes and ${sweep.purge.deletes} delete(s) each; reads are priced at the GET rate and writes at the PUT rate, deletes at nothing, as S3 bills none` + (conditionalDelete ? "." : "; a registry that only tombstones is read again, two reads per purged segment, by every later full sweep, which is not priced.")
|
|
2134
|
-
] : [],
|
|
2135
|
-
...input.extraNotes ?? [],
|
|
2136
|
-
// Last, so the notes a report already carried keep their places.
|
|
2137
|
-
redisNote
|
|
2138
|
-
].filter((n) => n !== null);
|
|
2139
|
-
return {
|
|
2140
|
-
monthlyUSD: {
|
|
2141
|
-
byOp: {
|
|
2142
|
-
reads: readsUSD,
|
|
2143
|
-
intersects: intersectsUSD,
|
|
2144
|
-
storage: storageUSD,
|
|
2145
|
-
loads: loadsUSD,
|
|
2146
|
-
pointerRefresh: refreshUSD,
|
|
2147
|
-
retention: retentionUSD
|
|
2148
|
-
},
|
|
2149
|
-
total
|
|
2150
|
-
},
|
|
2151
|
-
redisBaseline: sized === void 0 ? { basis: "fixed", monthlyUSD: baselineUSD } : {
|
|
2152
|
-
basis: "sized-to-data",
|
|
2153
|
-
monthlyUSD: baselineUSD,
|
|
2154
|
-
cluster: {
|
|
2155
|
-
nodeType: sized.cluster.nodeType.name,
|
|
2156
|
-
shards: sized.cluster.shards,
|
|
2157
|
-
nodes: sized.cluster.nodes,
|
|
2158
|
-
dataTiering
|
|
2159
|
-
}
|
|
2160
|
-
},
|
|
2161
|
-
redisCrossover: { readsPerSec: readsCross },
|
|
2162
|
-
verdict,
|
|
2163
|
-
rationale,
|
|
2164
|
-
assumptions: { cacheHitRate, pricingName: pricing.name, grounded, notes }
|
|
2165
|
-
};
|
|
2166
|
-
}
|
|
2167
|
-
function checkModelInputs(op, pricing, workload) {
|
|
2168
|
-
const storage = pricing !== null && typeof pricing === "object" ? pricing.storage : void 0;
|
|
2169
|
-
if (pricing !== void 0 && (storage === null || typeof storage !== "object")) {
|
|
2170
|
-
throw new ValidationError(`${op}: pricing must be a profile with a storage price list`);
|
|
2171
|
-
}
|
|
2172
|
-
if (workload !== void 0 && workload !== null && (typeof workload !== "object" || Array.isArray(workload))) {
|
|
2173
|
-
throw new ValidationError(`${op}: workload must be an object`);
|
|
2174
|
-
}
|
|
2175
|
-
}
|
|
2176
|
-
function estimateCost(input) {
|
|
2177
|
-
if (input === null || typeof input !== "object") {
|
|
2178
|
-
throw new ValidationError("estimateCost: input must be an object such as { segments }");
|
|
2179
|
-
}
|
|
2180
|
-
if (!Array.isArray(input.segments)) {
|
|
2181
|
-
throw new ValidationError(
|
|
2182
|
-
"estimateCost: segments must be an array of { sizeBytes } or { cardinality }"
|
|
2183
|
-
);
|
|
2184
|
-
}
|
|
2185
|
-
for (let i = 0; i < input.segments.length; i++) {
|
|
2186
|
-
const spec = input.segments[i];
|
|
2187
|
-
if (spec === null || typeof spec !== "object") {
|
|
2188
|
-
throw new ValidationError(`estimateCost: segments[${i}] must be an object`);
|
|
2189
|
-
}
|
|
2190
|
-
}
|
|
2191
|
-
checkModelInputs("estimateCost", input.pricing, input.workload);
|
|
2192
|
-
const pricing = input.pricing ?? AWS_US_EAST_1_ONDEMAND;
|
|
2193
|
-
const workload = input.workload ?? {};
|
|
2194
|
-
let storageBytes = 0;
|
|
2195
|
-
let sizesFromCardinality = false;
|
|
2196
|
-
for (const spec of input.segments) {
|
|
2197
|
-
const count = Math.floor(requireFiniteNonNeg(spec.count ?? 1, "segment.count"));
|
|
2198
|
-
storageBytes += sizingBytes(spec) * count;
|
|
2199
|
-
if (spec.sizeBytes === void 0 && spec.cardinality !== void 0 && sizingBytes(spec) * count > 0) {
|
|
2200
|
-
sizesFromCardinality = true;
|
|
2201
|
-
}
|
|
2202
|
-
}
|
|
2203
|
-
return buildReport({ storageBytes, workload, pricing, grounded: false, sizesFromCardinality });
|
|
2204
|
-
}
|
|
2205
|
-
function groundedReport(input) {
|
|
2206
|
-
checkModelInputs("costReport", input.pricing, input.workload);
|
|
2207
|
-
return buildReport({
|
|
2208
|
-
storageBytes: input.storageBytes,
|
|
2209
|
-
workload: input.workload ?? {},
|
|
2210
|
-
pricing: input.pricing ?? AWS_US_EAST_1_ONDEMAND,
|
|
2211
|
-
grounded: input.grounded ?? true,
|
|
2212
|
-
extraNotes: input.extraNotes
|
|
2213
|
-
});
|
|
2214
|
-
}
|
|
2215
|
-
|
|
2216
1675
|
// src/core/due-index.ts
|
|
2217
1676
|
var DUE_BUCKET_MS = 864e5;
|
|
2218
1677
|
function dueBucket(expiresAt) {
|
|
@@ -2335,6 +1794,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2335
1794
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
2336
1795
|
const segments = [];
|
|
2337
1796
|
const failed = [];
|
|
1797
|
+
const skipped = [];
|
|
2338
1798
|
let totalIds = 0;
|
|
2339
1799
|
if (typeof reader?.segment !== "function") {
|
|
2340
1800
|
throw new ValidationError(
|
|
@@ -2411,10 +1871,21 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2411
1871
|
};
|
|
2412
1872
|
for await (const rec of registry.list(options.namespace)) {
|
|
2413
1873
|
if (options.namespace === void 0 && isReservedRow(rec)) continue;
|
|
2414
|
-
if (rec.status === "destroyed")
|
|
2415
|
-
|
|
1874
|
+
if (rec.status === "destroyed") {
|
|
1875
|
+
skipped.push({ segment: rec.segment, namespace: rec.namespace, reason: "destroyed" });
|
|
1876
|
+
continue;
|
|
1877
|
+
}
|
|
1878
|
+
await exportRef({ segment: rec.segment, namespace: rec.namespace });
|
|
2416
1879
|
}
|
|
2417
|
-
return {
|
|
1880
|
+
return {
|
|
1881
|
+
version: 1,
|
|
1882
|
+
format,
|
|
1883
|
+
totalSegments: segments.length,
|
|
1884
|
+
totalIds,
|
|
1885
|
+
segments,
|
|
1886
|
+
failed,
|
|
1887
|
+
skipped
|
|
1888
|
+
};
|
|
2418
1889
|
}
|
|
2419
1890
|
|
|
2420
1891
|
// src/core/discover.ts
|
|
@@ -2528,7 +1999,7 @@ var MemoryRegistryDriver = class {
|
|
|
2528
1999
|
if (!existing || existing.token !== expected) {
|
|
2529
2000
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2530
2001
|
}
|
|
2531
|
-
const token = this.nextToken(incarnationOf(existing.token)
|
|
2002
|
+
const token = this.nextToken(incarnationOf(existing.token));
|
|
2532
2003
|
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
2533
2004
|
return { token };
|
|
2534
2005
|
}
|
|
@@ -2549,8 +2020,14 @@ var MemoryRegistryDriver = class {
|
|
|
2549
2020
|
};
|
|
2550
2021
|
var MemoryStorageDriver = class {
|
|
2551
2022
|
objects = /* @__PURE__ */ new Map();
|
|
2023
|
+
stored = 0;
|
|
2552
2024
|
capabilities() {
|
|
2553
|
-
return {
|
|
2025
|
+
return {
|
|
2026
|
+
rangeRead: true,
|
|
2027
|
+
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2028
|
+
conditionalPut: true,
|
|
2029
|
+
conditionalDelete: true
|
|
2030
|
+
};
|
|
2554
2031
|
}
|
|
2555
2032
|
async putImmutable(key, write) {
|
|
2556
2033
|
const k = genObjectKey(key);
|
|
@@ -2562,14 +2039,15 @@ var MemoryStorageDriver = class {
|
|
|
2562
2039
|
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
2563
2040
|
);
|
|
2564
2041
|
}
|
|
2565
|
-
this.
|
|
2042
|
+
this.stored += 1;
|
|
2043
|
+
this.objects.set(k, { body, version: String(this.stored) });
|
|
2566
2044
|
return { size: body.length, sha256: createHash("sha256").update(body).digest("hex") };
|
|
2567
2045
|
}
|
|
2568
2046
|
async getRange(key, offset, length) {
|
|
2569
2047
|
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
2570
2048
|
throw new ValidationError(`invalid range offset=${offset} length=${length}`);
|
|
2571
2049
|
}
|
|
2572
|
-
const body = this.require(key);
|
|
2050
|
+
const { body } = this.require(key);
|
|
2573
2051
|
if (offset + length > body.length) {
|
|
2574
2052
|
throw new ValidationError(
|
|
2575
2053
|
`range [${offset}, ${offset + length}) out of bounds for ${body.length}B`
|
|
@@ -2578,12 +2056,20 @@ var MemoryStorageDriver = class {
|
|
|
2578
2056
|
return body.slice(offset, offset + length);
|
|
2579
2057
|
}
|
|
2580
2058
|
async getTail(key, maxBytes) {
|
|
2581
|
-
const body = this.require(key);
|
|
2059
|
+
const { body, version } = this.require(key);
|
|
2582
2060
|
const take = Math.min(Math.max(maxBytes, 0), body.length);
|
|
2583
|
-
return { bytes: body.slice(body.length - take), size: body.length };
|
|
2061
|
+
return { bytes: body.slice(body.length - take), size: body.length, version };
|
|
2584
2062
|
}
|
|
2585
|
-
async delete(key) {
|
|
2586
|
-
|
|
2063
|
+
async delete(key, options) {
|
|
2064
|
+
const k = genObjectKey(key);
|
|
2065
|
+
const ifVersion = options?.ifVersion;
|
|
2066
|
+
const stored = this.objects.get(k);
|
|
2067
|
+
if (ifVersion !== void 0 && stored !== void 0 && stored.version !== ifVersion) {
|
|
2068
|
+
throw new WriteConflictError(
|
|
2069
|
+
`generation ${key.segment}.${key.generation} is another object than the version given; not deleted`
|
|
2070
|
+
);
|
|
2071
|
+
}
|
|
2072
|
+
this.objects.delete(k);
|
|
2587
2073
|
}
|
|
2588
2074
|
async *list(ref) {
|
|
2589
2075
|
validateSegmentRef(ref);
|
|
@@ -2599,11 +2085,11 @@ var MemoryStorageDriver = class {
|
|
|
2599
2085
|
}
|
|
2600
2086
|
}
|
|
2601
2087
|
require(key) {
|
|
2602
|
-
const
|
|
2603
|
-
if (
|
|
2088
|
+
const stored = this.objects.get(genObjectKey(key));
|
|
2089
|
+
if (stored === void 0) {
|
|
2604
2090
|
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
2605
2091
|
}
|
|
2606
|
-
return
|
|
2092
|
+
return stored;
|
|
2607
2093
|
}
|
|
2608
2094
|
};
|
|
2609
2095
|
function genObjectKey(key) {
|
|
@@ -2862,7 +2348,12 @@ var LocalFsStorageDriver = class {
|
|
|
2862
2348
|
root;
|
|
2863
2349
|
exactCase;
|
|
2864
2350
|
capabilities() {
|
|
2865
|
-
return {
|
|
2351
|
+
return {
|
|
2352
|
+
rangeRead: true,
|
|
2353
|
+
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2354
|
+
conditionalPut: false,
|
|
2355
|
+
conditionalDelete: false
|
|
2356
|
+
};
|
|
2866
2357
|
}
|
|
2867
2358
|
async putImmutable(key, write) {
|
|
2868
2359
|
const finalPath = storageObjectPath(this.root, key);
|
|
@@ -2939,7 +2430,12 @@ var LocalFsStorageDriver = class {
|
|
|
2939
2430
|
await handle.close();
|
|
2940
2431
|
}
|
|
2941
2432
|
}
|
|
2942
|
-
|
|
2433
|
+
/**
|
|
2434
|
+
* Idempotent. Ignores `options.ifVersion`, as `conditionalDelete: false` says: see the note on the class for why it
|
|
2435
|
+
* cannot apply one. The parameter is declared so that a wrapper of this driver can hand the port's options on.
|
|
2436
|
+
*/
|
|
2437
|
+
async delete(key, options) {
|
|
2438
|
+
void options;
|
|
2943
2439
|
const path = storageObjectPath(this.root, key);
|
|
2944
2440
|
if (await this.exactCase.differs(path)) return;
|
|
2945
2441
|
await unlink(path).catch((err) => {
|
|
@@ -3038,7 +2534,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3038
2534
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
3039
2535
|
}
|
|
3040
2536
|
const token = newIncarnationToken(this.entropy, current?.record);
|
|
3041
|
-
await this.writeRow(path,
|
|
2537
|
+
await this.writeRow(path, recordFromNew(ref, checked, this.now(), token));
|
|
3042
2538
|
return { token };
|
|
3043
2539
|
});
|
|
3044
2540
|
}
|
|
@@ -3056,11 +2552,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3056
2552
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3057
2553
|
}
|
|
3058
2554
|
const token = nextRegistryToken(current.record, this.entropy);
|
|
3059
|
-
await this.writeRow(
|
|
3060
|
-
path,
|
|
3061
|
-
false,
|
|
3062
|
-
applyRegistryPatch(current.record, checked, this.now(), token)
|
|
3063
|
-
);
|
|
2555
|
+
await this.writeRow(path, applyRegistryPatch(current.record, checked, this.now(), token));
|
|
3064
2556
|
return { token };
|
|
3065
2557
|
});
|
|
3066
2558
|
}
|
|
@@ -3095,12 +2587,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3095
2587
|
} else if (current === null || current.deleted) {
|
|
3096
2588
|
return;
|
|
3097
2589
|
}
|
|
3098
|
-
|
|
3099
|
-
await this.unlinkRow(path);
|
|
3100
|
-
return;
|
|
3101
|
-
}
|
|
3102
|
-
const token = nextRegistryToken(current.record, this.entropy);
|
|
3103
|
-
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
2590
|
+
await this.unlinkRow(path);
|
|
3104
2591
|
});
|
|
3105
2592
|
}
|
|
3106
2593
|
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
@@ -3150,10 +2637,10 @@ var LocalFsRegistryDriver = class {
|
|
|
3150
2637
|
await handle.close();
|
|
3151
2638
|
}
|
|
3152
2639
|
}
|
|
3153
|
-
async writeRow(path,
|
|
2640
|
+
async writeRow(path, record) {
|
|
3154
2641
|
await this.exactCase.refuseVariant(path);
|
|
3155
2642
|
await mkdir2(dirname3(path), { recursive: true });
|
|
3156
|
-
const out = Buffer.from(serializeRegistryEnvelope({ deleted, record }), "utf8");
|
|
2643
|
+
const out = Buffer.from(serializeRegistryEnvelope({ deleted: false, record }), "utf8");
|
|
3157
2644
|
if (out.length > DEFAULT_MAX_ROW_BYTES) {
|
|
3158
2645
|
throw new ValidationError(
|
|
3159
2646
|
`registry row ${out.length}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`
|
|
@@ -3227,6 +2714,14 @@ var LocalFsStorage = class {
|
|
|
3227
2714
|
registry;
|
|
3228
2715
|
};
|
|
3229
2716
|
|
|
2717
|
+
// src/core/reader-defaults.ts
|
|
2718
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
2719
|
+
var REFRESH_RETRY_MS = 500;
|
|
2720
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
2721
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
2722
|
+
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
2723
|
+
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
2724
|
+
|
|
3230
2725
|
// src/core/cooperative.ts
|
|
3231
2726
|
var YIELD_EVERY = 1024;
|
|
3232
2727
|
function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
@@ -3395,8 +2890,6 @@ function indexCapacity(indexLength) {
|
|
|
3395
2890
|
function magicMatches(bytes, offset) {
|
|
3396
2891
|
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
3397
2892
|
}
|
|
3398
|
-
var sizePart = (size) => `${size}:`;
|
|
3399
|
-
var fingerprintFor = (size, footerCrc) => `${sizePart(size)}${footerCrc}`;
|
|
3400
2893
|
function checkSize(tail, size) {
|
|
3401
2894
|
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3402
2895
|
throw new IntegrityError(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
@@ -3473,7 +2966,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3473
2966
|
get fingerprint() {
|
|
3474
2967
|
return fingerprintFor(this.objectSize, this.footerCrc);
|
|
3475
2968
|
}
|
|
3476
|
-
/** Total object bytes (from the one-GET tail read)
|
|
2969
|
+
/** Total object bytes (from the one-GET tail read): the `sizeBytes` a segment's `stat()` reports. */
|
|
3477
2970
|
get sizeBytes() {
|
|
3478
2971
|
return this.objectSize;
|
|
3479
2972
|
}
|
|
@@ -4382,7 +3875,16 @@ function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
|
4382
3875
|
var NONCE_BYTES = 12;
|
|
4383
3876
|
var TAG_BYTES = 16;
|
|
4384
3877
|
var COUNT_BYTES = 8;
|
|
3878
|
+
var SIZE_BYTES = 8;
|
|
3879
|
+
var CRC_BYTES = 4;
|
|
3880
|
+
var FIXED_BYTES = COUNT_BYTES + SIZE_BYTES + CRC_BYTES;
|
|
4385
3881
|
var MAX_CARDINALITY = 2 ** 32;
|
|
3882
|
+
function sizeOfFingerprint(fingerprint) {
|
|
3883
|
+
const parts = fingerprintParts(fingerprint);
|
|
3884
|
+
if (parts === void 0)
|
|
3885
|
+
throw new IntegrityError("registry record summary: a malformed fingerprint");
|
|
3886
|
+
return parts.size;
|
|
3887
|
+
}
|
|
4386
3888
|
var BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
4387
3889
|
function toBase64(bytes) {
|
|
4388
3890
|
let out = "";
|
|
@@ -4409,16 +3911,21 @@ function fromBase64(text) {
|
|
|
4409
3911
|
}
|
|
4410
3912
|
return out;
|
|
4411
3913
|
}
|
|
4412
|
-
function clearSummary(generation, cardinality, metadata) {
|
|
4413
|
-
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata };
|
|
3914
|
+
function clearSummary(generation, cardinality, fingerprint, metadata) {
|
|
3915
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality, fingerprint } : { generation, cardinality, fingerprint, metadata };
|
|
4414
3916
|
}
|
|
4415
|
-
function sealSummary(aead, ref, generation, cardinality, metadata) {
|
|
3917
|
+
function sealSummary(aead, ref, generation, cardinality, fingerprint, metadata) {
|
|
4416
3918
|
const meta = metadataBytes(metadata, (message) => {
|
|
4417
3919
|
throw new IntegrityError(`summary metadata: ${message}`);
|
|
4418
3920
|
});
|
|
4419
|
-
const
|
|
4420
|
-
|
|
4421
|
-
|
|
3921
|
+
const parts = fingerprintParts(fingerprint);
|
|
3922
|
+
if (parts === void 0) throw new IntegrityError("summary: a malformed fingerprint");
|
|
3923
|
+
const plain = new Uint8Array(FIXED_BYTES + (meta?.length ?? 0));
|
|
3924
|
+
const view = new DataView(plain.buffer);
|
|
3925
|
+
view.setBigUint64(0, BigInt(cardinality), true);
|
|
3926
|
+
view.setBigUint64(COUNT_BYTES, BigInt(parts.size), true);
|
|
3927
|
+
view.setUint32(COUNT_BYTES + SIZE_BYTES, parts.footerCrc, true);
|
|
3928
|
+
if (meta !== void 0) plain.set(meta, FIXED_BYTES);
|
|
4422
3929
|
const { nonce, ciphertext, tag } = aead.seal(plain, aadFor(ref, generation, "summary"));
|
|
4423
3930
|
const framed2 = new Uint8Array(nonce.length + ciphertext.length + tag.length);
|
|
4424
3931
|
framed2.set(nonce, 0);
|
|
@@ -4432,8 +3939,10 @@ function openSummary(aead, ref, summary) {
|
|
|
4432
3939
|
};
|
|
4433
3940
|
const framed2 = fromBase64(summary.sealed);
|
|
4434
3941
|
const framing = NONCE_BYTES + TAG_BYTES;
|
|
4435
|
-
if (framed2 === void 0 || framed2.length < framing +
|
|
4436
|
-
fail(
|
|
3942
|
+
if (framed2 === void 0 || framed2.length < framing + FIXED_BYTES || framed2.length > framing + FIXED_BYTES + MAX_METADATA_BYTES) {
|
|
3943
|
+
fail(
|
|
3944
|
+
"the sealed summary is not a nonce, a count, a fingerprint, up to 1 KiB of metadata and a tag"
|
|
3945
|
+
);
|
|
4437
3946
|
}
|
|
4438
3947
|
const bytes = framed2;
|
|
4439
3948
|
let plain;
|
|
@@ -4452,14 +3961,18 @@ function openSummary(aead, ref, summary) {
|
|
|
4452
3961
|
"the sealed summary does not open: tampered bytes, or a summary moved from another generation"
|
|
4453
3962
|
);
|
|
4454
3963
|
}
|
|
4455
|
-
if (plain.length <
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
);
|
|
3964
|
+
if (plain.length < FIXED_BYTES)
|
|
3965
|
+
fail("the sealed summary is shorter than its count and fingerprint");
|
|
3966
|
+
const view = new DataView(plain.buffer, plain.byteOffset, plain.byteLength);
|
|
3967
|
+
const count = view.getBigUint64(0, true);
|
|
4460
3968
|
if (count > BigInt(MAX_CARDINALITY)) fail(`cardinality ${count} is outside 0 to 2^32`);
|
|
4461
|
-
const
|
|
4462
|
-
|
|
3969
|
+
const size = view.getBigUint64(COUNT_BYTES, true);
|
|
3970
|
+
const fingerprint = fingerprintFor(Number(size), view.getUint32(COUNT_BYTES + SIZE_BYTES, true));
|
|
3971
|
+
if (size > BigInt(Number.MAX_SAFE_INTEGER) || fingerprintParts(fingerprint) === void 0) {
|
|
3972
|
+
fail(`the object size ${size} is not one a .crbm object can have`);
|
|
3973
|
+
}
|
|
3974
|
+
const metadata = plain.length === FIXED_BYTES ? void 0 : metadataFromBytes(plain.subarray(FIXED_BYTES), fail);
|
|
3975
|
+
return { cardinality: Number(count), metadata, fingerprint };
|
|
4463
3976
|
}
|
|
4464
3977
|
function usableSummary(ref, row, aead) {
|
|
4465
3978
|
const summary = row.summary;
|
|
@@ -4477,7 +3990,12 @@ function usableSummary(ref, row, aead) {
|
|
|
4477
3990
|
}
|
|
4478
3991
|
}
|
|
4479
3992
|
if (keyed) return void 0;
|
|
4480
|
-
|
|
3993
|
+
if (fingerprintParts(summary.fingerprint) === void 0) return void 0;
|
|
3994
|
+
return {
|
|
3995
|
+
cardinality: summary.cardinality,
|
|
3996
|
+
metadata: summary.metadata,
|
|
3997
|
+
fingerprint: summary.fingerprint
|
|
3998
|
+
};
|
|
4481
3999
|
}
|
|
4482
4000
|
function canonicalOrNone(metadata) {
|
|
4483
4001
|
if (metadata === void 0) return void 0;
|
|
@@ -4494,32 +4012,66 @@ function metadataToCarry(object, authenticated) {
|
|
|
4494
4012
|
return empty(object) && authenticated !== void 0 && !empty(authenticated.metadata) ? authenticated.metadata : object;
|
|
4495
4013
|
}
|
|
4496
4014
|
function summaryOf(ref, generation, description, aead) {
|
|
4497
|
-
|
|
4015
|
+
const { cardinality, fingerprint, metadata } = description;
|
|
4016
|
+
return aead === void 0 ? clearSummary(generation, cardinality, fingerprint, metadata) : sealSummary(aead, ref, generation, cardinality, fingerprint, metadata);
|
|
4498
4017
|
}
|
|
4499
4018
|
|
|
4500
4019
|
// src/core/crbm-storage-source.ts
|
|
4501
|
-
function storageBlobReader(driver, key) {
|
|
4020
|
+
function storageBlobReader(driver, key, noteTail) {
|
|
4502
4021
|
return {
|
|
4503
4022
|
getRange(offset, length) {
|
|
4504
4023
|
return driver.getRange(key, offset, length);
|
|
4505
4024
|
},
|
|
4506
|
-
getTail(maxBytes) {
|
|
4507
|
-
|
|
4025
|
+
getTail: noteTail === void 0 ? (maxBytes) => driver.getTail(key, maxBytes) : async (maxBytes) => {
|
|
4026
|
+
const tail = await driver.getTail(key, maxBytes);
|
|
4027
|
+
noteTail(tail);
|
|
4028
|
+
return tail;
|
|
4508
4029
|
}
|
|
4509
4030
|
};
|
|
4510
4031
|
}
|
|
4032
|
+
function resolvedOf(record) {
|
|
4033
|
+
if (record === null || record.status === "destroyed" || record.currentGen === null) return null;
|
|
4034
|
+
return {
|
|
4035
|
+
generation: record.currentGen,
|
|
4036
|
+
lineage: pointerIdOf(record),
|
|
4037
|
+
wrappedDeks: record.wrappedDeks,
|
|
4038
|
+
summary: record.summary
|
|
4039
|
+
};
|
|
4040
|
+
}
|
|
4041
|
+
function weightOf(resolved) {
|
|
4042
|
+
return SNAPSHOT_BASE_BYTES + (resolved.summary === void 0 ? 0 : JSON.stringify(resolved.summary).length) + (resolved.wrappedDeks === void 0 ? 0 : JSON.stringify(resolved.wrappedDeks).length);
|
|
4043
|
+
}
|
|
4511
4044
|
var notThePinned = (ref, generation) => new NotFoundError(
|
|
4512
4045
|
`segment "${ref.segment}" generation ${generation} is no longer the object this handle pinned`
|
|
4513
4046
|
);
|
|
4514
4047
|
function versionOf(generation, lineage) {
|
|
4515
4048
|
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
4516
4049
|
}
|
|
4050
|
+
var readerVersions = /* @__PURE__ */ new WeakMap();
|
|
4051
|
+
function versionOfReader(reader) {
|
|
4052
|
+
let version = readerVersions.get(reader);
|
|
4053
|
+
if (version === void 0) {
|
|
4054
|
+
version = `${versionOf(reader.generation, reader.lineage)}#${compactFingerprint(reader.fingerprint)}`;
|
|
4055
|
+
readerVersions.set(reader, version);
|
|
4056
|
+
}
|
|
4057
|
+
return version;
|
|
4058
|
+
}
|
|
4059
|
+
var OBJECT_SUFFIX = /#[0-9a-z]+\.[0-9a-z]+$/;
|
|
4060
|
+
function rowVersionOf(version) {
|
|
4061
|
+
return version.replace(OBJECT_SUFFIX, "");
|
|
4062
|
+
}
|
|
4517
4063
|
var PinMoved = class extends Error {
|
|
4518
4064
|
};
|
|
4519
4065
|
function summaryOfGeneration(generation, description) {
|
|
4520
4066
|
const { cardinality, metadata } = description;
|
|
4521
4067
|
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
4522
4068
|
}
|
|
4069
|
+
function sizedSummary(reader) {
|
|
4070
|
+
return {
|
|
4071
|
+
...summaryOfGeneration(reader.generation, describe(reader)),
|
|
4072
|
+
sizeBytes: reader.sizeBytes
|
|
4073
|
+
};
|
|
4074
|
+
}
|
|
4523
4075
|
var sameKeys = (a, b) => a !== void 0 && JSON.stringify(a) === JSON.stringify(b);
|
|
4524
4076
|
function once(fn) {
|
|
4525
4077
|
let pending;
|
|
@@ -4535,62 +4087,95 @@ function once(fn) {
|
|
|
4535
4087
|
};
|
|
4536
4088
|
}
|
|
4537
4089
|
var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
|
|
4538
|
-
async function
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4090
|
+
async function readsObjectOf(snap, reader) {
|
|
4091
|
+
const settled2 = snap.settled;
|
|
4092
|
+
if (settled2 !== void 0) {
|
|
4093
|
+
const other = settled2.reader;
|
|
4094
|
+
return other !== null && (other === reader || versionOfReader(other) === versionOfReader(reader));
|
|
4542
4095
|
}
|
|
4543
4096
|
const live = await snap.target;
|
|
4544
|
-
return live
|
|
4097
|
+
return live !== null && versionOf(live.target.generation, live.target.lineage) === versionOf(reader.generation, reader.lineage);
|
|
4545
4098
|
}
|
|
4546
4099
|
function describe(reader) {
|
|
4547
4100
|
return { cardinality: reader.count(), metadata: reader.metadata };
|
|
4548
4101
|
}
|
|
4549
4102
|
var Snapshot = class _Snapshot {
|
|
4550
|
-
constructor(target, open4, prior) {
|
|
4103
|
+
constructor(resolution, target, open4, prior) {
|
|
4104
|
+
this.resolution = resolution;
|
|
4551
4105
|
this.target = target;
|
|
4552
4106
|
this.open = open4;
|
|
4553
4107
|
this.prior = prior;
|
|
4554
4108
|
}
|
|
4109
|
+
resolution;
|
|
4555
4110
|
target;
|
|
4556
4111
|
open;
|
|
4557
4112
|
prior;
|
|
4558
|
-
/** When the snapshot's refresh clock started. A refresh that failed transiently moves it back, so it lapses sooner. */
|
|
4559
|
-
installedAtMs = 0;
|
|
4560
4113
|
/** Set by the cache, to hear of a reader as it is opened. */
|
|
4561
4114
|
onReader;
|
|
4562
4115
|
opened;
|
|
4563
|
-
|
|
4564
|
-
|
|
4116
|
+
outcome;
|
|
4117
|
+
static live(resolution, target, open4, prior) {
|
|
4118
|
+
return new _Snapshot(resolution, target, open4, prior);
|
|
4565
4119
|
}
|
|
4566
4120
|
static eager(reader) {
|
|
4567
|
-
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
4568
|
-
snap.
|
|
4121
|
+
const snap = new _Snapshot(void 0, void 0, void 0, void 0);
|
|
4122
|
+
snap.hold(reader);
|
|
4569
4123
|
return snap;
|
|
4570
4124
|
}
|
|
4571
4125
|
/** The reader, opened by the first call. */
|
|
4572
4126
|
get reader() {
|
|
4573
4127
|
if (this.opened === void 0) {
|
|
4574
4128
|
const target = this.target;
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4129
|
+
const opened = this.hold(
|
|
4130
|
+
target.then(async (live) => {
|
|
4131
|
+
if (live === null) return null;
|
|
4132
|
+
const kept = await this.reuse(live);
|
|
4133
|
+
this.prior = void 0;
|
|
4134
|
+
return kept ?? this.open(live);
|
|
4135
|
+
})
|
|
4136
|
+
);
|
|
4137
|
+
this.onReader?.(opened);
|
|
4138
|
+
return opened;
|
|
4582
4139
|
}
|
|
4583
4140
|
return this.opened;
|
|
4584
4141
|
}
|
|
4142
|
+
/**
|
|
4143
|
+
* Take `reader` as this snapshot's, and note the reader once it has opened. A failed open is not noted: the reader
|
|
4144
|
+
* cache forgets the snapshot as the failure lands, and whoever awaits the reader meets the error.
|
|
4145
|
+
*/
|
|
4146
|
+
hold(reader) {
|
|
4147
|
+
this.opened = reader;
|
|
4148
|
+
reader.then(
|
|
4149
|
+
(r) => {
|
|
4150
|
+
this.outcome = { reader: r };
|
|
4151
|
+
},
|
|
4152
|
+
() => void 0
|
|
4153
|
+
);
|
|
4154
|
+
return reader;
|
|
4155
|
+
}
|
|
4585
4156
|
/** The reader if one has been asked for, without asking for one. */
|
|
4586
4157
|
get openedReader() {
|
|
4587
4158
|
return this.opened;
|
|
4588
4159
|
}
|
|
4589
|
-
/**
|
|
4160
|
+
/**
|
|
4161
|
+
* The reader once it has opened (`null` for no generation). Undefined while no reader has been asked for, while the
|
|
4162
|
+
* open is under way, and after an open that failed.
|
|
4163
|
+
*/
|
|
4164
|
+
get settled() {
|
|
4165
|
+
return this.outcome;
|
|
4166
|
+
}
|
|
4167
|
+
/**
|
|
4168
|
+
* The prior snapshot's reader, if it is of what this snapshot resolved: the same generation and `pointerId`, and the
|
|
4169
|
+
* object the row's summary names, where it names one. A registry renews `pointerId` with every write that changes the
|
|
4170
|
+
* summary, so the last holds whenever the first two do; it is checked anyway, since it costs no request, and a reader
|
|
4171
|
+
* kept across a write that should have renewed it would serve another object's chunks.
|
|
4172
|
+
*/
|
|
4590
4173
|
async reuse(live) {
|
|
4591
4174
|
const prior = this.prior;
|
|
4592
4175
|
const reader = prior === void 0 ? null : await prior.catch(() => null);
|
|
4593
|
-
|
|
4176
|
+
if (reader === null || !readerIsOf(reader, live)) return null;
|
|
4177
|
+
const named = await live.summary().catch(() => void 0);
|
|
4178
|
+
return named === void 0 || named.fingerprint === reader.fingerprint ? reader : null;
|
|
4594
4179
|
}
|
|
4595
4180
|
/** Keep the prior snapshot's reader when the refresh found its generation, so a count does not drop it. */
|
|
4596
4181
|
async adopt(live) {
|
|
@@ -4598,15 +4183,13 @@ var Snapshot = class _Snapshot {
|
|
|
4598
4183
|
const reader = await this.reuse(live);
|
|
4599
4184
|
this.prior = void 0;
|
|
4600
4185
|
if (reader !== null && this.opened === void 0) {
|
|
4601
|
-
this.
|
|
4602
|
-
this.onReader?.(this.opened);
|
|
4186
|
+
this.onReader?.(this.hold(Promise.resolve(reader)));
|
|
4603
4187
|
}
|
|
4604
4188
|
}
|
|
4605
4189
|
};
|
|
4606
4190
|
var SNAPSHOT_BASE_BYTES = 256;
|
|
4607
4191
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
4608
4192
|
var YIELD_EVERY_IDS = 1 << 14;
|
|
4609
|
-
var RECENT_KEYS = 8;
|
|
4610
4193
|
var CrbmStorageChunkSource = class {
|
|
4611
4194
|
constructor(driver, options = {}) {
|
|
4612
4195
|
this.driver = driver;
|
|
@@ -4651,6 +4234,11 @@ var CrbmStorageChunkSource = class {
|
|
|
4651
4234
|
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
4652
4235
|
clock: clock ?? { now: () => 0 }
|
|
4653
4236
|
});
|
|
4237
|
+
this.resolutions = this.timedRefresh ? new BoundedLru({
|
|
4238
|
+
maxEntries: RESOLUTIONS_PER_OPEN_SEGMENT * (maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS),
|
|
4239
|
+
maxBytes: (maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES) / RESOLUTION_BYTES_DIVISOR,
|
|
4240
|
+
clock: { now: () => 0 }
|
|
4241
|
+
}) : void 0;
|
|
4654
4242
|
this.distrusted = new BoundedLru({
|
|
4655
4243
|
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
4656
4244
|
clock: { now: () => 0 }
|
|
@@ -4666,7 +4254,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4666
4254
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
4667
4255
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
4668
4256
|
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
4669
|
-
* this with a
|
|
4257
|
+
* this with a cache keyed by {@link currentVersion}, which names the object a reader opened, so a bump never serves
|
|
4258
|
+
* a stale decoded chunk, and neither does a number taken again by another object. Without a clock or a
|
|
4670
4259
|
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
4671
4260
|
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
4672
4261
|
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
@@ -4674,6 +4263,21 @@ var CrbmStorageChunkSource = class {
|
|
|
4674
4263
|
* steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
|
|
4675
4264
|
*/
|
|
4676
4265
|
snapshots;
|
|
4266
|
+
/**
|
|
4267
|
+
* Each segment's resolution, kept apart from its reader, by segment: what a read of its row found, for the TTL from
|
|
4268
|
+
* the instant that read was sent, whether or not the reader cache still holds the segment's reader. So letting a reader
|
|
4269
|
+
* go moves nothing: the next read builds a snapshot on the resolution, reads no row, and opens the object when it needs
|
|
4270
|
+
* the index or a chunk the chunk cache does not hold (a version, a count and a stat from the row's summary need
|
|
4271
|
+
* neither). It holds the resolved fields only ({@link Resolved}): the wrapped keys, never an unwrapped one, so a
|
|
4272
|
+
* snapshot built on it unwraps the key again through the keystore. It never holds a resolution that found no
|
|
4273
|
+
* generation: the next read of such a segment reads the row.
|
|
4274
|
+
*
|
|
4275
|
+
* Only a source with a timed refresh has one: without it, letting a reader go is one of the few things that ever moves
|
|
4276
|
+
* a read on, and keeping the resolution would keep it forever. **Bounded** by count and by bytes, both derived from
|
|
4277
|
+
* the reader cache's: {@link RESOLUTIONS_PER_OPEN_SEGMENT} times its count bound, and its byte bound over
|
|
4278
|
+
* {@link RESOLUTION_BYTES_DIVISOR}. An entry heavier than the byte bound is kept alone, as the reader cache keeps one.
|
|
4279
|
+
*/
|
|
4280
|
+
resolutions;
|
|
4677
4281
|
/**
|
|
4678
4282
|
* Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
|
|
4679
4283
|
* and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
|
|
@@ -4703,6 +4307,13 @@ var CrbmStorageChunkSource = class {
|
|
|
4703
4307
|
* the disagreement again.
|
|
4704
4308
|
*/
|
|
4705
4309
|
distrusted;
|
|
4310
|
+
/**
|
|
4311
|
+
* The readers held to the row's summary: by a live open ({@link openLive}), or by a live pin that shared one. A pin
|
|
4312
|
+
* that finds one memoised under its version shares it as it is: the version names the row's `pointerId`, and every
|
|
4313
|
+
* write of the summary renews it, so the summary it was held to is the one the row has now. Any other memoised reader
|
|
4314
|
+
* (a `pinAt`'s, held to the fingerprint its caller named) is held to the row's summary before a live pin shares it.
|
|
4315
|
+
*/
|
|
4316
|
+
rowChecked = /* @__PURE__ */ new WeakSet();
|
|
4706
4317
|
registry;
|
|
4707
4318
|
keystore;
|
|
4708
4319
|
requireEncryption;
|
|
@@ -4713,97 +4324,149 @@ var CrbmStorageChunkSource = class {
|
|
|
4713
4324
|
currentGenTtlMs;
|
|
4714
4325
|
/**
|
|
4715
4326
|
* Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
|
|
4716
|
-
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes
|
|
4717
|
-
*
|
|
4327
|
+
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes and the TTL
|
|
4328
|
+
* lapse, so it is decided once.
|
|
4718
4329
|
*/
|
|
4719
4330
|
timedRefresh;
|
|
4720
4331
|
/**
|
|
4721
|
-
* The
|
|
4722
|
-
*
|
|
4723
|
-
*
|
|
4332
|
+
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
4333
|
+
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
4334
|
+
* microsecond each time.
|
|
4724
4335
|
*/
|
|
4725
|
-
|
|
4336
|
+
keys = new KeptSegmentKeys(segmentKey);
|
|
4726
4337
|
keyOf(ref) {
|
|
4727
|
-
|
|
4728
|
-
if (k.segment === ref.segment && k.namespace === ref.namespace) return k.key;
|
|
4729
|
-
}
|
|
4730
|
-
const key = segmentKey(ref);
|
|
4731
|
-
this.recentKeys.unshift({ namespace: ref.namespace, segment: ref.segment, key });
|
|
4732
|
-
if (this.recentKeys.length > RECENT_KEYS) this.recentKeys.pop();
|
|
4733
|
-
return key;
|
|
4338
|
+
return this.keys.of(ref);
|
|
4734
4339
|
}
|
|
4735
4340
|
/**
|
|
4736
|
-
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window
|
|
4737
|
-
*
|
|
4341
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window: the snapshot the reader cache holds,
|
|
4342
|
+
* while its resolution is inside the TTL. With a timed refresh, a segment whose reader the cache let go is served a
|
|
4343
|
+
* snapshot built on its kept resolution, while that is inside the TTL: no row is read, and the reader opens only when a
|
|
4344
|
+
* read needs one. Otherwise the pointer is read now. The snapshot holds the resolved target, and its reader once a read
|
|
4345
|
+
* has asked for one.
|
|
4738
4346
|
*/
|
|
4739
4347
|
liveSnapshot(ref) {
|
|
4740
4348
|
const key = this.keyOf(ref);
|
|
4741
4349
|
const existing = this.snapshots.get(key);
|
|
4742
|
-
if (existing
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
);
|
|
4350
|
+
if (existing !== void 0 && !this.expired(existing.resolution)) return existing;
|
|
4351
|
+
const kept = this.resolutions?.get(key);
|
|
4352
|
+
if (kept !== void 0 && kept !== existing?.resolution && !this.expired(kept)) {
|
|
4353
|
+
return this.install(key, this.snapshotOn(ref, kept, existing));
|
|
4747
4354
|
}
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
Snapshot.live(
|
|
4752
|
-
this.refreshedTarget(ref, existing, () => this.retrySoon(snap)),
|
|
4753
|
-
(live) => this.openLive(ref, live),
|
|
4754
|
-
existing.openedReader
|
|
4755
|
-
)
|
|
4756
|
-
);
|
|
4757
|
-
return snap;
|
|
4355
|
+
const resolution = this.resolveNow(ref, existing?.resolution);
|
|
4356
|
+
this.remember(key, resolution);
|
|
4357
|
+
return this.install(key, this.snapshotOn(ref, resolution, existing));
|
|
4758
4358
|
}
|
|
4759
|
-
/**
|
|
4760
|
-
|
|
4761
|
-
|
|
4359
|
+
/**
|
|
4360
|
+
* Read the segment's pointer now. The read is sent before this returns, and the TTL counts from this instant: a read
|
|
4361
|
+
* that answers late does not lengthen the life of what it found. The row read is always fresh, so what a snapshot on
|
|
4362
|
+
* it says of the generation (its summary included) is what the row says now, never `prior`'s.
|
|
4363
|
+
*
|
|
4364
|
+
* A read that fails with a transient fault answers what `prior` found, when it found a generation, and is asked again
|
|
4365
|
+
* soon ({@link retrySoon}): `prior` is the resolution of the snapshot the reader cache still holds, so an outage of the
|
|
4366
|
+
* registry keeps that snapshot's generation, and its reader and unwrapped key where it has them; one resolved without
|
|
4367
|
+
* an open, for a count or a generation lookup, unwraps the key from `prior`'s wrapped keys when a read first needs
|
|
4368
|
+
* it. The outage ends shortly after the registry answers.
|
|
4369
|
+
* That answer is the snapshot's alone ({@link Resolution.rodeOut}): it is not kept as the segment's resolution, and a
|
|
4370
|
+
* snapshot built on it after the reader cache let that one go fails with the fault. With no `prior` the fault
|
|
4371
|
+
* fails the read, as a cold resolve's does. If `prior` found none or failed, the pointer is read once more rather than
|
|
4372
|
+
* a dead resolution re-armed (else the segment reads empty for a whole TTL window). Anything else — an access denial, a corrupt row, a
|
|
4373
|
+
* registry that answers NotFound — fails the read exactly as a cold resolve of the segment would, and the resolution
|
|
4374
|
+
* and the snapshot on it are forgotten, so a reader and the key it unwrapped do not outlive a refresh that could not
|
|
4375
|
+
* be trusted.
|
|
4376
|
+
*/
|
|
4377
|
+
resolveNow(ref, prior) {
|
|
4378
|
+
const sentAtMs = this.now();
|
|
4379
|
+
const sent = this.resolve(ref);
|
|
4380
|
+
if (prior === void 0) return { resolved: sent, sentAtMs };
|
|
4381
|
+
const resolution = {
|
|
4382
|
+
sentAtMs,
|
|
4383
|
+
resolved: sent.catch(async (err) => {
|
|
4384
|
+
if (!isTransientError(err)) throw err;
|
|
4385
|
+
const before = await prior.resolved.catch(() => null);
|
|
4386
|
+
if (before === null) return this.resolve(ref);
|
|
4387
|
+
this.retrySoon(resolution);
|
|
4388
|
+
const key = this.keyOf(ref);
|
|
4389
|
+
if (this.resolutions?.peek(key) === resolution) this.resolutions.delete(key);
|
|
4390
|
+
resolution.rodeOut = { from: prior, fault: err };
|
|
4391
|
+
prior.rodeOut = void 0;
|
|
4392
|
+
return before;
|
|
4393
|
+
})
|
|
4394
|
+
};
|
|
4395
|
+
return resolution;
|
|
4762
4396
|
}
|
|
4763
4397
|
/**
|
|
4764
|
-
* A refresh failed transiently and the prior
|
|
4765
|
-
*
|
|
4766
|
-
*
|
|
4398
|
+
* A refresh failed transiently and the prior resolution keeps serving: make it lapse after {@link REFRESH_RETRY_MS}
|
|
4399
|
+
* (never longer than the TTL) rather than after a whole TTL, so the next read after that asks the registry again. It
|
|
4400
|
+
* costs no read of its own; it only changes when the next refresh happens.
|
|
4767
4401
|
*/
|
|
4768
|
-
retrySoon(
|
|
4769
|
-
|
|
4402
|
+
retrySoon(resolution) {
|
|
4403
|
+
resolution.sentAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
4770
4404
|
}
|
|
4771
4405
|
/**
|
|
4772
|
-
*
|
|
4773
|
-
*
|
|
4774
|
-
*
|
|
4406
|
+
* Keep `resolution` as the segment's, from the instant its read was sent, so the reads that come while it is in flight
|
|
4407
|
+
* share it, and a read after the reader cache let the segment go builds on it. Its weight is reported once it lands,
|
|
4408
|
+
* and one that finds no generation, or fails, is forgotten. Both are identity-guarded, as the reader cache's are, and
|
|
4409
|
+
* nothing is installed when the read lands: an `invalidate` while it is in flight, or a newer resolution, is never
|
|
4410
|
+
* undone by its answer.
|
|
4775
4411
|
*/
|
|
4776
|
-
|
|
4777
|
-
const
|
|
4778
|
-
|
|
4779
|
-
|
|
4780
|
-
|
|
4781
|
-
if (
|
|
4782
|
-
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
}
|
|
4412
|
+
remember(key, resolution) {
|
|
4413
|
+
const cache = this.resolutions;
|
|
4414
|
+
if (cache === void 0) return;
|
|
4415
|
+
cache.set(key, resolution);
|
|
4416
|
+
const forgetIfStale = () => {
|
|
4417
|
+
if (cache.peek(key) === resolution) cache.delete(key);
|
|
4418
|
+
};
|
|
4419
|
+
resolution.resolved.then((resolved) => {
|
|
4420
|
+
if (resolved === null) forgetIfStale();
|
|
4421
|
+
else if (cache.peek(key) === resolution) cache.setWeight(key, weightOf(resolved));
|
|
4422
|
+
}, forgetIfStale);
|
|
4787
4423
|
}
|
|
4788
4424
|
/**
|
|
4789
|
-
*
|
|
4790
|
-
*
|
|
4791
|
-
*
|
|
4425
|
+
* A live snapshot on `resolution`. `before` is the snapshot it replaces, while the reader cache still holds one: its
|
|
4426
|
+
* reader is kept if the resolution finds the same generation, `pointerId` and object ({@link Snapshot.reuse}), and its
|
|
4427
|
+
* unwrapped key if the same wrapped keys. A snapshot built after the reader cache let the segment go has neither, so
|
|
4428
|
+
* its open asks the keystore again: a key is held only while a reader of the segment is.
|
|
4429
|
+
*/
|
|
4430
|
+
snapshotOn(ref, resolution, before) {
|
|
4431
|
+
return Snapshot.live(
|
|
4432
|
+
resolution,
|
|
4433
|
+
this.targetOn(ref, resolution, before),
|
|
4434
|
+
(live) => this.openLive(ref, live),
|
|
4435
|
+
before?.openedReader
|
|
4436
|
+
);
|
|
4437
|
+
}
|
|
4438
|
+
/** What the snapshot on `resolution` that replaces `before` reads: see {@link snapshotOn}. */
|
|
4439
|
+
targetOn(ref, resolution, before) {
|
|
4440
|
+
const prior = before?.target;
|
|
4441
|
+
const replaces = before?.resolution;
|
|
4442
|
+
return resolution.resolved.then(async (resolved) => {
|
|
4443
|
+
if (resolved === null) return null;
|
|
4444
|
+
const rodeOut = resolution.rodeOut;
|
|
4445
|
+
if (rodeOut !== void 0 && replaces !== rodeOut.from) throw rodeOut.fault;
|
|
4446
|
+
return this.liveOf(ref, resolved, prior === void 0 ? null : await prior.catch(() => null));
|
|
4447
|
+
});
|
|
4448
|
+
}
|
|
4449
|
+
/**
|
|
4450
|
+
* Memoize a snapshot as the segment's, forgetting it later if it resolves to null / throws (identity-guarded, so a
|
|
4451
|
+
* stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot **synchronously** is what lets
|
|
4452
|
+
* concurrent callers coalesce onto one in-flight resolve.
|
|
4792
4453
|
*/
|
|
4793
4454
|
install(key, snap) {
|
|
4794
|
-
snap.installedAtMs = this.now();
|
|
4795
4455
|
this.snapshots.set(key, snap);
|
|
4796
4456
|
const forgetIfStale = () => {
|
|
4797
4457
|
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
4798
4458
|
};
|
|
4799
4459
|
snap.onReader = (reader) => {
|
|
4800
|
-
reader.then(
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4460
|
+
reader.then(
|
|
4461
|
+
(r) => {
|
|
4462
|
+
if (r === null) {
|
|
4463
|
+
forgetIfStale();
|
|
4464
|
+
return;
|
|
4465
|
+
}
|
|
4466
|
+
if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedBytes);
|
|
4467
|
+
},
|
|
4468
|
+
() => this.dropStale(key, snap)
|
|
4469
|
+
);
|
|
4807
4470
|
};
|
|
4808
4471
|
if (snap.target !== void 0) {
|
|
4809
4472
|
snap.target.then(async (live) => {
|
|
@@ -4823,53 +4486,46 @@ var CrbmStorageChunkSource = class {
|
|
|
4823
4486
|
now() {
|
|
4824
4487
|
return this.clock ? this.clock.now() : 0;
|
|
4825
4488
|
}
|
|
4826
|
-
/**
|
|
4827
|
-
* How often this source re-reads a segment's pointer while the segment is being read, in ms: its TTL, or 0 when
|
|
4828
|
-
* it has no timed refresh (no clock, no registry, or a TTL of 0). The grounded cost report prices the refresh at
|
|
4829
|
-
* this.
|
|
4830
|
-
*/
|
|
4831
|
-
get pointerRefreshMs() {
|
|
4832
|
-
return this.timedRefresh ? this.currentGenTtlMs : 0;
|
|
4833
|
-
}
|
|
4834
4489
|
/**
|
|
4835
4490
|
* Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
|
|
4836
|
-
* would
|
|
4837
|
-
*
|
|
4838
|
-
*
|
|
4491
|
+
* would. In this order: an invalidation has happened since `epoch` (what the store that erased stops on, at once);
|
|
4492
|
+
* the TTL of the snapshot's resolution has lapsed; or that resolution is no longer the segment's: another read
|
|
4493
|
+
* installed a newer one (a refresh, or a heal), or the resolution cache let it go. With no resolution cache (no timed
|
|
4494
|
+
* refresh) the last test is of the snapshot itself, so there the reader cache letting it go is a move, as it is one of
|
|
4495
|
+
* the few things that ever moves such a store's read on. With one, it is not. Else nothing can have moved it, and the
|
|
4496
|
+
* check costs a compare and a lookup in each cache.
|
|
4839
4497
|
*/
|
|
4840
4498
|
mayHaveMoved(key, snap, epoch) {
|
|
4841
|
-
|
|
4499
|
+
const resolution = snap.resolution;
|
|
4500
|
+
if (this.invalidations !== epoch || this.expired(resolution)) return true;
|
|
4501
|
+
const held = this.snapshots.get(key);
|
|
4502
|
+
return this.resolutions === void 0 ? held !== snap : this.resolutions.get(key) !== resolution;
|
|
4842
4503
|
}
|
|
4843
|
-
expired(
|
|
4844
|
-
return this.timedRefresh && this.now() -
|
|
4504
|
+
expired(resolution) {
|
|
4505
|
+
return this.timedRefresh && this.now() - resolution.sentAtMs >= this.currentGenTtlMs;
|
|
4845
4506
|
}
|
|
4846
4507
|
/**
|
|
4847
|
-
* The segment's current generation number
|
|
4848
|
-
*
|
|
4849
|
-
*
|
|
4508
|
+
* The segment's current generation number, as its snapshot's resolution found it: no backend read within the TTL
|
|
4509
|
+
* window, and no open of the object, ever. `null` if the segment has no committed generation. A generation swept from
|
|
4510
|
+
* under the resolution is met by the read that fetches from it, which heals ({@link withFreshSnapshot}). Since it
|
|
4511
|
+
* opens nothing, the only error it meets is the resolution's own, and it fails with that error at once: there is
|
|
4512
|
+
* nothing to heal. The reads that can open the object (`listChunkKeys`, `summary`, `currentVersion` and the rest)
|
|
4513
|
+
* resolve once more on any `NotFoundError`, a registry's among them, since theirs can be a swept generation.
|
|
4850
4514
|
*
|
|
4851
|
-
*
|
|
4852
|
-
*
|
|
4853
|
-
*
|
|
4854
|
-
*
|
|
4855
|
-
* operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
|
|
4856
|
-
* no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
|
|
4857
|
-
* frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
|
|
4858
|
-
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
4515
|
+
* It is spelled out rather than delegated because a lookup like this runs once per operand of every
|
|
4516
|
+
* `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with no backend call at all.
|
|
4517
|
+
* Routing it through the generic helper cost ~115 ns/op on that path (a second async frame, a per-call closure, and
|
|
4518
|
+
* an `await` on a plain number).
|
|
4859
4519
|
*/
|
|
4860
4520
|
async currentGeneration(ref) {
|
|
4861
4521
|
const snap = this.liveSnapshot(ref);
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
if (!isNotFoundError(err)) throw err;
|
|
4866
|
-
this.dropStale(segmentKey(ref), snap);
|
|
4867
|
-
return (await this.resolvedReader(ref))?.generation ?? null;
|
|
4868
|
-
}
|
|
4522
|
+
const settled2 = snap.settled;
|
|
4523
|
+
if (settled2 !== void 0) return settled2.reader?.generation ?? null;
|
|
4524
|
+
return (await snap.target)?.target.generation ?? null;
|
|
4869
4525
|
}
|
|
4870
4526
|
/**
|
|
4871
|
-
* Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with
|
|
4872
|
-
* reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
4527
|
+
* Forget everything derived from `ref`: its resolution, the resolved snapshot and the open reader behind it (and with
|
|
4528
|
+
* the reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
4873
4529
|
* resolves the segment again.
|
|
4874
4530
|
*
|
|
4875
4531
|
* Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
|
|
@@ -4882,6 +4538,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4882
4538
|
this.invalidations += 1;
|
|
4883
4539
|
const key = segmentKey(ref);
|
|
4884
4540
|
this.snapshots.delete(key);
|
|
4541
|
+
this.resolutions?.delete(key);
|
|
4885
4542
|
const pinnedPrefix = `${key}@`;
|
|
4886
4543
|
this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
4887
4544
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
@@ -4889,31 +4546,53 @@ var CrbmStorageChunkSource = class {
|
|
|
4889
4546
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
4890
4547
|
}
|
|
4891
4548
|
/**
|
|
4892
|
-
* `<generation>` for a registry-less source, or `<generation>:<
|
|
4893
|
-
*
|
|
4894
|
-
*
|
|
4895
|
-
*
|
|
4896
|
-
*
|
|
4897
|
-
*
|
|
4898
|
-
*
|
|
4549
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<pointerId>#<object>` with one: the generation
|
|
4550
|
+
* the segment's snapshot resolved, and the object a reader of it opens, by that object's size and footer checksum in
|
|
4551
|
+
* base 36. The row's `pointerId` moves only with a write that names a field a read resolves through, at any value, so
|
|
4552
|
+
* a lease or a policy write (a `setRetention`) leaves the version, the open reader and the decoded chunks as they were.
|
|
4553
|
+
* Two objects do not share a version string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
4554
|
+
* a number came to name a second one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
4555
|
+
* number), a number taken again within one row once its object was deleted, or an object put back under its key from
|
|
4556
|
+
* outside the library. The version is opaque: compare two for equality.
|
|
4557
|
+
*
|
|
4558
|
+
* Where the row's summary names the object, the version is answered from the resolution, with no open: every live
|
|
4559
|
+
* open of the generation is held to that object ({@link openLive}), refused as a move otherwise, so the one object any
|
|
4560
|
+
* reader of this resolution can name is the one the summary names, and a version named from it is the one that
|
|
4561
|
+
* reader's would be. So a `has()` whose chunk is in the chunk cache opens nothing, after the reader cache let
|
|
4562
|
+
* the segment go included (a read that needs the index opens the object for it, and on an encrypted segment the
|
|
4563
|
+
* key is unwrapped to read the row's sealed summary); an object under the number that is not the row's is met by the read that fetches from it,
|
|
4564
|
+
* which refuses it and heals. Where the row has no summary it can use, or there is no registry, nothing names the
|
|
4565
|
+
* object but the object: the reader is opened, and the version names what it opened, however long after its row read
|
|
4566
|
+
* that is.
|
|
4899
4567
|
*
|
|
4900
|
-
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
4901
|
-
*
|
|
4902
|
-
*
|
|
4903
|
-
*
|
|
4904
|
-
*
|
|
4568
|
+
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its chunk
|
|
4569
|
+
* cache, so an open it makes heals a swept generation as a read does: unhealed, a cold read racing a publish and a
|
|
4570
|
+
* `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race on a chunk fetch heals. Spelled
|
|
4571
|
+
* out rather than delegated for the reason `currentGeneration` gives: it is almost always served from the cached
|
|
4572
|
+
* snapshot with no backend call at all. One retry, then propagate.
|
|
4905
4573
|
*/
|
|
4906
4574
|
async currentVersion(ref) {
|
|
4907
4575
|
const snap = this.liveSnapshot(ref);
|
|
4908
|
-
|
|
4576
|
+
const settled2 = snap.settled;
|
|
4577
|
+
if (settled2 !== void 0) {
|
|
4578
|
+
return settled2.reader === null ? null : versionOfReader(settled2.reader);
|
|
4579
|
+
}
|
|
4909
4580
|
try {
|
|
4910
|
-
|
|
4581
|
+
return await this.versionOn(snap);
|
|
4911
4582
|
} catch (err) {
|
|
4912
4583
|
if (!isNotFoundError(err)) throw err;
|
|
4913
4584
|
this.dropStale(segmentKey(ref), snap);
|
|
4914
|
-
|
|
4585
|
+
return await this.versionOn(this.liveSnapshot(ref));
|
|
4915
4586
|
}
|
|
4916
|
-
|
|
4587
|
+
}
|
|
4588
|
+
/** {@link currentVersion} of one snapshot: named by the row's summary when it can be, else by the reader, opened. */
|
|
4589
|
+
async versionOn(snap) {
|
|
4590
|
+
const live = await snap.target;
|
|
4591
|
+
if (live === null) return null;
|
|
4592
|
+
const named = await live.version();
|
|
4593
|
+
if (named !== void 0) return named;
|
|
4594
|
+
const reader = await snap.reader;
|
|
4595
|
+
return reader === null ? null : versionOfReader(reader);
|
|
4917
4596
|
}
|
|
4918
4597
|
/**
|
|
4919
4598
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
@@ -4973,11 +4652,10 @@ var CrbmStorageChunkSource = class {
|
|
|
4973
4652
|
if (record.currentGen === null || generation > record.currentGen) {
|
|
4974
4653
|
throw gone("it is not a published generation of this segment");
|
|
4975
4654
|
}
|
|
4976
|
-
|
|
4655
|
+
const lineage = pointerIdOf(record);
|
|
4977
4656
|
if (lease !== void 0) {
|
|
4978
4657
|
const taken = await lease.take(record);
|
|
4979
4658
|
if (taken === "moved") throw gone("the segment moved while its lease was taken");
|
|
4980
|
-
lineage = taken.token;
|
|
4981
4659
|
}
|
|
4982
4660
|
version = versionOf(generation, lineage);
|
|
4983
4661
|
const key = this.pinnedKey(ref, version);
|
|
@@ -5015,12 +4693,11 @@ var CrbmStorageChunkSource = class {
|
|
|
5015
4693
|
return { generation, version, fingerprint };
|
|
5016
4694
|
}
|
|
5017
4695
|
async pinOnce(ref, lease) {
|
|
5018
|
-
|
|
4696
|
+
const live = await this.resolveLive(ref);
|
|
5019
4697
|
if (live === null) return null;
|
|
5020
4698
|
if (lease !== void 0 && live.row !== void 0) {
|
|
5021
4699
|
const taken = await lease.take(live.row);
|
|
5022
4700
|
if (taken === "moved") throw new PinMoved();
|
|
5023
|
-
live = { ...live, target: { ...live.target, lineage: taken.token } };
|
|
5024
4701
|
}
|
|
5025
4702
|
const target = live.target;
|
|
5026
4703
|
const version = versionOf(target.generation, target.lineage);
|
|
@@ -5040,8 +4717,17 @@ var CrbmStorageChunkSource = class {
|
|
|
5040
4717
|
entry ??= this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5041
4718
|
reader = await entry.reader;
|
|
5042
4719
|
if (reader === null) return null;
|
|
5043
|
-
|
|
5044
|
-
const
|
|
4720
|
+
let named;
|
|
4721
|
+
const replaced = async (r) => {
|
|
4722
|
+
if (this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0)
|
|
4723
|
+
return true;
|
|
4724
|
+
if (this.rowChecked.has(r)) return false;
|
|
4725
|
+
const fingerprint = await (named ??= live.summary().then((d) => d?.fingerprint));
|
|
4726
|
+
if (fingerprint !== void 0 && r.fingerprint !== fingerprint) return true;
|
|
4727
|
+
this.rowChecked.add(r);
|
|
4728
|
+
return false;
|
|
4729
|
+
};
|
|
4730
|
+
const known = !opened && await replaced(reader);
|
|
5045
4731
|
if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
|
|
5046
4732
|
this.install(key, Snapshot.eager(Promise.resolve(reader)));
|
|
5047
4733
|
}
|
|
@@ -5050,7 +4736,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5050
4736
|
let fresh = current !== void 0 && current !== entry ? current : this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5051
4737
|
reader = await fresh.reader;
|
|
5052
4738
|
if (reader === null) return null;
|
|
5053
|
-
if (fresh === current && replaced(reader)) {
|
|
4739
|
+
if (fresh === current && await replaced(reader)) {
|
|
5054
4740
|
fresh = this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5055
4741
|
reader = await fresh.reader;
|
|
5056
4742
|
if (reader === null) return null;
|
|
@@ -5067,7 +4753,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5067
4753
|
}
|
|
5068
4754
|
/**
|
|
5069
4755
|
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
5070
|
-
* a name purged and loaded again gets a new row, and with it a new
|
|
4756
|
+
* a name purged and loaded again gets a new row, and with it a new `pointerId`. Without one the version is the bare
|
|
5071
4757
|
* generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
|
|
5072
4758
|
*/
|
|
5073
4759
|
pinnedKey(ref, version, fingerprint) {
|
|
@@ -5103,9 +4789,10 @@ var CrbmStorageChunkSource = class {
|
|
|
5103
4789
|
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
5104
4790
|
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
5105
4791
|
* so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
|
|
5106
|
-
* already cached from the old one. The row's
|
|
5107
|
-
*
|
|
5108
|
-
*
|
|
4792
|
+
* already cached from the old one. The row's `pointerId` cannot tell them apart, since it moves with every write
|
|
4793
|
+
* that changes what the row resolves to, a publish included, while the pinned object stays what it was; the
|
|
4794
|
+
* object's fingerprint can, and a pin whose object has been replaced fails, as one whose generation has been swept
|
|
4795
|
+
* does. The memo is keyed so that two pins of one generation number in two
|
|
5109
4796
|
* incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
|
|
5110
4797
|
*/
|
|
5111
4798
|
async readerAt(ref, generation, version, fingerprint) {
|
|
@@ -5147,7 +4834,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5147
4834
|
if (record === null || record.status === "destroyed") return null;
|
|
5148
4835
|
return this.openForTarget(ref, {
|
|
5149
4836
|
generation,
|
|
5150
|
-
lineage: record
|
|
4837
|
+
lineage: pointerIdOf(record),
|
|
5151
4838
|
wrappedDeks: record.wrappedDeks
|
|
5152
4839
|
});
|
|
5153
4840
|
}
|
|
@@ -5228,7 +4915,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5228
4915
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5229
4916
|
return reader === null ? null : reader.cardinalities();
|
|
5230
4917
|
}
|
|
5231
|
-
/**
|
|
4918
|
+
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
5232
4919
|
async sizeOfAt(ref, generation, held) {
|
|
5233
4920
|
validateUserRef(ref);
|
|
5234
4921
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
@@ -5256,19 +4943,29 @@ var CrbmStorageChunkSource = class {
|
|
|
5256
4943
|
return false;
|
|
5257
4944
|
}
|
|
5258
4945
|
/**
|
|
5259
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
5260
|
-
* fresher snapshot is never clobbered. The identity guard is the whole point: `install`
|
|
5261
|
-
* wholesale, so comparing anything else would drop a good
|
|
4946
|
+
* Evict a snapshot we just failed to read from, and the resolution it was built on — each matched by identity, so a
|
|
4947
|
+
* concurrent call's fresher snapshot or resolution is never clobbered. The identity guard is the whole point: `install`
|
|
4948
|
+
* and `remember` replace an entry wholesale, so comparing anything else would drop a good one on the floor. The
|
|
4949
|
+
* resolution goes too, or the read's retry would build its snapshot on it again and reopen the generation it just
|
|
4950
|
+
* found gone; and so does a snapshot the reader cache holds on that same resolution, which another read built once the
|
|
4951
|
+
* cache had let `snap` go, or the retry would open that one.
|
|
5262
4952
|
*/
|
|
5263
4953
|
dropStale(key, snap) {
|
|
5264
|
-
|
|
4954
|
+
const resolution = snap.resolution;
|
|
4955
|
+
const held = this.snapshots.peek(key);
|
|
4956
|
+
if (held === snap || resolution !== void 0 && held?.resolution === resolution) {
|
|
4957
|
+
this.snapshots.delete(key);
|
|
4958
|
+
}
|
|
4959
|
+
if (resolution !== void 0 && this.resolutions?.peek(key) === resolution) {
|
|
4960
|
+
this.resolutions.delete(key);
|
|
4961
|
+
}
|
|
5265
4962
|
}
|
|
5266
4963
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
5267
4964
|
async openForTarget(ref, target) {
|
|
5268
4965
|
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
5269
4966
|
return this.openGeneration(ref, target, crypto);
|
|
5270
4967
|
}
|
|
5271
|
-
openGeneration(ref, target, crypto) {
|
|
4968
|
+
openGeneration(ref, target, crypto, named) {
|
|
5272
4969
|
const genKey = {
|
|
5273
4970
|
namespace: ref.namespace,
|
|
5274
4971
|
segment: ref.segment,
|
|
@@ -5278,17 +4975,23 @@ var CrbmStorageChunkSource = class {
|
|
|
5278
4975
|
this.driver,
|
|
5279
4976
|
genKey,
|
|
5280
4977
|
{ ...this.readerOptions, crypto, lineage: target.lineage },
|
|
5281
|
-
this.keepChunkBytesUpTo
|
|
4978
|
+
this.keepChunkBytesUpTo,
|
|
4979
|
+
named
|
|
5282
4980
|
);
|
|
5283
4981
|
}
|
|
5284
4982
|
/**
|
|
5285
|
-
* Open the generation a resolution found, and hold the row's summary against the object
|
|
5286
|
-
* the
|
|
5287
|
-
*
|
|
4983
|
+
* Open the generation a resolution found, and hold the row's summary against the object: the object must be the one
|
|
4984
|
+
* the summary names by its fingerprint, and is refused before its index is opened otherwise, as another object under
|
|
4985
|
+
* the generation's number ({@link NotFoundError}, so the read resolves the row again, as for a swept generation); and
|
|
4986
|
+
* it should hold the same count and metadata, or the summary is not believed again in this process
|
|
4987
|
+
* ({@link distrusted}). Neither check costs a request. A row with no summary it can use names no object, and its
|
|
4988
|
+
* generation's object is opened as it is.
|
|
5288
4989
|
*/
|
|
5289
4990
|
async openLive(ref, live) {
|
|
5290
|
-
const
|
|
4991
|
+
const crypto = await live.crypto();
|
|
5291
4992
|
const summary = await live.summary();
|
|
4993
|
+
const reader = await this.openGeneration(ref, live.target, crypto, summary?.fingerprint);
|
|
4994
|
+
this.rowChecked.add(reader);
|
|
5292
4995
|
if (summary !== void 0 && !summaryAgrees(summary, describe(reader))) {
|
|
5293
4996
|
this.distrusted.set(this.distrustKey(ref, live.target.generation, live.target.lineage), true);
|
|
5294
4997
|
}
|
|
@@ -5305,43 +5008,70 @@ var CrbmStorageChunkSource = class {
|
|
|
5305
5008
|
return live.summary();
|
|
5306
5009
|
}
|
|
5307
5010
|
/**
|
|
5308
|
-
*
|
|
5309
|
-
* `list`
|
|
5310
|
-
*
|
|
5311
|
-
* asks.
|
|
5011
|
+
* What the segment resolves to now: the fields of the registry's authoritative record a read resolves through (one
|
|
5012
|
+
* strong read), or the highest generation a `list` of the bucket finds when there is no registry (a registry-less
|
|
5013
|
+
* source can only read cleartext: there is nowhere a wrapped DEK could live). `null` for no generation.
|
|
5312
5014
|
*/
|
|
5313
|
-
async
|
|
5314
|
-
if (this.registry !== void 0)
|
|
5315
|
-
const record = await this.registry.get(ref);
|
|
5316
|
-
if (record === null) return null;
|
|
5317
|
-
if (record.status === "destroyed") return null;
|
|
5318
|
-
if (record.currentGen === null) return null;
|
|
5319
|
-
const generation = record.currentGen;
|
|
5320
|
-
const keys = record.wrappedDeks;
|
|
5321
|
-
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
5322
|
-
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
5323
|
-
return {
|
|
5324
|
-
row: record,
|
|
5325
|
-
target: { generation, lineage: record.token, wrappedDeks: record.wrappedDeks },
|
|
5326
|
-
crypto,
|
|
5327
|
-
unwrap,
|
|
5328
|
-
// The same checks as a read of the object makes (`requireEncryption`, a keystore for an encrypted row)
|
|
5329
|
-
// come first, then the row's summary, used only if it names this generation in the shape the keys call for.
|
|
5330
|
-
summary: once(async () => usableSummary(ref, record, (await crypto())?.aead)),
|
|
5331
|
-
bytes: SNAPSHOT_BASE_BYTES + (record.summary === void 0 ? 0 : JSON.stringify(record.summary).length)
|
|
5332
|
-
};
|
|
5333
|
-
}
|
|
5015
|
+
async resolve(ref) {
|
|
5016
|
+
if (this.registry !== void 0) return resolvedOf(await this.registry.get(ref));
|
|
5334
5017
|
let maxGen = -1;
|
|
5335
5018
|
for await (const key of this.driver.list(ref)) {
|
|
5336
5019
|
if (key.generation > maxGen) maxGen = key.generation;
|
|
5337
5020
|
}
|
|
5338
|
-
|
|
5021
|
+
return maxGen < 0 ? null : { generation: maxGen };
|
|
5022
|
+
}
|
|
5023
|
+
/**
|
|
5024
|
+
* The segment resolved afresh for a pin, with the row it read, which a lease is written against. Never a kept
|
|
5025
|
+
* resolution, and kept nowhere: the generation current right now is the whole promise of a pin.
|
|
5026
|
+
*/
|
|
5027
|
+
async resolveLive(ref) {
|
|
5028
|
+
if (this.registry === void 0) {
|
|
5029
|
+
const resolved2 = await this.resolve(ref);
|
|
5030
|
+
return resolved2 === null ? null : this.liveOf(ref, resolved2, null);
|
|
5031
|
+
}
|
|
5032
|
+
const row = await this.registry.get(ref);
|
|
5033
|
+
const resolved = resolvedOf(row);
|
|
5034
|
+
return resolved === null || row === null ? null : this.liveOf(ref, resolved, null, row);
|
|
5035
|
+
}
|
|
5036
|
+
/**
|
|
5037
|
+
* What a snapshot, or a pin, makes of a resolution: its decryption context and what the row's summary says of the
|
|
5038
|
+
* generation, each made only when asked for. A snapshot that replaces `prior` with the same wrapped keys keeps the key
|
|
5039
|
+
* `prior` unwrapped, so a count polling an encrypted segment asks the keystore once, not once each `genTtlMs`.
|
|
5040
|
+
* Different keys (a shred, a re-created name) unwrap afresh, and so does a snapshot with no prior: the resolution
|
|
5041
|
+
* holds the wrapped keys, never an unwrapped one.
|
|
5042
|
+
*/
|
|
5043
|
+
liveOf(ref, resolved, prior, row) {
|
|
5044
|
+
if (this.registry === void 0) {
|
|
5045
|
+
return {
|
|
5046
|
+
target: resolved,
|
|
5047
|
+
crypto: () => Promise.resolve(void 0),
|
|
5048
|
+
unwrap: void 0,
|
|
5049
|
+
summary: () => Promise.resolve(void 0),
|
|
5050
|
+
version: () => Promise.resolve(void 0),
|
|
5051
|
+
bytes: weightOf(resolved)
|
|
5052
|
+
};
|
|
5053
|
+
}
|
|
5054
|
+
const { generation, lineage, wrappedDeks: keys, summary } = resolved;
|
|
5055
|
+
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
5056
|
+
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
5057
|
+
const described = once(
|
|
5058
|
+
async () => usableSummary(
|
|
5059
|
+
ref,
|
|
5060
|
+
{ status: "active", currentGen: generation, summary, wrappedDeks: keys },
|
|
5061
|
+
(await crypto())?.aead
|
|
5062
|
+
)
|
|
5063
|
+
);
|
|
5339
5064
|
return {
|
|
5340
|
-
target:
|
|
5341
|
-
|
|
5342
|
-
|
|
5343
|
-
|
|
5344
|
-
|
|
5065
|
+
target: resolved,
|
|
5066
|
+
...row === void 0 ? {} : { row },
|
|
5067
|
+
crypto,
|
|
5068
|
+
unwrap,
|
|
5069
|
+
summary: described,
|
|
5070
|
+
version: once(async () => {
|
|
5071
|
+
const named = (await described())?.fingerprint;
|
|
5072
|
+
return named === void 0 ? void 0 : `${versionOf(generation, lineage)}#${compactFingerprint(named)}`;
|
|
5073
|
+
}),
|
|
5074
|
+
bytes: weightOf(resolved)
|
|
5345
5075
|
};
|
|
5346
5076
|
}
|
|
5347
5077
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
@@ -5405,7 +5135,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5405
5135
|
}
|
|
5406
5136
|
return;
|
|
5407
5137
|
}
|
|
5408
|
-
const version =
|
|
5138
|
+
const version = versionOfReader(reader);
|
|
5139
|
+
let seen = snap.settled;
|
|
5409
5140
|
chunks = reader.readChunks(keys.slice(yielded), {
|
|
5410
5141
|
...options?.concurrency === void 0 ? {} : { concurrency: options.concurrency },
|
|
5411
5142
|
...options?.ramp === void 0 ? {} : { ramp: options.ramp },
|
|
@@ -5417,22 +5148,24 @@ var CrbmStorageChunkSource = class {
|
|
|
5417
5148
|
let movedOn = false;
|
|
5418
5149
|
try {
|
|
5419
5150
|
while (await pull.advance()) {
|
|
5420
|
-
if (this.mayHaveMoved(key, snap, epoch)) {
|
|
5151
|
+
if (this.mayHaveMoved(key, snap, epoch) || snap.settled !== seen) {
|
|
5421
5152
|
const live = this.liveSnapshot(ref);
|
|
5422
|
-
if (live !== snap) {
|
|
5423
|
-
|
|
5153
|
+
if (live !== snap || live.settled !== seen) {
|
|
5154
|
+
const looked = live.settled;
|
|
5155
|
+
let same;
|
|
5424
5156
|
try {
|
|
5425
|
-
|
|
5157
|
+
same = await readsObjectOf(live, reader);
|
|
5426
5158
|
} catch (err) {
|
|
5427
5159
|
if (!isTransientError(err)) throw err;
|
|
5428
5160
|
movedOn = true;
|
|
5429
5161
|
break;
|
|
5430
5162
|
}
|
|
5431
|
-
if (
|
|
5163
|
+
if (!same) {
|
|
5432
5164
|
movedOn = true;
|
|
5433
5165
|
break;
|
|
5434
5166
|
}
|
|
5435
5167
|
snap = live;
|
|
5168
|
+
seen = looked;
|
|
5436
5169
|
}
|
|
5437
5170
|
epoch = this.invalidations;
|
|
5438
5171
|
}
|
|
@@ -5526,24 +5259,37 @@ var CrbmStorageChunkSource = class {
|
|
|
5526
5259
|
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
5527
5260
|
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
5528
5261
|
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
5529
|
-
* With no summary it can use (
|
|
5530
|
-
* generation and answers from the object. A row that names an object that is gone
|
|
5531
|
-
* of the object throws: the number is true of the generation the row
|
|
5262
|
+
* With no summary it can use (a row written without one, one for another generation, one that does not open), it
|
|
5263
|
+
* opens the generation and answers from the object. A row that names an object that is gone, or another object
|
|
5264
|
+
* under the number, still answers, while a read of the object throws: the number is true of the generation the row
|
|
5265
|
+
* names.
|
|
5532
5266
|
*
|
|
5533
5267
|
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
5534
5268
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
5535
5269
|
*/
|
|
5536
5270
|
async summary(ref) {
|
|
5537
5271
|
validateUserRef(ref);
|
|
5272
|
+
return this.described(
|
|
5273
|
+
ref,
|
|
5274
|
+
(generation, row) => summaryOfGeneration(generation, row),
|
|
5275
|
+
(reader) => summaryOfGeneration(reader.generation, describe(reader))
|
|
5276
|
+
);
|
|
5277
|
+
}
|
|
5278
|
+
/**
|
|
5279
|
+
* What one resolution of the segment says of its generation: from the row's summary when this process may use it,
|
|
5280
|
+
* else from the generation's object, opened. `null` for no generation. One heal, as a read of the object has: a
|
|
5281
|
+
* generation swept from under the snapshot, or replaced under its number, is resolved again once.
|
|
5282
|
+
*/
|
|
5283
|
+
async described(ref, fromRow, fromObject) {
|
|
5538
5284
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
5539
5285
|
const snap = this.liveSnapshot(ref);
|
|
5540
5286
|
try {
|
|
5541
5287
|
const live = await snap.target;
|
|
5542
5288
|
if (live === null) return null;
|
|
5543
5289
|
const row = await this.trustedSummary(ref, live);
|
|
5544
|
-
if (row !== void 0) return
|
|
5290
|
+
if (row !== void 0) return fromRow(live.target.generation, row);
|
|
5545
5291
|
const reader = await snap.reader;
|
|
5546
|
-
return reader === null ? null :
|
|
5292
|
+
return reader === null ? null : fromObject(reader);
|
|
5547
5293
|
} catch (err) {
|
|
5548
5294
|
if (attempt === 1) throw err;
|
|
5549
5295
|
const checks = isNotFoundError(err) || isIntegrityError(err) || isValidationError(err);
|
|
@@ -5561,6 +5307,32 @@ var CrbmStorageChunkSource = class {
|
|
|
5561
5307
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5562
5308
|
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
5563
5309
|
}
|
|
5310
|
+
/**
|
|
5311
|
+
* The current generation's number, id count and metadata, with its object's size, all from one resolution. The answer
|
|
5312
|
+
* comes from the row when it can, as {@link summary}'s does: the row's summary of the generation it names, whose
|
|
5313
|
+
* fingerprint carries the object's size. That is one registry read when the snapshot is cold, none when it is warm,
|
|
5314
|
+
* and no read of the object; on an encrypted segment the key is unwrapped to open the summary, as for a count. With no
|
|
5315
|
+
* summary it can use, it opens the generation and answers from its footer and index, with no payload read: a tail read
|
|
5316
|
+
* more when cold. A row that names an object that is gone, or another object, still answers from the row: the figures
|
|
5317
|
+
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
5318
|
+
*/
|
|
5319
|
+
async stat(ref) {
|
|
5320
|
+
validateUserRef(ref);
|
|
5321
|
+
return this.described(
|
|
5322
|
+
ref,
|
|
5323
|
+
(generation, row) => ({
|
|
5324
|
+
...summaryOfGeneration(generation, row),
|
|
5325
|
+
sizeBytes: sizeOfFingerprint(row.fingerprint)
|
|
5326
|
+
}),
|
|
5327
|
+
(reader) => sizedSummary(reader)
|
|
5328
|
+
);
|
|
5329
|
+
}
|
|
5330
|
+
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5331
|
+
async statAt(ref, generation, held) {
|
|
5332
|
+
validateUserRef(ref);
|
|
5333
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5334
|
+
return reader === null ? null : sizedSummary(reader);
|
|
5335
|
+
}
|
|
5564
5336
|
/**
|
|
5565
5337
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
5566
5338
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -5568,8 +5340,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5568
5340
|
* caller-visible error, so instead we drop the stale snapshot,
|
|
5569
5341
|
* re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
|
|
5570
5342
|
* a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
|
|
5571
|
-
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
5572
|
-
* as its lineage rather than trusting the generation number (
|
|
5343
|
+
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
5344
|
+
* `pointerId` as its lineage rather than trusting the generation number (invariants 1 and 2).
|
|
5573
5345
|
*
|
|
5574
5346
|
* **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
|
|
5575
5347
|
* generation's object are two backend round trips with a gap between them, so the object can be swept after
|
|
@@ -5699,15 +5471,30 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
5699
5471
|
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
5700
5472
|
};
|
|
5701
5473
|
}
|
|
5702
|
-
async function
|
|
5474
|
+
async function provenObject(storage, key, fingerprint) {
|
|
5475
|
+
let version;
|
|
5703
5476
|
try {
|
|
5704
|
-
|
|
5477
|
+
const blob = storageBlobReader(storage, key, (tail) => {
|
|
5478
|
+
version ??= tail.version;
|
|
5479
|
+
});
|
|
5480
|
+
return await CrbmReader.sameObject(blob, fingerprint) ? { version } : null;
|
|
5705
5481
|
} catch {
|
|
5706
|
-
return
|
|
5482
|
+
return null;
|
|
5707
5483
|
}
|
|
5708
5484
|
}
|
|
5709
|
-
function
|
|
5710
|
-
return
|
|
5485
|
+
function sameObjectUnder(storage, key, fingerprint) {
|
|
5486
|
+
return CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
5487
|
+
}
|
|
5488
|
+
function objectIsEncrypted(storage, key, noteVersion) {
|
|
5489
|
+
return footerSaysEncrypted(
|
|
5490
|
+
storageBlobReader(
|
|
5491
|
+
storage,
|
|
5492
|
+
key,
|
|
5493
|
+
noteVersion === void 0 ? void 0 : (tail) => {
|
|
5494
|
+
if (tail.version !== void 0) noteVersion(tail.version);
|
|
5495
|
+
}
|
|
5496
|
+
)
|
|
5497
|
+
);
|
|
5711
5498
|
}
|
|
5712
5499
|
function rememberingTail(blob) {
|
|
5713
5500
|
let first;
|
|
@@ -5781,7 +5568,7 @@ function refuseCleartextOntoKey(key, record, cleartext) {
|
|
|
5781
5568
|
`segment "${key.segment}" is encrypted, and generation ${key.generation} was written in cleartext: another writer created the segment with a key while this one was writing. Nothing was published. Re-run the write with the keystore that holds the segment's key.`
|
|
5782
5569
|
);
|
|
5783
5570
|
}
|
|
5784
|
-
var
|
|
5571
|
+
var UNANSWERED_RESENDS2 = 3;
|
|
5785
5572
|
var UNANSWERED_RESEND_BASE_MS = 500;
|
|
5786
5573
|
function leasesPatch(record, options) {
|
|
5787
5574
|
const held = record.leases;
|
|
@@ -5823,7 +5610,8 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5823
5610
|
if (landed)
|
|
5824
5611
|
return {
|
|
5825
5612
|
published: true,
|
|
5826
|
-
kept: { ...nextKept(failedOn, key.generation, options.keep), token: void 0 }
|
|
5613
|
+
kept: { ...nextKept(failedOn, key.generation, options.keep), token: void 0 },
|
|
5614
|
+
token: record?.token
|
|
5827
5615
|
};
|
|
5828
5616
|
if (record !== null && record.status === "active" && record.currentGen === key.generation) {
|
|
5829
5617
|
return REFUSED;
|
|
@@ -5891,7 +5679,7 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5891
5679
|
return REFUSED;
|
|
5892
5680
|
} else if (record.currentGen === key.generation) {
|
|
5893
5681
|
if (reused) continue;
|
|
5894
|
-
return { published: true };
|
|
5682
|
+
return { published: true, token: record.token };
|
|
5895
5683
|
} else {
|
|
5896
5684
|
if (options.wrappedDeks !== void 0 && options.wrappedDeks.length > 0) {
|
|
5897
5685
|
throw new ValidationError(
|
|
@@ -5914,7 +5702,8 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5914
5702
|
}
|
|
5915
5703
|
return {
|
|
5916
5704
|
published: true,
|
|
5917
|
-
kept: { ...next ?? { list: void 0, evict: [] }, token: wrote }
|
|
5705
|
+
kept: { ...next ?? { list: void 0, evict: [] }, token: wrote },
|
|
5706
|
+
token: wrote
|
|
5918
5707
|
};
|
|
5919
5708
|
} catch (err) {
|
|
5920
5709
|
if (isWriteConflictError(err)) {
|
|
@@ -5934,7 +5723,7 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5934
5723
|
}
|
|
5935
5724
|
if (sameRow(now, record)) {
|
|
5936
5725
|
const sleep = options.clock?.sleep;
|
|
5937
|
-
if (sleep === void 0 || resends >=
|
|
5726
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS2) throw outcomeUnknown(key, err);
|
|
5938
5727
|
const bound = UNANSWERED_RESEND_BASE_MS * 2 ** resends;
|
|
5939
5728
|
resends += 1;
|
|
5940
5729
|
await sleep.call(options.clock, Math.floor((options.rng?.next() ?? 1) * bound));
|
|
@@ -5949,11 +5738,10 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5949
5738
|
const final = await registry.get(key);
|
|
5950
5739
|
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation) {
|
|
5951
5740
|
if (final.currentGen !== key.generation) return REFUSED;
|
|
5952
|
-
if (!sawUnanswered) return { published: true };
|
|
5741
|
+
if (!sawUnanswered) return { published: true, token: final.token };
|
|
5953
5742
|
try {
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
};
|
|
5743
|
+
const published = await landedHere(final, lastRecord ?? null, key, options.holdsOwnObject);
|
|
5744
|
+
return published ? { published, token: final.token } : REFUSED;
|
|
5957
5745
|
} catch (proofErr) {
|
|
5958
5746
|
throw outcomeUnknown(key, proofErr);
|
|
5959
5747
|
}
|
|
@@ -6053,22 +5841,24 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6053
5841
|
const summary = summaryOf(
|
|
6054
5842
|
key,
|
|
6055
5843
|
key.generation,
|
|
6056
|
-
{ cardinality, metadata: options.metadata },
|
|
5844
|
+
{ cardinality, metadata: options.metadata, fingerprint },
|
|
6057
5845
|
crypto?.aead
|
|
6058
5846
|
);
|
|
6059
5847
|
if (options.registry !== void 0 && options.publish !== false) {
|
|
6060
|
-
const
|
|
5848
|
+
const published = await publishGenerationKept(options.registry, key, {
|
|
6061
5849
|
wrappedDeks: newWrapped,
|
|
6062
5850
|
summary,
|
|
6063
5851
|
cleartext: crypto === void 0,
|
|
6064
5852
|
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
6065
5853
|
clock: options.clock
|
|
6066
5854
|
});
|
|
5855
|
+
const becameCurrent = published.published;
|
|
6067
5856
|
if (becameCurrent) {
|
|
6068
5857
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
6069
5858
|
kind: "segment.publish",
|
|
6070
5859
|
namespace: key.namespace,
|
|
6071
5860
|
segment: key.segment,
|
|
5861
|
+
...incarnationField(published.token),
|
|
6072
5862
|
generation: key.generation
|
|
6073
5863
|
});
|
|
6074
5864
|
}
|
|
@@ -6181,15 +5971,72 @@ async function bucketIds(ids, codec, clock) {
|
|
|
6181
5971
|
await flushPending();
|
|
6182
5972
|
return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
|
|
6183
5973
|
}
|
|
6184
|
-
|
|
6185
|
-
|
|
5974
|
+
var openedVersions = /* @__PURE__ */ new WeakMap();
|
|
5975
|
+
function openGenerationReader(storage, key, crypto, options = {}, named) {
|
|
5976
|
+
return openNoting(storage, key, crypto, options, named);
|
|
5977
|
+
}
|
|
5978
|
+
function openGenerationReaderNoting(storage, key, crypto, noteVersion) {
|
|
5979
|
+
return openNoting(storage, key, crypto, {}, void 0, noteVersion);
|
|
6186
5980
|
}
|
|
6187
|
-
async function
|
|
6188
|
-
|
|
6189
|
-
|
|
6190
|
-
|
|
6191
|
-
|
|
5981
|
+
async function openNoting(storage, key, crypto, options, named, noteVersion) {
|
|
5982
|
+
let version;
|
|
5983
|
+
const reader = await openChecked(
|
|
5984
|
+
storage,
|
|
5985
|
+
key,
|
|
5986
|
+
{ ...options, crypto },
|
|
5987
|
+
void 0,
|
|
5988
|
+
named,
|
|
5989
|
+
(tail) => {
|
|
5990
|
+
if (version !== void 0 || tail.version === void 0) return;
|
|
5991
|
+
version = tail.version;
|
|
5992
|
+
noteVersion?.(version);
|
|
5993
|
+
}
|
|
6192
5994
|
);
|
|
5995
|
+
if (version !== void 0) openedVersions.set(reader, version);
|
|
5996
|
+
return reader;
|
|
5997
|
+
}
|
|
5998
|
+
function objectVersionOf(reader) {
|
|
5999
|
+
return openedVersions.get(reader);
|
|
6000
|
+
}
|
|
6001
|
+
async function openNamedGeneration(storage, key, crypto, fingerprint) {
|
|
6002
|
+
return openHeldTo(storage, key, { crypto }, void 0, fingerprint);
|
|
6003
|
+
}
|
|
6004
|
+
async function openChecked(storage, key, options, keepChunkBytesUpTo, named, noteTail) {
|
|
6005
|
+
const reader = await openHeldTo(storage, key, options, keepChunkBytesUpTo, named, noteTail);
|
|
6006
|
+
if (reader === void 0) {
|
|
6007
|
+
throw new NotFoundError(
|
|
6008
|
+
`segment "${key.segment}" generation ${key.generation} is another object than its registry row names`
|
|
6009
|
+
);
|
|
6010
|
+
}
|
|
6011
|
+
return reader;
|
|
6012
|
+
}
|
|
6013
|
+
async function openHeldTo(storage, key, options, keepChunkBytesUpTo, named, noteTail) {
|
|
6014
|
+
const raw = storageBlobReader(storage, key, noteTail);
|
|
6015
|
+
let reader;
|
|
6016
|
+
if (named === void 0) {
|
|
6017
|
+
reader = await openCrbmReaderKeeping(raw, options, keepChunkBytesUpTo);
|
|
6018
|
+
} else {
|
|
6019
|
+
let tail = await raw.getTail(
|
|
6020
|
+
Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES)
|
|
6021
|
+
);
|
|
6022
|
+
const blob = {
|
|
6023
|
+
getRange: (offset, length) => raw.getRange(offset, length),
|
|
6024
|
+
getTail: async (maxBytes) => {
|
|
6025
|
+
const held = tail;
|
|
6026
|
+
if (held !== void 0 && (maxBytes <= held.bytes.length || held.bytes.length === held.size)) {
|
|
6027
|
+
const take = Math.min(maxBytes, held.bytes.length);
|
|
6028
|
+
return { bytes: held.bytes.subarray(held.bytes.length - take), size: held.size };
|
|
6029
|
+
}
|
|
6030
|
+
return raw.getTail(maxBytes);
|
|
6031
|
+
}
|
|
6032
|
+
};
|
|
6033
|
+
try {
|
|
6034
|
+
if (!await CrbmReader.sameObject(blob, named)) return void 0;
|
|
6035
|
+
reader = await openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo);
|
|
6036
|
+
} finally {
|
|
6037
|
+
tail = void 0;
|
|
6038
|
+
}
|
|
6039
|
+
}
|
|
6193
6040
|
if (reader.generation !== key.generation) {
|
|
6194
6041
|
throw new IntegrityError(
|
|
6195
6042
|
`segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
|
|
@@ -6231,10 +6078,6 @@ var PinnedStorageChunkSource = class {
|
|
|
6231
6078
|
}
|
|
6232
6079
|
inner;
|
|
6233
6080
|
pins;
|
|
6234
|
-
/** The unpinned segments refresh as the store's own source does; a pinned one never does. */
|
|
6235
|
-
get pointerRefreshMs() {
|
|
6236
|
-
return this.inner.pointerRefreshMs;
|
|
6237
|
-
}
|
|
6238
6081
|
/**
|
|
6239
6082
|
* The pin for this segment, or `undefined` if it is not pinned here.
|
|
6240
6083
|
*
|
|
@@ -6259,76 +6102,250 @@ var PinnedStorageChunkSource = class {
|
|
|
6259
6102
|
if (pin === void 0) return this.inner.getChunks(ref, keys, options);
|
|
6260
6103
|
return this.pinnedChunks(ref, pin, keys, options);
|
|
6261
6104
|
}
|
|
6262
|
-
async *pinnedChunks(ref, pin, keys, options) {
|
|
6263
|
-
if (pin.generation === null) {
|
|
6264
|
-
for (const key of keys) yield { key, bytes: null, version: null };
|
|
6265
|
-
return;
|
|
6105
|
+
async *pinnedChunks(ref, pin, keys, options) {
|
|
6106
|
+
if (pin.generation === null) {
|
|
6107
|
+
for (const key of keys) yield { key, bytes: null, version: null };
|
|
6108
|
+
return;
|
|
6109
|
+
}
|
|
6110
|
+
const version = await this.currentVersion(ref);
|
|
6111
|
+
const pull = new ItemPull(
|
|
6112
|
+
this.inner.getChunksAt(ref, pin.generation, keys, heldBy(pin), options)
|
|
6113
|
+
);
|
|
6114
|
+
try {
|
|
6115
|
+
while (await pull.advance()) yield pull.take((chunk) => ({ ...chunk, version }));
|
|
6116
|
+
} finally {
|
|
6117
|
+
await pull.close();
|
|
6118
|
+
}
|
|
6119
|
+
}
|
|
6120
|
+
listChunkKeys(ref) {
|
|
6121
|
+
const pin = this.pinFor(ref);
|
|
6122
|
+
if (pin === void 0) return this.inner.listChunkKeys(ref);
|
|
6123
|
+
return pin.generation === null ? Promise.resolve([]) : this.inner.listChunkKeysAt(ref, pin.generation, heldBy(pin));
|
|
6124
|
+
}
|
|
6125
|
+
sizeOf(ref) {
|
|
6126
|
+
const pin = this.pinFor(ref);
|
|
6127
|
+
if (pin === void 0) return this.inner.sizeOf(ref);
|
|
6128
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.sizeOfAt(ref, pin.generation, heldBy(pin));
|
|
6129
|
+
}
|
|
6130
|
+
cardinalities(ref) {
|
|
6131
|
+
const pin = this.pinFor(ref);
|
|
6132
|
+
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
6133
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
6134
|
+
}
|
|
6135
|
+
summary(ref) {
|
|
6136
|
+
const pin = this.pinFor(ref);
|
|
6137
|
+
if (pin === void 0) return this.inner.summary(ref);
|
|
6138
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
6139
|
+
}
|
|
6140
|
+
stat(ref) {
|
|
6141
|
+
const pin = this.pinFor(ref);
|
|
6142
|
+
if (pin === void 0) return this.inner.stat(ref);
|
|
6143
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.statAt(ref, pin.generation, heldBy(pin));
|
|
6144
|
+
}
|
|
6145
|
+
currentGeneration(ref) {
|
|
6146
|
+
const pin = this.pinFor(ref);
|
|
6147
|
+
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
6148
|
+
}
|
|
6149
|
+
/**
|
|
6150
|
+
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
6151
|
+
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
6152
|
+
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
6153
|
+
* and a lapsed `cache.genTtlMs`, a reader-cache eviction on a store with no timed refresh, the resolution
|
|
6154
|
+
* cache letting the segment's resolution go on a store with one, a sweep that heals the read forward, or an
|
|
6155
|
+
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
6156
|
+
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
6157
|
+
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
6158
|
+
* read fetches exactly its own generation, so the entries it fills are always that generation's.
|
|
6159
|
+
*
|
|
6160
|
+
* The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
|
|
6161
|
+
* live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
|
|
6162
|
+
* though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
|
|
6163
|
+
* the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
|
|
6164
|
+
* key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
|
|
6165
|
+
*/
|
|
6166
|
+
currentVersion(ref) {
|
|
6167
|
+
const pin = this.pinFor(ref);
|
|
6168
|
+
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
6169
|
+
if (pin.version === null) return Promise.resolve(null);
|
|
6170
|
+
return Promise.resolve(
|
|
6171
|
+
`pin ${pin.version}${pin.fingerprint == null ? "" : `#${pin.fingerprint}`}`
|
|
6172
|
+
);
|
|
6173
|
+
}
|
|
6174
|
+
exists(ref) {
|
|
6175
|
+
return this.inner.exists(ref);
|
|
6176
|
+
}
|
|
6177
|
+
invalidate(ref) {
|
|
6178
|
+
this.inner.invalidate(ref);
|
|
6179
|
+
}
|
|
6180
|
+
};
|
|
6181
|
+
|
|
6182
|
+
// src/core/generation-gc.ts
|
|
6183
|
+
async function nextGeneration(ref, deps) {
|
|
6184
|
+
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
6185
|
+
}
|
|
6186
|
+
async function aboveEverything(ref, deps, record) {
|
|
6187
|
+
let highest = record?.currentGen ?? -1;
|
|
6188
|
+
for await (const key of deps.storage.list(ref)) {
|
|
6189
|
+
if (key.generation > highest) highest = key.generation;
|
|
6190
|
+
}
|
|
6191
|
+
return numberAbove(highest);
|
|
6192
|
+
}
|
|
6193
|
+
function numberAbove(highest) {
|
|
6194
|
+
if (highest >= Number.MAX_SAFE_INTEGER) {
|
|
6195
|
+
throw new IntegrityError("the segment has no generation number left above its highest");
|
|
6196
|
+
}
|
|
6197
|
+
return highest + 1;
|
|
6198
|
+
}
|
|
6199
|
+
async function nextLoadGeneration(ref, deps, row) {
|
|
6200
|
+
const generation = numberAbove(row?.currentGen ?? -1);
|
|
6201
|
+
try {
|
|
6202
|
+
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
6203
|
+
} catch (err) {
|
|
6204
|
+
if (isNotFoundError(err)) return { generation, checked: true };
|
|
6205
|
+
}
|
|
6206
|
+
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
6207
|
+
}
|
|
6208
|
+
var LIST_COLLECTION_CADENCE = 16;
|
|
6209
|
+
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
6210
|
+
return (await listingPass(ref, deps, { keep: options.keep })).deleted;
|
|
6211
|
+
}
|
|
6212
|
+
async function listingPass(ref, deps, options) {
|
|
6213
|
+
const keep = options.keep ?? 1;
|
|
6214
|
+
const quiet = options.quiet === true;
|
|
6215
|
+
if (!Number.isInteger(keep) || keep < 0) {
|
|
6216
|
+
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
6217
|
+
}
|
|
6218
|
+
const record = await deps.registry.get(ref);
|
|
6219
|
+
if (record === null) return { deleted: [], kept: [], complete: true };
|
|
6220
|
+
const current = record.currentGen;
|
|
6221
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6222
|
+
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
6223
|
+
const gens = [...seen];
|
|
6224
|
+
const after = await deps.registry.get(ref);
|
|
6225
|
+
if (after === null) {
|
|
6226
|
+
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
6227
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
6228
|
+
}
|
|
6229
|
+
if (record.status === "destroyed" && after.token !== record.token) {
|
|
6230
|
+
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
6231
|
+
throw new WriteConflictError(
|
|
6232
|
+
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
6233
|
+
);
|
|
6234
|
+
}
|
|
6235
|
+
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
6236
|
+
const named = record.status === "destroyed" ? void 0 : usableKeptGens(after);
|
|
6237
|
+
const protect = options.protect === void 0 || named === void 0 ? void 0 : /* @__PURE__ */ new Set([...options.protect, ...named]);
|
|
6238
|
+
const stillCollectable = async (generation) => {
|
|
6239
|
+
const still = await deps.registry.get(ref);
|
|
6240
|
+
if (still === null) {
|
|
6241
|
+
if (quiet) return "stop";
|
|
6242
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
6243
|
+
}
|
|
6244
|
+
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
6245
|
+
const now = protect === void 0 ? void 0 : usableKeptGens(still);
|
|
6246
|
+
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
6247
|
+
if (!ok || !unnamed) {
|
|
6248
|
+
if (quiet) return "stop";
|
|
6249
|
+
throw new WriteConflictError(
|
|
6250
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
6251
|
+
);
|
|
6252
|
+
}
|
|
6253
|
+
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
6254
|
+
return "delete";
|
|
6255
|
+
};
|
|
6256
|
+
const guard = record.status === "destroyed" ? void 0 : options.leases;
|
|
6257
|
+
const held = guard === void 0 ? void 0 : heldGenerations(after, guard.now());
|
|
6258
|
+
const below = record.status === "destroyed" || cutoff === null ? [] : gens.filter((g) => g < cutoff).sort((a, b) => b - a);
|
|
6259
|
+
const candidates = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
6260
|
+
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
6261
|
+
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
6262
|
+
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
6263
|
+
[]
|
|
6264
|
+
) : protect !== void 0 ? (
|
|
6265
|
+
// The row's list is the window: everything below the pointer it does not name is garbage.
|
|
6266
|
+
below.filter((g) => !protect.has(g))
|
|
6267
|
+
) : (
|
|
6268
|
+
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
6269
|
+
below.slice(keep)
|
|
6270
|
+
);
|
|
6271
|
+
const leasedSpared = new Set(candidates.filter((g) => held?.has(g) === true));
|
|
6272
|
+
const toDelete = candidates.filter((g) => !leasedSpared.has(g));
|
|
6273
|
+
const done = [];
|
|
6274
|
+
for (const generation of toDelete) {
|
|
6275
|
+
const verdict = await stillCollectable(generation);
|
|
6276
|
+
if (verdict === "stop") return { deleted: done, kept: [], complete: false };
|
|
6277
|
+
if (verdict === "skip") {
|
|
6278
|
+
leasedSpared.add(generation);
|
|
6279
|
+
continue;
|
|
6280
|
+
}
|
|
6281
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
6282
|
+
done.push(generation);
|
|
6283
|
+
}
|
|
6284
|
+
const deleted = new Set(done);
|
|
6285
|
+
return {
|
|
6286
|
+
deleted: done,
|
|
6287
|
+
kept: below.filter((g) => !deleted.has(g) && !leasedSpared.has(g)).reverse(),
|
|
6288
|
+
complete: true
|
|
6289
|
+
};
|
|
6290
|
+
}
|
|
6291
|
+
async function collectAfterLoad(ref, deps, options) {
|
|
6292
|
+
const { generation, keep, byName, currentGone, kept, proveCurrent, leases } = options;
|
|
6293
|
+
if (keep >= generation) return [];
|
|
6294
|
+
const list = kept?.list;
|
|
6295
|
+
if (list === void 0 || currentGone === true)
|
|
6296
|
+
return reconcile(ref, deps, { keep, kept, leases });
|
|
6297
|
+
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
6298
|
+
if (byName && !periodic) {
|
|
6299
|
+
if (keep >= 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
6300
|
+
return reconcile(ref, deps, { keep, kept, leases });
|
|
6266
6301
|
}
|
|
6267
|
-
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6302
|
+
return deleteEvicted(ref, deps, { generation, evict: kept.evict, leases });
|
|
6303
|
+
}
|
|
6304
|
+
return (await listingPass(ref, deps, { keep, protect: list, quiet: true, leases })).deleted;
|
|
6305
|
+
}
|
|
6306
|
+
async function reconcile(ref, deps, options) {
|
|
6307
|
+
const { keep, kept, leases } = options;
|
|
6308
|
+
const {
|
|
6309
|
+
deleted,
|
|
6310
|
+
kept: present,
|
|
6311
|
+
complete
|
|
6312
|
+
} = await listingPass(ref, deps, { keep, quiet: true, leases });
|
|
6313
|
+
const token = kept?.token;
|
|
6314
|
+
if (complete && token !== void 0 && keep <= MAX_KEPT_GENERATIONS && !sameList(kept?.list, present)) {
|
|
6271
6315
|
try {
|
|
6272
|
-
|
|
6273
|
-
}
|
|
6274
|
-
|
|
6316
|
+
await deps.registry.compareAndSwap(ref, token, { keptGens: present });
|
|
6317
|
+
} catch (err) {
|
|
6318
|
+
if (!isWriteConflictError(err)) throw err;
|
|
6275
6319
|
}
|
|
6276
6320
|
}
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
return
|
|
6286
|
-
}
|
|
6287
|
-
|
|
6288
|
-
const pin = this.pinFor(ref);
|
|
6289
|
-
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
6290
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
6291
|
-
}
|
|
6292
|
-
summary(ref) {
|
|
6293
|
-
const pin = this.pinFor(ref);
|
|
6294
|
-
if (pin === void 0) return this.inner.summary(ref);
|
|
6295
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
6296
|
-
}
|
|
6297
|
-
currentGeneration(ref) {
|
|
6298
|
-
const pin = this.pinFor(ref);
|
|
6299
|
-
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
6300
|
-
}
|
|
6301
|
-
/**
|
|
6302
|
-
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
6303
|
-
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
6304
|
-
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
6305
|
-
* and a lapsed `cache.genTtlMs`, a reader-cache eviction, a sweep that heals the read forward, or an
|
|
6306
|
-
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
6307
|
-
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
6308
|
-
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
6309
|
-
* read fetches exactly its own generation, so the entries it fills are always that generation's.
|
|
6310
|
-
*
|
|
6311
|
-
* The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
|
|
6312
|
-
* live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
|
|
6313
|
-
* though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
|
|
6314
|
-
* the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
|
|
6315
|
-
* key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
|
|
6316
|
-
*/
|
|
6317
|
-
currentVersion(ref) {
|
|
6318
|
-
const pin = this.pinFor(ref);
|
|
6319
|
-
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
6320
|
-
if (pin.version === null) return Promise.resolve(null);
|
|
6321
|
-
return Promise.resolve(
|
|
6322
|
-
`pin ${pin.version}${pin.fingerprint == null ? "" : `#${pin.fingerprint}`}`
|
|
6323
|
-
);
|
|
6324
|
-
}
|
|
6325
|
-
exists(ref) {
|
|
6326
|
-
return this.inner.exists(ref);
|
|
6321
|
+
return deleted;
|
|
6322
|
+
}
|
|
6323
|
+
function sameList(a, b) {
|
|
6324
|
+
return a !== void 0 && a.length === b.length && a.every((g, i) => g === b[i]);
|
|
6325
|
+
}
|
|
6326
|
+
async function objectIsThere(storage, key) {
|
|
6327
|
+
try {
|
|
6328
|
+
await storage.getTail(key, 0);
|
|
6329
|
+
return true;
|
|
6330
|
+
} catch {
|
|
6331
|
+
return false;
|
|
6327
6332
|
}
|
|
6328
|
-
|
|
6329
|
-
|
|
6333
|
+
}
|
|
6334
|
+
async function deleteEvicted(ref, deps, options) {
|
|
6335
|
+
const deleted = [];
|
|
6336
|
+
for (const generation of options.evict) {
|
|
6337
|
+
const still = await deps.registry.get(ref);
|
|
6338
|
+
if (still === null || still.currentGen === null || still.currentGen < options.generation) break;
|
|
6339
|
+
const named = usableKeptGens(still);
|
|
6340
|
+
if (named === void 0 || generation >= still.currentGen || named.includes(generation)) break;
|
|
6341
|
+
if (options.leases !== void 0 && heldGenerations(still, options.leases.now()).has(generation)) {
|
|
6342
|
+
continue;
|
|
6343
|
+
}
|
|
6344
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
6345
|
+
deleted.push(generation);
|
|
6330
6346
|
}
|
|
6331
|
-
|
|
6347
|
+
return deleted;
|
|
6348
|
+
}
|
|
6332
6349
|
|
|
6333
6350
|
// src/core/retry.ts
|
|
6334
6351
|
var DEFAULT_RETRY_POLICY = {
|
|
@@ -6402,10 +6419,10 @@ async function withRetry(op, policy, deps) {
|
|
|
6402
6419
|
// src/core/load.ts
|
|
6403
6420
|
function checkedGuard(guard, where) {
|
|
6404
6421
|
if (guard === void 0 || guard === null) return void 0;
|
|
6405
|
-
const
|
|
6422
|
+
const shown = (v) => typeof v === "number" ? String(v) : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
6406
6423
|
if (typeof guard !== "object" || Array.isArray(guard)) {
|
|
6407
6424
|
throw new ValidationError(
|
|
6408
|
-
`${where}guard must be an object of bounds, such as { minRetained: 0.5 }; got ${
|
|
6425
|
+
`${where}guard must be an object of bounds, such as { minRetained: 0.5 }; got ${shown(guard)}`
|
|
6409
6426
|
);
|
|
6410
6427
|
}
|
|
6411
6428
|
const bounds = guard;
|
|
@@ -6420,17 +6437,17 @@ function checkedGuard(guard, where) {
|
|
|
6420
6437
|
const { minCardinality, minRetained, maxGrowth } = guard;
|
|
6421
6438
|
if (minCardinality !== void 0 && (!Number.isInteger(minCardinality) || minCardinality < 0)) {
|
|
6422
6439
|
throw new ValidationError(
|
|
6423
|
-
`${where}guard.minCardinality must be a non-negative integer; got ${
|
|
6440
|
+
`${where}guard.minCardinality must be a non-negative integer; got ${shown(minCardinality)}`
|
|
6424
6441
|
);
|
|
6425
6442
|
}
|
|
6426
6443
|
if (minRetained !== void 0 && (!Number.isFinite(minRetained) || minRetained < 0 || minRetained > 1)) {
|
|
6427
6444
|
throw new ValidationError(
|
|
6428
|
-
`${where}guard.minRetained must be a fraction in 0..1; got ${
|
|
6445
|
+
`${where}guard.minRetained must be a fraction in 0..1; got ${shown(minRetained)}`
|
|
6429
6446
|
);
|
|
6430
6447
|
}
|
|
6431
6448
|
if (maxGrowth !== void 0 && (!Number.isFinite(maxGrowth) || maxGrowth !== 0 && maxGrowth < 1)) {
|
|
6432
6449
|
throw new ValidationError(
|
|
6433
|
-
`${where}guard.maxGrowth must be 0 (no bound) or a factor of at least 1; got ${
|
|
6450
|
+
`${where}guard.maxGrowth must be 0 (no bound) or a factor of at least 1; got ${shown(maxGrowth)}`
|
|
6434
6451
|
);
|
|
6435
6452
|
}
|
|
6436
6453
|
return Object.freeze({
|
|
@@ -6606,6 +6623,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6606
6623
|
kind: "segment.load-refused",
|
|
6607
6624
|
segment: ref.segment,
|
|
6608
6625
|
namespace: ref.namespace,
|
|
6626
|
+
...incarnationField(fromToken),
|
|
6609
6627
|
generation: generation2,
|
|
6610
6628
|
reason: "superseded",
|
|
6611
6629
|
cardinality: 0
|
|
@@ -6631,9 +6649,9 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6631
6649
|
return;
|
|
6632
6650
|
}
|
|
6633
6651
|
const keyed = now !== null && now.wrappedDeks !== void 0 && now.wrappedDeks.length > 0;
|
|
6634
|
-
if (
|
|
6635
|
-
|
|
6636
|
-
}
|
|
6652
|
+
if (now !== null && !(keyed && !written.encrypted)) return;
|
|
6653
|
+
const own = await provenObject(deps.storage, key, written.fingerprint);
|
|
6654
|
+
if (own !== null) await deps.storage.delete(key, { ifVersion: own.version });
|
|
6637
6655
|
};
|
|
6638
6656
|
let unanswered = false;
|
|
6639
6657
|
const tidy = () => reclaim().catch(() => void 0);
|
|
@@ -6643,6 +6661,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6643
6661
|
kind: "segment.load-refused",
|
|
6644
6662
|
segment: ref.segment,
|
|
6645
6663
|
namespace: ref.namespace,
|
|
6664
|
+
...incarnationField(fromToken),
|
|
6646
6665
|
generation,
|
|
6647
6666
|
reason,
|
|
6648
6667
|
cardinality: written.cardinality,
|
|
@@ -6674,14 +6693,14 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6674
6693
|
cleartext: !written.encrypted,
|
|
6675
6694
|
...fromToken === void 0 ? {} : { expectToken: fromToken, expectRow: row },
|
|
6676
6695
|
...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
|
|
6677
|
-
//
|
|
6678
|
-
//
|
|
6679
|
-
//
|
|
6680
|
-
//
|
|
6681
|
-
//
|
|
6682
|
-
//
|
|
6683
|
-
//
|
|
6684
|
-
...
|
|
6696
|
+
// A load that found NO ROW fences on that absence, guarded or not, so a row that appeared since refuses it.
|
|
6697
|
+
// `expectFrom` and `expectToken` both compare against a value read from a row, so with no row both are omitted,
|
|
6698
|
+
// and the publish would otherwise be a bare forward-only advance over whatever appeared in between. For a guarded
|
|
6699
|
+
// load, `before` was `null` and its bounds passed vacuously, so an empty generation would land over the ids
|
|
6700
|
+
// another writer had just loaded. For an unguarded one, the pointer would move over another first load's
|
|
6701
|
+
// generation or onto a row `setRetention` made, and, where an erasure had already deleted this load's object
|
|
6702
|
+
// above that row's pointer, onto an object that is gone, so that every read of the segment would fail.
|
|
6703
|
+
...row === null ? { expectAbsent: true } : {},
|
|
6685
6704
|
// A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
|
|
6686
6705
|
// is this load's publish only over the object this load wrote, which one footer read proves.
|
|
6687
6706
|
holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
|
|
@@ -6701,6 +6720,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6701
6720
|
kind: "segment.publish",
|
|
6702
6721
|
segment: ref.segment,
|
|
6703
6722
|
namespace: ref.namespace,
|
|
6723
|
+
...incarnationField(published.token),
|
|
6704
6724
|
generation
|
|
6705
6725
|
});
|
|
6706
6726
|
const currentObjectGone = needsBefore && before === null;
|
|
@@ -8669,7 +8689,7 @@ function pinnedStillCurrent(spec, now) {
|
|
|
8669
8689
|
}
|
|
8670
8690
|
function liveMoved(op) {
|
|
8671
8691
|
if (op.endRow === void 0 || !op.read) return void 0;
|
|
8672
|
-
const was = tokenOfVersion(op.startVersion);
|
|
8692
|
+
const was = op.startVersion == null ? void 0 : tokenOfVersion(rowVersionOf(op.startVersion));
|
|
8673
8693
|
if (was !== void 0 && op.endRow !== null && !sameRow2(was, op.endRow.token)) return true;
|
|
8674
8694
|
return op.startGeneration === void 0 ? void 0 : op.endGeneration !== op.startGeneration;
|
|
8675
8695
|
}
|
|
@@ -8870,7 +8890,11 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8870
8890
|
const summary = reader === void 0 ? void 0 : summaryOf(
|
|
8871
8891
|
ref,
|
|
8872
8892
|
toGeneration,
|
|
8873
|
-
{
|
|
8893
|
+
{
|
|
8894
|
+
cardinality: reader.count(),
|
|
8895
|
+
metadata: reader.metadata,
|
|
8896
|
+
fingerprint: reader.fingerprint
|
|
8897
|
+
},
|
|
8874
8898
|
aead
|
|
8875
8899
|
);
|
|
8876
8900
|
const churn = leaseChurn(deps);
|
|
@@ -8925,7 +8949,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8925
8949
|
break;
|
|
8926
8950
|
} catch (err) {
|
|
8927
8951
|
if (!isWriteConflictError(err)) break;
|
|
8928
|
-
const moved = ["currentGen", "summary", "keptGens"];
|
|
8952
|
+
const moved = ["currentGen", "summary", "keptGens", "pointerId"];
|
|
8929
8953
|
let now;
|
|
8930
8954
|
try {
|
|
8931
8955
|
const seen = await deps.registry.get(ref);
|
|
@@ -8935,7 +8959,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8935
8959
|
} catch {
|
|
8936
8960
|
break;
|
|
8937
8961
|
}
|
|
8938
|
-
if (now === null || now.currentGen !== toGeneration || !onlyLeasesDiffer(record, now, moved))
|
|
8962
|
+
if (now === null || now.currentGen !== toGeneration || now.pointerId !== token || !onlyLeasesDiffer(record, now, moved))
|
|
8939
8963
|
break;
|
|
8940
8964
|
undoAgainst = now.token;
|
|
8941
8965
|
}
|
|
@@ -8948,6 +8972,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8948
8972
|
kind: "segment.rollback",
|
|
8949
8973
|
segment: ref.segment,
|
|
8950
8974
|
namespace: ref.namespace,
|
|
8975
|
+
...incarnationField(record.token),
|
|
8951
8976
|
fromGeneration: record.currentGen,
|
|
8952
8977
|
generation: toGeneration
|
|
8953
8978
|
});
|
|
@@ -8966,12 +8991,27 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8966
8991
|
const first = {};
|
|
8967
8992
|
const result = await eraseOnce(ref, id, deps, options, first);
|
|
8968
8993
|
if (result.reason === "destroyed" && first.tombstoned === false) {
|
|
8969
|
-
const
|
|
8994
|
+
const second = {};
|
|
8995
|
+
const again = await eraseOnce(ref, id, deps, options, second);
|
|
8970
8996
|
if (again.reason === "destroyed") return result;
|
|
8971
8997
|
if (!again.erased) return again;
|
|
8972
8998
|
const collected = [.../* @__PURE__ */ new Set([...result.collected, ...again.collected])].sort((a, b) => a - b);
|
|
8973
8999
|
const newest = Math.max(result.fromGeneration ?? -1, again.fromGeneration ?? -1);
|
|
8974
|
-
|
|
9000
|
+
const merged = { ...again, collected, ...newest >= 0 ? { fromGeneration: newest } : {} };
|
|
9001
|
+
return recordCollect(ref, merged, second, options);
|
|
9002
|
+
}
|
|
9003
|
+
return recordCollect(ref, result, first, options);
|
|
9004
|
+
}
|
|
9005
|
+
function recordCollect(ref, result, pass, options) {
|
|
9006
|
+
if (pass.collectedOn !== void 0) {
|
|
9007
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9008
|
+
kind: "segment.collect",
|
|
9009
|
+
namespace: ref.namespace,
|
|
9010
|
+
segment: ref.segment,
|
|
9011
|
+
...incarnationField(pass.collectedOn),
|
|
9012
|
+
...pass.found === true && result.fromGeneration !== void 0 ? { fromGeneration: result.fromGeneration } : {},
|
|
9013
|
+
collected: [...result.collected]
|
|
9014
|
+
});
|
|
8975
9015
|
}
|
|
8976
9016
|
return result;
|
|
8977
9017
|
}
|
|
@@ -8982,6 +9022,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
8982
9022
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
8983
9023
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
8984
9024
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9025
|
+
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
8985
9026
|
const read = (op) => retryRead(op, deps.readRetry);
|
|
8986
9027
|
const record = await deps.registry.get(ref);
|
|
8987
9028
|
if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
|
|
@@ -9017,30 +9058,62 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9017
9058
|
const rowVerdict = (row) => {
|
|
9018
9059
|
if (row === null) return "absent";
|
|
9019
9060
|
if (row.status === "destroyed") return "destroyed";
|
|
9061
|
+
const same = row.token === premiseToken || onlyLeasesDiffer(premise, row);
|
|
9062
|
+
if (pointerless) return row.currentGen === null && same ? null : "superseded";
|
|
9020
9063
|
if (row.currentGen === null) return "no-generation";
|
|
9021
|
-
return row.currentGen === from &&
|
|
9064
|
+
return row.currentGen === from && same ? null : "superseded";
|
|
9022
9065
|
};
|
|
9023
|
-
const
|
|
9066
|
+
const sealed = /* @__PURE__ */ new Set();
|
|
9067
|
+
const versions = /* @__PURE__ */ new Map();
|
|
9024
9068
|
const holds = async (generation2) => {
|
|
9025
9069
|
const key2 = { ...base, generation: generation2 };
|
|
9026
|
-
const
|
|
9070
|
+
const crypto = cryptoAt(generation2);
|
|
9027
9071
|
try {
|
|
9028
|
-
let
|
|
9072
|
+
let reader;
|
|
9073
|
+
const index = { refused: false };
|
|
9074
|
+
let opened;
|
|
9029
9075
|
try {
|
|
9030
|
-
|
|
9076
|
+
reader = await read(
|
|
9077
|
+
() => openGenerationReaderNoting(
|
|
9078
|
+
deps.storage,
|
|
9079
|
+
key2,
|
|
9080
|
+
crypto === void 0 ? void 0 : noticingIndex(crypto, index),
|
|
9081
|
+
(version) => opened = version
|
|
9082
|
+
)
|
|
9083
|
+
);
|
|
9031
9084
|
} catch (err) {
|
|
9032
|
-
if (
|
|
9033
|
-
|
|
9085
|
+
if (crypto === void 0) {
|
|
9086
|
+
let footer;
|
|
9087
|
+
const encrypted = !isTransientError(err) && await read(
|
|
9088
|
+
() => objectIsEncrypted(deps.storage, key2, (version) => footer = version)
|
|
9089
|
+
);
|
|
9090
|
+
if (encrypted) {
|
|
9034
9091
|
if (tombstoned) return false;
|
|
9035
|
-
|
|
9092
|
+
sealed.add(generation2);
|
|
9093
|
+
versions.set(generation2, footer);
|
|
9036
9094
|
return true;
|
|
9037
9095
|
}
|
|
9038
9096
|
throw err;
|
|
9039
9097
|
}
|
|
9098
|
+
if (isIntegrityError(err) && index.refused) {
|
|
9099
|
+
sealed.add(generation2);
|
|
9100
|
+
versions.set(generation2, opened);
|
|
9101
|
+
return true;
|
|
9102
|
+
}
|
|
9040
9103
|
if (!isIntegrityError(err) || await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
9041
9104
|
throw err;
|
|
9042
9105
|
}
|
|
9043
|
-
|
|
9106
|
+
reader = await read(() => openGenerationReader(deps.storage, key2, void 0));
|
|
9107
|
+
}
|
|
9108
|
+
versions.set(generation2, objectVersionOf(reader));
|
|
9109
|
+
let bytes;
|
|
9110
|
+
try {
|
|
9111
|
+
bytes = await read(() => reader.getChunk(chunkKey));
|
|
9112
|
+
} catch (err) {
|
|
9113
|
+
if (!isTransientError(err) && !await read(() => sameObjectUnder(deps.storage, key2, reader.fingerprint))) {
|
|
9114
|
+
return null;
|
|
9115
|
+
}
|
|
9116
|
+
throw err;
|
|
9044
9117
|
}
|
|
9045
9118
|
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
9046
9119
|
} catch (err) {
|
|
@@ -9048,6 +9121,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9048
9121
|
throw err;
|
|
9049
9122
|
}
|
|
9050
9123
|
};
|
|
9124
|
+
const found = (generation2, held) => held === true && !sealed.has(generation2);
|
|
9051
9125
|
const holdsEach = async (generations, stopAtHolder) => {
|
|
9052
9126
|
let firstHolder = Infinity;
|
|
9053
9127
|
const settled2 = await mapWithConcurrency(
|
|
@@ -9059,7 +9133,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9059
9133
|
}
|
|
9060
9134
|
try {
|
|
9061
9135
|
const held = await holds(generation2);
|
|
9062
|
-
if (held
|
|
9136
|
+
if (found(generation2, held) && index < firstHolder) firstHolder = index;
|
|
9063
9137
|
return { generation: generation2, held, fault: void 0 };
|
|
9064
9138
|
} catch (fault) {
|
|
9065
9139
|
return { generation: generation2, held: null, fault: { error: fault } };
|
|
@@ -9070,7 +9144,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9070
9144
|
for (const { generation: generation2, held, fault } of settled2) {
|
|
9071
9145
|
if (fault !== void 0) throw fault.error;
|
|
9072
9146
|
outcomes.push({ generation: generation2, held });
|
|
9073
|
-
if (stopAtHolder && held
|
|
9147
|
+
if (stopAtHolder && found(generation2, held)) break;
|
|
9074
9148
|
}
|
|
9075
9149
|
return outcomes;
|
|
9076
9150
|
};
|
|
@@ -9081,41 +9155,119 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9081
9155
|
return (await holdsEach(toRead, true)).find((outcome) => outcome.held === true)?.generation;
|
|
9082
9156
|
};
|
|
9083
9157
|
const cannotRemove = (generation2) => new WriteConflictError(
|
|
9084
|
-
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which still holds the id and is still in the bucket; re-run`
|
|
9158
|
+
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which ${sealed.has(generation2) ? "cannot be searched, so may hold the id," : "still holds the id"} and is still in the bucket; re-run`
|
|
9085
9159
|
);
|
|
9086
9160
|
const fenceInFlight = async () => {
|
|
9087
|
-
const
|
|
9161
|
+
const rng = deps.rng ?? deps.readRetry?.rng;
|
|
9162
|
+
const churn = leaseChurn({ clock: deps.clock, rng });
|
|
9163
|
+
let resends = 0;
|
|
9088
9164
|
let row = await deps.registry.get(ref);
|
|
9089
9165
|
for (; ; ) {
|
|
9090
9166
|
const verdict = rowVerdict(row);
|
|
9091
9167
|
if (verdict !== null) return verdict;
|
|
9092
9168
|
const current = row;
|
|
9093
|
-
|
|
9169
|
+
let failure;
|
|
9094
9170
|
try {
|
|
9095
|
-
const { token } = await deps.registry.compareAndSwap(
|
|
9096
|
-
|
|
9171
|
+
const { token } = await deps.registry.compareAndSwap(
|
|
9172
|
+
ref,
|
|
9173
|
+
current.token,
|
|
9174
|
+
renewPointer(current)
|
|
9175
|
+
);
|
|
9176
|
+
premise = { ...current, token, pointerId: token };
|
|
9097
9177
|
premiseToken = token;
|
|
9098
9178
|
return null;
|
|
9099
9179
|
} catch (err) {
|
|
9100
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9180
|
+
if (!isWriteConflictError(err) && !isTransientError(err)) throw err;
|
|
9181
|
+
failure = err;
|
|
9182
|
+
}
|
|
9183
|
+
const unanswered = isTransientError(failure);
|
|
9184
|
+
let now;
|
|
9185
|
+
try {
|
|
9186
|
+
now = await deps.registry.get(ref);
|
|
9187
|
+
} catch (readErr) {
|
|
9188
|
+
if (!unanswered || isIntegrityError(readErr)) throw readErr;
|
|
9189
|
+
throw failure;
|
|
9190
|
+
}
|
|
9191
|
+
if (now !== null && renewedSince(current, now)) {
|
|
9192
|
+
premise = now;
|
|
9193
|
+
premiseToken = now.token;
|
|
9194
|
+
return null;
|
|
9195
|
+
}
|
|
9196
|
+
if (unanswered && now !== null && now.token === current.token) {
|
|
9197
|
+
const sleep = deps.clock?.sleep;
|
|
9198
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS) throw failure;
|
|
9199
|
+
const bound = UNANSWERED_BASE_MS * 2 ** resends;
|
|
9200
|
+
resends += 1;
|
|
9201
|
+
await sleep.call(deps.clock, Math.floor((rng?.next() ?? 1) * bound));
|
|
9202
|
+
row = now;
|
|
9203
|
+
continue;
|
|
9101
9204
|
}
|
|
9102
|
-
const now = await deps.registry.get(ref);
|
|
9103
9205
|
const settled2 = await churn.settle(current, now, () => deps.registry.get(ref));
|
|
9104
9206
|
if (settled2 === void 0) return rowVerdict(now) ?? "superseded";
|
|
9105
9207
|
row = settled2.row;
|
|
9106
9208
|
}
|
|
9107
9209
|
};
|
|
9210
|
+
const deleteFenced = async (holders) => {
|
|
9211
|
+
const deleted = [];
|
|
9212
|
+
let moved = await fenceInFlight();
|
|
9213
|
+
for (const generation2 of holders) {
|
|
9214
|
+
if (moved !== null) break;
|
|
9215
|
+
moved = rowVerdict(await deps.registry.get(ref));
|
|
9216
|
+
if (moved !== null) break;
|
|
9217
|
+
try {
|
|
9218
|
+
await deps.storage.delete({ ...base, generation: generation2 }, { ifVersion: versions.get(generation2) });
|
|
9219
|
+
} catch (err) {
|
|
9220
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9221
|
+
moved = rowVerdict(await deps.registry.get(ref)) ?? "superseded";
|
|
9222
|
+
break;
|
|
9223
|
+
}
|
|
9224
|
+
deleted.push(generation2);
|
|
9225
|
+
}
|
|
9226
|
+
return { deleted, moved };
|
|
9227
|
+
};
|
|
9228
|
+
const collectedAll = (fromGeneration, collected2) => {
|
|
9229
|
+
seen.collectedOn = record.token;
|
|
9230
|
+
seen.found = true;
|
|
9231
|
+
return {
|
|
9232
|
+
...base,
|
|
9233
|
+
erased: true,
|
|
9234
|
+
fromGeneration,
|
|
9235
|
+
collected: [...collected2].sort((a, b) => a - b)
|
|
9236
|
+
};
|
|
9237
|
+
};
|
|
9238
|
+
const sealedOnly = (answer, collected2) => {
|
|
9239
|
+
if (collected2.length > 0) seen.collectedOn = record.token;
|
|
9240
|
+
return { ...answer, collected: [...collected2].sort((a, b) => a - b) };
|
|
9241
|
+
};
|
|
9108
9242
|
const unpublished = async () => {
|
|
9109
9243
|
const generations = [];
|
|
9110
9244
|
for await (const key2 of deps.storage.list(ref)) generations.push(key2.generation);
|
|
9111
9245
|
const newestFirst = generations.sort((a, b) => b - a);
|
|
9112
|
-
const
|
|
9113
|
-
|
|
9114
|
-
|
|
9115
|
-
|
|
9116
|
-
|
|
9117
|
-
|
|
9118
|
-
|
|
9246
|
+
const holders = [];
|
|
9247
|
+
const clean = /* @__PURE__ */ new Set();
|
|
9248
|
+
for (const { generation: generation2, held } of await holdsEach(newestFirst, false)) {
|
|
9249
|
+
if (held === true) holders.push(generation2);
|
|
9250
|
+
else if (held === false) clean.add(generation2);
|
|
9251
|
+
}
|
|
9252
|
+
const none = { ...base, erased: false, reason: "no-generation", collected: [] };
|
|
9253
|
+
if (holders.length === 0) return none;
|
|
9254
|
+
const newest = holders.find((generation2) => !sealed.has(generation2));
|
|
9255
|
+
const { deleted, moved } = await deleteFenced(holders);
|
|
9256
|
+
const collected2 = [...deleted].sort((a, b) => a - b);
|
|
9257
|
+
const left2 = await holderLeft(clean);
|
|
9258
|
+
if (left2 === void 0) {
|
|
9259
|
+
return newest === void 0 ? sealedOnly(none, collected2) : collectedAll(newest, collected2);
|
|
9260
|
+
}
|
|
9261
|
+
if (moved !== null) {
|
|
9262
|
+
return {
|
|
9263
|
+
...base,
|
|
9264
|
+
erased: false,
|
|
9265
|
+
reason: moved,
|
|
9266
|
+
...newest === void 0 ? {} : { fromGeneration: newest },
|
|
9267
|
+
collected: collected2
|
|
9268
|
+
};
|
|
9269
|
+
}
|
|
9270
|
+
throw cannotRemove(left2);
|
|
9119
9271
|
};
|
|
9120
9272
|
const underTombstone = async () => {
|
|
9121
9273
|
const generations = [];
|
|
@@ -9126,7 +9278,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9126
9278
|
const collected2 = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
9127
9279
|
const left2 = await holderLeft(/* @__PURE__ */ new Set());
|
|
9128
9280
|
if (left2 !== void 0) throw cannotRemove(left2);
|
|
9129
|
-
return
|
|
9281
|
+
return collectedAll(holder.generation, collected2);
|
|
9130
9282
|
};
|
|
9131
9283
|
const notInCurrent = async () => {
|
|
9132
9284
|
const notMember = {
|
|
@@ -9151,38 +9303,47 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9151
9303
|
if (held === true) holdersAbove.push(generation2);
|
|
9152
9304
|
else if (held === false) clean.add(generation2);
|
|
9153
9305
|
}
|
|
9306
|
+
const foundAbove = holdersAbove.find((generation2) => !sealed.has(generation2));
|
|
9154
9307
|
let holderBelow;
|
|
9155
|
-
|
|
9308
|
+
const sealedBelow = [];
|
|
9309
|
+
if (foundAbove === void 0) {
|
|
9156
9310
|
for (const { generation: generation2, held } of await holdsEach(
|
|
9157
9311
|
newestFirst.filter((g) => g < from),
|
|
9158
9312
|
true
|
|
9159
9313
|
)) {
|
|
9160
|
-
if (held
|
|
9314
|
+
if (found(generation2, held)) holderBelow = generation2;
|
|
9315
|
+
else if (held === true) sealedBelow.push(generation2);
|
|
9161
9316
|
else if (held === false) clean.add(generation2);
|
|
9162
9317
|
}
|
|
9163
9318
|
}
|
|
9164
|
-
const newest =
|
|
9165
|
-
|
|
9166
|
-
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
|
|
9170
|
-
|
|
9171
|
-
|
|
9172
|
-
|
|
9173
|
-
collected2.push(generation2);
|
|
9319
|
+
const newest = foundAbove ?? holderBelow;
|
|
9320
|
+
const fenced = newest === void 0 ? [...holdersAbove, ...sealedBelow] : holdersAbove;
|
|
9321
|
+
if (newest === void 0 && fenced.length === 0) return notMember;
|
|
9322
|
+
const collected2 = newest === void 0 ? [] : [...await gcOrphanGenerations(ref, deps, { keep: 0 })];
|
|
9323
|
+
let moved = null;
|
|
9324
|
+
if (fenced.length > 0) {
|
|
9325
|
+
const deleted = await deleteFenced(fenced);
|
|
9326
|
+
collected2.push(...deleted.deleted);
|
|
9327
|
+
moved = deleted.moved;
|
|
9174
9328
|
}
|
|
9175
9329
|
const left2 = await holderLeft(clean);
|
|
9176
|
-
if (left2 === void 0)
|
|
9330
|
+
if (left2 === void 0) {
|
|
9331
|
+
if (newest !== void 0) return collectedAll(newest, collected2);
|
|
9332
|
+
return sealedOnly(notMember, collected2);
|
|
9333
|
+
}
|
|
9177
9334
|
if (moved !== null) {
|
|
9178
|
-
return { ...base, erased: false, reason: moved, fromGeneration: newest, collected: collected2 };
|
|
9335
|
+
return { ...base, erased: false, reason: moved, fromGeneration: newest ?? from, collected: collected2 };
|
|
9179
9336
|
}
|
|
9180
9337
|
throw cannotRemove(left2);
|
|
9181
9338
|
};
|
|
9339
|
+
let rewrote;
|
|
9182
9340
|
const stage = async () => {
|
|
9183
9341
|
let written;
|
|
9342
|
+
const described = usableSummary(ref, record, aead);
|
|
9184
9343
|
try {
|
|
9185
|
-
const reader = await read(
|
|
9344
|
+
const reader = await read(
|
|
9345
|
+
() => openGenerationReader(deps.storage, fromKey, cryptoAt(from), {}, described?.fingerprint)
|
|
9346
|
+
);
|
|
9186
9347
|
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
9187
9348
|
if (bytes === null) return await notInCurrent();
|
|
9188
9349
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
@@ -9192,10 +9353,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9192
9353
|
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
9193
9354
|
const generation2 = await nextGeneration(ref, deps);
|
|
9194
9355
|
const key2 = { ...base, generation: generation2 };
|
|
9195
|
-
const metadata = metadataToCarry(
|
|
9196
|
-
reader.metadata,
|
|
9197
|
-
aead === void 0 ? void 0 : usableSummary(ref, record, aead)
|
|
9198
|
-
);
|
|
9356
|
+
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
9199
9357
|
const tally = await writeCrbmGenerationStream(
|
|
9200
9358
|
deps.storage,
|
|
9201
9359
|
key2,
|
|
@@ -9203,6 +9361,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9203
9361
|
{ crypto: cryptoAt(generation2), clock: deps.clock, metadata }
|
|
9204
9362
|
);
|
|
9205
9363
|
written = generation2;
|
|
9364
|
+
rewrote = { generation: generation2, fingerprint: tally.fingerprint };
|
|
9206
9365
|
const beforeVerify = await deps.registry.get(ref);
|
|
9207
9366
|
const early = rowVerdict(beforeVerify);
|
|
9208
9367
|
if (early !== null) return refused(early, written);
|
|
@@ -9212,7 +9371,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9212
9371
|
const summary2 = summaryOf(
|
|
9213
9372
|
ref,
|
|
9214
9373
|
generation2,
|
|
9215
|
-
{ cardinality: tally.cardinality, metadata },
|
|
9374
|
+
{ cardinality: tally.cardinality, metadata, fingerprint: tally.fingerprint },
|
|
9216
9375
|
aead
|
|
9217
9376
|
);
|
|
9218
9377
|
return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
|
|
@@ -9229,22 +9388,31 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9229
9388
|
throw err;
|
|
9230
9389
|
}
|
|
9231
9390
|
};
|
|
9232
|
-
const discardRefused = async (
|
|
9391
|
+
const discardRefused = async (own) => {
|
|
9392
|
+
const key2 = { ...base, generation: own.generation };
|
|
9233
9393
|
const row = await deps.registry.get(ref);
|
|
9234
9394
|
if (row !== null && row.status === "destroyed") {
|
|
9235
|
-
await deps.storage.delete(
|
|
9395
|
+
await deps.storage.delete(key2);
|
|
9236
9396
|
return row;
|
|
9237
9397
|
}
|
|
9238
9398
|
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
9239
|
-
if (row.currentGen <= from || row.currentGen >=
|
|
9240
|
-
await deps.storage
|
|
9399
|
+
if (row.currentGen <= from || row.currentGen >= own.generation) return row;
|
|
9400
|
+
const proven = await provenObject(deps.storage, key2, own.fingerprint);
|
|
9401
|
+
if (proven === null) return row;
|
|
9402
|
+
try {
|
|
9403
|
+
await deps.storage.delete(key2, { ifVersion: proven.version });
|
|
9404
|
+
} catch (err) {
|
|
9405
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9406
|
+
}
|
|
9241
9407
|
return row;
|
|
9242
9408
|
};
|
|
9243
9409
|
if (tombstoned) return await underTombstone();
|
|
9244
9410
|
if (pointerless) return await unpublished();
|
|
9245
9411
|
const staged = await stage();
|
|
9246
9412
|
if ("erased" in staged) {
|
|
9247
|
-
if (!staged.erased && staged.generation !== void 0
|
|
9413
|
+
if (!staged.erased && staged.generation !== void 0 && rewrote !== void 0) {
|
|
9414
|
+
await discardRefused(rewrote);
|
|
9415
|
+
}
|
|
9248
9416
|
return staged;
|
|
9249
9417
|
}
|
|
9250
9418
|
const { generation, key, fingerprint, summary } = staged;
|
|
@@ -9265,7 +9433,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9265
9433
|
rng: deps.rng ?? deps.readRetry?.rng
|
|
9266
9434
|
});
|
|
9267
9435
|
if (!published) {
|
|
9268
|
-
const now = await discardRefused(generation);
|
|
9436
|
+
const now = await discardRefused({ generation, fingerprint });
|
|
9269
9437
|
return {
|
|
9270
9438
|
...base,
|
|
9271
9439
|
erased: false,
|
|
@@ -9275,10 +9443,11 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9275
9443
|
collected: []
|
|
9276
9444
|
};
|
|
9277
9445
|
}
|
|
9278
|
-
|
|
9446
|
+
audit.onEvent({
|
|
9279
9447
|
kind: "segment.rewrite",
|
|
9280
9448
|
namespace: ref.namespace,
|
|
9281
9449
|
segment: ref.segment,
|
|
9450
|
+
...incarnationField(fromToken),
|
|
9282
9451
|
fromGeneration: from,
|
|
9283
9452
|
generation
|
|
9284
9453
|
});
|
|
@@ -9287,6 +9456,29 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9287
9456
|
if (left !== void 0) throw cannotRemove(left);
|
|
9288
9457
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
9289
9458
|
}
|
|
9459
|
+
function noticingIndex(crypto, index) {
|
|
9460
|
+
const indexAad = crypto.aadFor("index");
|
|
9461
|
+
return {
|
|
9462
|
+
aadFor: crypto.aadFor,
|
|
9463
|
+
aead: {
|
|
9464
|
+
seal: (plaintext, aad) => crypto.aead.seal(plaintext, aad),
|
|
9465
|
+
open: (sealed, aad) => {
|
|
9466
|
+
try {
|
|
9467
|
+
return crypto.aead.open(sealed, aad);
|
|
9468
|
+
} catch (err) {
|
|
9469
|
+
if (sameBytes(aad, indexAad)) index.refused = true;
|
|
9470
|
+
throw err;
|
|
9471
|
+
}
|
|
9472
|
+
}
|
|
9473
|
+
}
|
|
9474
|
+
};
|
|
9475
|
+
}
|
|
9476
|
+
function sameBytes(a, b) {
|
|
9477
|
+
return a.length === b.length && a.every((byte, i) => byte === b[i]);
|
|
9478
|
+
}
|
|
9479
|
+
function renewedSince(held, now) {
|
|
9480
|
+
return now.pointerId !== held.pointerId && onlyLeasesDiffer(held, now, ["pointerId"]);
|
|
9481
|
+
}
|
|
9290
9482
|
var REWRITE_RANGES_AHEAD = 4;
|
|
9291
9483
|
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
9292
9484
|
const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
@@ -9605,12 +9797,13 @@ async function destroySegment(ref, deps, options) {
|
|
|
9605
9797
|
`destroySegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental crypto-shred)`
|
|
9606
9798
|
);
|
|
9607
9799
|
}
|
|
9608
|
-
const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9800
|
+
const { token, ...result } = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9609
9801
|
if (result.cryptoShredded) {
|
|
9610
9802
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9611
9803
|
kind: "segment.erase",
|
|
9612
9804
|
namespace: ref.namespace,
|
|
9613
|
-
segment: ref.segment
|
|
9805
|
+
segment: ref.segment,
|
|
9806
|
+
...incarnationField(token)
|
|
9614
9807
|
});
|
|
9615
9808
|
}
|
|
9616
9809
|
return result;
|
|
@@ -9641,8 +9834,9 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9641
9834
|
let segmentsShredded = 0;
|
|
9642
9835
|
const destroyed = await mapWithConcurrency(refs, ERASE_CONCURRENCY, async (ref) => {
|
|
9643
9836
|
let result;
|
|
9837
|
+
let token;
|
|
9644
9838
|
try {
|
|
9645
|
-
result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9839
|
+
({ token, ...result } = await shredSegment(ref, deps, options.allowCleartext ?? false));
|
|
9646
9840
|
} catch (err) {
|
|
9647
9841
|
result = {
|
|
9648
9842
|
segment: ref.segment,
|
|
@@ -9654,7 +9848,12 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9654
9848
|
}
|
|
9655
9849
|
if (result.cryptoShredded) {
|
|
9656
9850
|
segmentsShredded += 1;
|
|
9657
|
-
audit.onEvent({
|
|
9851
|
+
audit.onEvent({
|
|
9852
|
+
kind: "segment.erase",
|
|
9853
|
+
namespace: ref.namespace,
|
|
9854
|
+
segment: ref.segment,
|
|
9855
|
+
...incarnationField(token)
|
|
9856
|
+
});
|
|
9658
9857
|
}
|
|
9659
9858
|
return result;
|
|
9660
9859
|
});
|
|
@@ -9703,13 +9902,13 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9703
9902
|
};
|
|
9704
9903
|
}
|
|
9705
9904
|
try {
|
|
9706
|
-
await deps.registry.create(ref, {
|
|
9905
|
+
const tombstone = await deps.registry.create(ref, {
|
|
9707
9906
|
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
9708
9907
|
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
9709
9908
|
currentGen: orphans[orphans.length - 1],
|
|
9710
9909
|
status: "destroyed"
|
|
9711
9910
|
});
|
|
9712
|
-
shred = { ...shred, destroyed: true, reason: void 0 };
|
|
9911
|
+
shred = { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
9713
9912
|
} catch (err) {
|
|
9714
9913
|
if (!isWriteConflictError(err)) throw err;
|
|
9715
9914
|
shred = await shredSegment(ref, deps, true, "dropSegment");
|
|
@@ -9719,7 +9918,8 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9719
9918
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9720
9919
|
kind: "segment.erase",
|
|
9721
9920
|
namespace: ref.namespace,
|
|
9722
|
-
segment: ref.segment
|
|
9921
|
+
segment: ref.segment,
|
|
9922
|
+
...incarnationField(shred.token)
|
|
9723
9923
|
});
|
|
9724
9924
|
}
|
|
9725
9925
|
const generationsDeleted = [];
|
|
@@ -9749,6 +9949,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9749
9949
|
kind: "segment.dispose",
|
|
9750
9950
|
namespace: ref.namespace,
|
|
9751
9951
|
segment: ref.segment,
|
|
9952
|
+
...incarnationField(shred.token),
|
|
9752
9953
|
generationsDeleted: generationsDeleted.length
|
|
9753
9954
|
});
|
|
9754
9955
|
}
|
|
@@ -9786,7 +9987,13 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9786
9987
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "absent" };
|
|
9787
9988
|
}
|
|
9788
9989
|
if (record.status === "destroyed") {
|
|
9789
|
-
return {
|
|
9990
|
+
return {
|
|
9991
|
+
...base,
|
|
9992
|
+
destroyed: true,
|
|
9993
|
+
cryptoShredded: false,
|
|
9994
|
+
reason: "already",
|
|
9995
|
+
token: record.token
|
|
9996
|
+
};
|
|
9790
9997
|
}
|
|
9791
9998
|
const encrypted = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
9792
9999
|
if (!encrypted && !allowCleartext) {
|
|
@@ -9794,7 +10001,7 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9794
10001
|
}
|
|
9795
10002
|
const unmarked = sweep === void 0 ? withoutSweepMark(record.retention) : null;
|
|
9796
10003
|
try {
|
|
9797
|
-
await deps.registry.compareAndSwap(ref, record.token, {
|
|
10004
|
+
const tombstone = await deps.registry.compareAndSwap(ref, record.token, {
|
|
9798
10005
|
status: "destroyed",
|
|
9799
10006
|
wrappedDeks: void 0,
|
|
9800
10007
|
// ← the crypto-shred: the only copy of the DEK wrappings is gone
|
|
@@ -9808,7 +10015,7 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9808
10015
|
// one a restored or hand-edited row carried.
|
|
9809
10016
|
...sweep !== void 0 ? { retention: sweep.stamp(record.retention) } : unmarked === null ? {} : { retention: unmarked }
|
|
9810
10017
|
});
|
|
9811
|
-
return { ...base, destroyed: true, cryptoShredded: encrypted };
|
|
10018
|
+
return { ...base, destroyed: true, cryptoShredded: encrypted, token: tombstone.token };
|
|
9812
10019
|
} catch (err) {
|
|
9813
10020
|
if (!isWriteConflictError(err)) throw err;
|
|
9814
10021
|
lost = record;
|
|
@@ -10223,17 +10430,7 @@ async function checkSummary(ref, live, deps) {
|
|
|
10223
10430
|
throw err;
|
|
10224
10431
|
}
|
|
10225
10432
|
}
|
|
10226
|
-
const
|
|
10227
|
-
if (described === void 0) return { kind: "ok", unchecked: keyed };
|
|
10228
|
-
const reader = await openGenerationReader(
|
|
10229
|
-
deps.storage,
|
|
10230
|
-
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
10231
|
-
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) }
|
|
10232
|
-
);
|
|
10233
|
-
let cardinality = 0;
|
|
10234
|
-
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
10235
|
-
if (summaryAgrees(described, { cardinality, metadata: reader.metadata })) return { kind: "ok" };
|
|
10236
|
-
return {
|
|
10433
|
+
const mismatch = {
|
|
10237
10434
|
kind: "issue",
|
|
10238
10435
|
issue: {
|
|
10239
10436
|
segment: ref.segment,
|
|
@@ -10242,6 +10439,24 @@ async function checkSummary(ref, live, deps) {
|
|
|
10242
10439
|
issue: "summary-mismatch"
|
|
10243
10440
|
}
|
|
10244
10441
|
};
|
|
10442
|
+
const described = usableSummary(ref, live, aead);
|
|
10443
|
+
if (described === void 0) {
|
|
10444
|
+
const summary = live.summary;
|
|
10445
|
+
const unnamed = !keyed && live.status === "active" && summary !== void 0 && !("sealed" in summary) && summary.generation === generation && fingerprintParts(summary.fingerprint) === void 0;
|
|
10446
|
+
return unnamed ? mismatch : { kind: "ok", unchecked: keyed };
|
|
10447
|
+
}
|
|
10448
|
+
const reader = await openNamedGeneration(
|
|
10449
|
+
deps.storage,
|
|
10450
|
+
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
10451
|
+
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) },
|
|
10452
|
+
described.fingerprint
|
|
10453
|
+
);
|
|
10454
|
+
let cardinality = 0;
|
|
10455
|
+
for (const n of reader?.cardinalities().values() ?? []) cardinality += n;
|
|
10456
|
+
if (reader !== void 0 && summaryAgrees(described, { cardinality, metadata: reader.metadata })) {
|
|
10457
|
+
return { kind: "ok" };
|
|
10458
|
+
}
|
|
10459
|
+
return mismatch;
|
|
10245
10460
|
}
|
|
10246
10461
|
async function runConsistencyCheck(deps, options = {}) {
|
|
10247
10462
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
@@ -10328,17 +10543,11 @@ var RetryingStorageChunkSource = class {
|
|
|
10328
10543
|
getChunks;
|
|
10329
10544
|
cardinalities;
|
|
10330
10545
|
summary;
|
|
10546
|
+
stat;
|
|
10331
10547
|
currentGeneration;
|
|
10332
10548
|
invalidate;
|
|
10333
10549
|
exists;
|
|
10334
10550
|
currentVersion;
|
|
10335
|
-
/**
|
|
10336
|
-
* Forwarded, not retried: a property, with no request behind it. Without it the grounded cost report priced a
|
|
10337
|
-
* retrying store's refresh at the default TTL whatever the store was configured with.
|
|
10338
|
-
*/
|
|
10339
|
-
get pointerRefreshMs() {
|
|
10340
|
-
return this.inner.pointerRefreshMs;
|
|
10341
|
-
}
|
|
10342
10551
|
constructor(inner, opts) {
|
|
10343
10552
|
this.inner = inner;
|
|
10344
10553
|
const r = toRetry(opts);
|
|
@@ -10366,6 +10575,10 @@ var RetryingStorageChunkSource = class {
|
|
|
10366
10575
|
if (innerSummary) {
|
|
10367
10576
|
this.summary = (ref) => withRetry(() => innerSummary.call(inner, ref), this.policy, this.deps);
|
|
10368
10577
|
}
|
|
10578
|
+
const innerStat = inner.stat;
|
|
10579
|
+
if (innerStat) {
|
|
10580
|
+
this.stat = (ref) => withRetry(() => innerStat.call(inner, ref), this.policy, this.deps);
|
|
10581
|
+
}
|
|
10369
10582
|
const innerCurrentGeneration = inner.currentGeneration;
|
|
10370
10583
|
if (innerCurrentGeneration) {
|
|
10371
10584
|
this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);
|
|
@@ -10393,7 +10606,6 @@ var RetryingStorageChunkSource = class {
|
|
|
10393
10606
|
}
|
|
10394
10607
|
};
|
|
10395
10608
|
export {
|
|
10396
|
-
AWS_US_EAST_1_ONDEMAND,
|
|
10397
10609
|
BoundedLru,
|
|
10398
10610
|
BudgetExceededError,
|
|
10399
10611
|
BufferReader,
|
|
@@ -10404,7 +10616,6 @@ export {
|
|
|
10404
10616
|
CrbmStorageChunkSource,
|
|
10405
10617
|
DEFAULT_BUDGET,
|
|
10406
10618
|
DEFAULT_RETRY_POLICY,
|
|
10407
|
-
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
10408
10619
|
InProcessKeystore,
|
|
10409
10620
|
IntegrityError,
|
|
10410
10621
|
KeyUnavailableError,
|
|
@@ -10419,7 +10630,6 @@ export {
|
|
|
10419
10630
|
NOOP_METRICS,
|
|
10420
10631
|
NodeAead,
|
|
10421
10632
|
NotFoundError,
|
|
10422
|
-
ONE_REDIS_HA_CLUSTER,
|
|
10423
10633
|
PinnedStorageChunkSource,
|
|
10424
10634
|
RecordingAuditSink,
|
|
10425
10635
|
RetryingStorageChunkSource,
|
|
@@ -10439,10 +10649,8 @@ export {
|
|
|
10439
10649
|
dropSegment,
|
|
10440
10650
|
eraseIdFromSegment,
|
|
10441
10651
|
eraseNamespace,
|
|
10442
|
-
estimateCost,
|
|
10443
10652
|
excludingReservedRows,
|
|
10444
10653
|
getSegmentRetention,
|
|
10445
|
-
groundedReport,
|
|
10446
10654
|
isBudgetExceededError,
|
|
10447
10655
|
isCapabilityError,
|
|
10448
10656
|
isCloudRoaringError,
|