@cloudbitmaps/core 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RSLVTRJY.js → chunk-YSC7NJWV.js} +314 -216
- package/dist/chunk-YSC7NJWV.js.map +7 -0
- package/dist/core/audit.d.ts +56 -3
- package/dist/core/combine-many.d.ts +9 -6
- package/dist/core/consistency.d.ts +10 -4
- package/dist/core/crbm/fingerprint.d.ts +18 -0
- package/dist/core/crbm/format.d.ts +2 -4
- package/dist/core/crbm/frame.d.ts +10 -0
- package/dist/core/crbm/reader.d.ts +1 -8
- package/dist/core/crbm-storage-source.d.ts +229 -89
- package/dist/core/engine.d.ts +17 -14
- package/dist/core/erase-id.d.ts +62 -27
- package/dist/core/generation-gc.d.ts +2 -1
- package/dist/core/leases.d.ts +7 -0
- package/dist/core/load.d.ts +5 -3
- package/dist/core/metrics.d.ts +5 -5
- package/dist/core/pinned-storage-source.d.ts +3 -3
- package/dist/core/pointer-id.d.ts +19 -0
- package/dist/core/ports.d.ts +127 -35
- package/dist/core/reader-defaults.d.ts +14 -3
- package/dist/core/rollback.d.ts +2 -1
- package/dist/core/summary.d.ts +38 -21
- package/dist/core/token.d.ts +13 -8
- package/dist/driver-kit.d.ts +2 -1
- package/dist/driver-kit.js +11 -4
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +1 -1
- package/dist/drivers/_shared/registry.d.ts +26 -17
- package/dist/drivers/localfs/storage.d.ts +7 -6
- package/dist/drivers/memory.d.ts +8 -6
- package/dist/drivers/retry/retrying-chunk-source.d.ts +1 -5
- package/dist/export/index.d.ts +16 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.js +1169 -1041
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-RSLVTRJY.js.map +0 -7
- package/dist/core/cost.d.ts +0 -351
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
CapabilityError,
|
|
4
4
|
CloudRoaringError,
|
|
5
5
|
DEFAULT_NAMESPACE,
|
|
6
|
+
FOOTER_BYTES,
|
|
6
7
|
IntegrityError,
|
|
7
8
|
KeyUnavailableError,
|
|
8
9
|
LEASE_SKEW_MS,
|
|
@@ -14,9 +15,12 @@ import {
|
|
|
14
15
|
MAX_METADATA_BYTES,
|
|
15
16
|
MAX_NAME_LENGTH,
|
|
16
17
|
NotFoundError,
|
|
18
|
+
PREAMBLE_BYTES,
|
|
17
19
|
RESERVED_NAMESPACE_PREFIX,
|
|
18
20
|
StaleOperandError,
|
|
19
21
|
TransientError,
|
|
22
|
+
UNANSWERED_BASE_MS,
|
|
23
|
+
UNANSWERED_RESENDS,
|
|
20
24
|
UnsupportedError,
|
|
21
25
|
ValidationError,
|
|
22
26
|
WriteConflictError,
|
|
@@ -24,6 +28,7 @@ import {
|
|
|
24
28
|
assertRegistryCanWrite,
|
|
25
29
|
brandAsBackend,
|
|
26
30
|
canonicalMetadataJson,
|
|
31
|
+
compactFingerprint,
|
|
27
32
|
copiedMetadata,
|
|
28
33
|
decodeNameFromPath,
|
|
29
34
|
drawIncarnation,
|
|
@@ -32,7 +37,10 @@ import {
|
|
|
32
37
|
encodeNameForPath,
|
|
33
38
|
encodedNameLength,
|
|
34
39
|
entropyIsAvailable,
|
|
40
|
+
fingerprintFor,
|
|
41
|
+
fingerprintParts,
|
|
35
42
|
heldGenerations,
|
|
43
|
+
incarnationField,
|
|
36
44
|
incarnationOf,
|
|
37
45
|
isBudgetExceededError,
|
|
38
46
|
isCapabilityError,
|
|
@@ -60,10 +68,13 @@ import {
|
|
|
60
68
|
nextRegistryToken,
|
|
61
69
|
onlyLeasesDiffer,
|
|
62
70
|
parseRegistryEnvelope,
|
|
71
|
+
pointerIdOf,
|
|
63
72
|
recordFromNew,
|
|
64
73
|
releaseLease,
|
|
74
|
+
renewPointer,
|
|
65
75
|
sameIncarnation,
|
|
66
76
|
serializeRegistryEnvelope,
|
|
77
|
+
sizePart,
|
|
67
78
|
takeLease,
|
|
68
79
|
usableKeptGens,
|
|
69
80
|
validateChunkKeyOrder,
|
|
@@ -74,7 +85,7 @@ import {
|
|
|
74
85
|
validateUserNamespace,
|
|
75
86
|
validateUserRef,
|
|
76
87
|
webCryptoEntropy
|
|
77
|
-
} from "./chunk-
|
|
88
|
+
} from "./chunk-YSC7NJWV.js";
|
|
78
89
|
|
|
79
90
|
// src/core/bit-route.ts
|
|
80
91
|
var U32_MAX = 4294967295;
|
|
@@ -252,9 +263,7 @@ var ChunkWindow = class {
|
|
|
252
263
|
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
253
264
|
var VERSION_MAJOR = 1;
|
|
254
265
|
var VERSION_MINOR = 0;
|
|
255
|
-
var PREAMBLE_BYTES = 8;
|
|
256
266
|
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
257
|
-
var FOOTER_BYTES = 104;
|
|
258
267
|
var FOOTER = {
|
|
259
268
|
indexOffset: 0,
|
|
260
269
|
// u64
|
|
@@ -412,7 +421,9 @@ var RecordingAuditSink = class {
|
|
|
412
421
|
}
|
|
413
422
|
/** An independent copy of the recorded events, in emission order. */
|
|
414
423
|
snapshot() {
|
|
415
|
-
return this.recorded.map(
|
|
424
|
+
return this.recorded.map(
|
|
425
|
+
(e) => e.kind === "segment.collect" ? { ...e, collected: [...e.collected] } : { ...e }
|
|
426
|
+
);
|
|
416
427
|
}
|
|
417
428
|
reset() {
|
|
418
429
|
this.recorded.length = 0;
|
|
@@ -704,27 +715,30 @@ var SegmentEngine = class {
|
|
|
704
715
|
return total;
|
|
705
716
|
}
|
|
706
717
|
/**
|
|
707
|
-
* What the segment's current generation is: its number, its id count
|
|
708
|
-
*
|
|
718
|
+
* What the segment's current generation is: its number, its id count, its metadata and the size of its object in
|
|
719
|
+
* bytes. A source with a `stat` (the `.crbm` source) answers all four from one resolution, so they cannot straddle
|
|
720
|
+
* a publish: from the registry row's summary when it can use it, else from the opened generation's footer and
|
|
721
|
+
* index, with no payload read. A source without one answers from its other methods, each its own resolution: its
|
|
722
|
+
* summary when it has one, else `count()` and the generation it resolves, and its `sizeOf`, or `sizeBytes: null`
|
|
723
|
+
* when it has none. A segment with no generation answers `{ generation: null, cardinality: 0, sizeBytes: null }`.
|
|
709
724
|
*/
|
|
710
725
|
async stat(seg) {
|
|
726
|
+
if (this.storage.stat) {
|
|
727
|
+
const found = await this.storage.stat(seg);
|
|
728
|
+
if (found === null) return { generation: null, cardinality: 0, sizeBytes: null };
|
|
729
|
+
return found;
|
|
730
|
+
}
|
|
711
731
|
if (this.storage.summary) {
|
|
712
|
-
|
|
732
|
+
const summary = await this.storage.summary(seg);
|
|
733
|
+
if (summary === null) return { generation: null, cardinality: 0, sizeBytes: null };
|
|
734
|
+
return { ...summary, sizeBytes: await this.sizeOf(seg) };
|
|
713
735
|
}
|
|
714
736
|
const generation = this.storage.currentGeneration ? await this.storage.currentGeneration(seg) : null;
|
|
715
|
-
return { generation, cardinality: await this.count(seg) };
|
|
716
|
-
}
|
|
717
|
-
/** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
718
|
-
get supportsStorageSize() {
|
|
719
|
-
return typeof this.storage.sizeOf === "function";
|
|
720
|
-
}
|
|
721
|
-
/** How often the Storage source re-reads a pointer while reading (cost reporting); undefined if it does not say. */
|
|
722
|
-
get pointerRefreshMs() {
|
|
723
|
-
return this.storage.pointerRefreshMs;
|
|
737
|
+
return { generation, cardinality: await this.count(seg), sizeBytes: await this.sizeOf(seg) };
|
|
724
738
|
}
|
|
725
|
-
/**
|
|
726
|
-
|
|
727
|
-
return this.storage.sizeOf ? this.storage.sizeOf(seg) :
|
|
739
|
+
/** The size of the segment's current generation from the source's `sizeOf`, or `null` when it cannot say. */
|
|
740
|
+
async sizeOf(seg) {
|
|
741
|
+
return this.storage.sizeOf ? (await this.storage.sizeOf(seg))?.sizeBytes ?? null : null;
|
|
728
742
|
}
|
|
729
743
|
/** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
|
|
730
744
|
chunkWindow(seg, chunkKeys, gen, ramp) {
|
|
@@ -956,11 +970,13 @@ var SegmentEngine = class {
|
|
|
956
970
|
* the result — every chunk read is a whole, checksum-verified, immutable generation. The edge a *long* call can
|
|
957
971
|
* hit is invariant 3's: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
|
|
958
972
|
* operand's not-yet-requested chunks may re-resolve forward to the newer generation (a generation hop within one long
|
|
959
|
-
* call) — the call never crashes or returns a torn object, but may mix generations.
|
|
960
|
-
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
|
|
961
|
-
* (`maxOpenSegments`), whose re-read re-resolves fresh
|
|
962
|
-
*
|
|
963
|
-
*
|
|
973
|
+
* call) — the call never crashes or returns a torn object, but may mix generations. Some things hop it without
|
|
974
|
+
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call** on a
|
|
975
|
+
* store with no timed refresh (`maxOpenSegments`), whose re-read re-resolves fresh, and on a store with one the
|
|
976
|
+
* resolution cache letting the operand's resolution go; a sweep deleting the generation it was reading, or another
|
|
977
|
+
* object under its number, met by this read or another, which heals the read forward, as another read's failed open
|
|
978
|
+
* of the operand does; and an invalidation, which this store's own `load`, `rollback` and `eraseSubject` make and
|
|
979
|
+
* `invalidate()` makes on request. Still whole/immutable per read, never torn. An operand's stream resolves its
|
|
964
980
|
* segment again before it serves each chunk, so each of those moves a running read as it would a read of one chunk, and
|
|
965
981
|
* the ranges it had requested of the earlier generation are dropped, not served.
|
|
966
982
|
*/
|
|
@@ -1656,599 +1672,6 @@ var BoundedLru = class {
|
|
|
1656
1672
|
}
|
|
1657
1673
|
};
|
|
1658
1674
|
|
|
1659
|
-
// src/core/generation-gc.ts
|
|
1660
|
-
async function nextGeneration(ref, deps) {
|
|
1661
|
-
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
1662
|
-
}
|
|
1663
|
-
async function aboveEverything(ref, deps, record) {
|
|
1664
|
-
let highest = record?.currentGen ?? -1;
|
|
1665
|
-
for await (const key of deps.storage.list(ref)) {
|
|
1666
|
-
if (key.generation > highest) highest = key.generation;
|
|
1667
|
-
}
|
|
1668
|
-
return numberAbove(highest);
|
|
1669
|
-
}
|
|
1670
|
-
function numberAbove(highest) {
|
|
1671
|
-
if (highest >= Number.MAX_SAFE_INTEGER) {
|
|
1672
|
-
throw new IntegrityError("the segment has no generation number left above its highest");
|
|
1673
|
-
}
|
|
1674
|
-
return highest + 1;
|
|
1675
|
-
}
|
|
1676
|
-
async function nextLoadGeneration(ref, deps, row) {
|
|
1677
|
-
const generation = numberAbove(row?.currentGen ?? -1);
|
|
1678
|
-
try {
|
|
1679
|
-
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
1680
|
-
} catch (err) {
|
|
1681
|
-
if (isNotFoundError(err)) return { generation, checked: true };
|
|
1682
|
-
}
|
|
1683
|
-
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
1684
|
-
}
|
|
1685
|
-
var LIST_COLLECTION_CADENCE = 16;
|
|
1686
|
-
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
1687
|
-
return (await listingPass(ref, deps, { keep: options.keep })).deleted;
|
|
1688
|
-
}
|
|
1689
|
-
async function listingPass(ref, deps, options) {
|
|
1690
|
-
const keep = options.keep ?? 1;
|
|
1691
|
-
const quiet = options.quiet === true;
|
|
1692
|
-
if (!Number.isInteger(keep) || keep < 0) {
|
|
1693
|
-
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
1694
|
-
}
|
|
1695
|
-
const record = await deps.registry.get(ref);
|
|
1696
|
-
if (record === null) return { deleted: [], kept: [], complete: true };
|
|
1697
|
-
const current = record.currentGen;
|
|
1698
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1699
|
-
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
1700
|
-
const gens = [...seen];
|
|
1701
|
-
const after = await deps.registry.get(ref);
|
|
1702
|
-
if (after === null) {
|
|
1703
|
-
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
1704
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
1705
|
-
}
|
|
1706
|
-
if (record.status === "destroyed" && after.token !== record.token) {
|
|
1707
|
-
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
1708
|
-
throw new WriteConflictError(
|
|
1709
|
-
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
1710
|
-
);
|
|
1711
|
-
}
|
|
1712
|
-
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
1713
|
-
const named = record.status === "destroyed" ? void 0 : usableKeptGens(after);
|
|
1714
|
-
const protect = options.protect === void 0 || named === void 0 ? void 0 : /* @__PURE__ */ new Set([...options.protect, ...named]);
|
|
1715
|
-
const stillCollectable = async (generation) => {
|
|
1716
|
-
const still = await deps.registry.get(ref);
|
|
1717
|
-
if (still === null) {
|
|
1718
|
-
if (quiet) return "stop";
|
|
1719
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
1720
|
-
}
|
|
1721
|
-
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
1722
|
-
const now = protect === void 0 ? void 0 : usableKeptGens(still);
|
|
1723
|
-
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
1724
|
-
if (!ok || !unnamed) {
|
|
1725
|
-
if (quiet) return "stop";
|
|
1726
|
-
throw new WriteConflictError(
|
|
1727
|
-
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
1728
|
-
);
|
|
1729
|
-
}
|
|
1730
|
-
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
1731
|
-
return "delete";
|
|
1732
|
-
};
|
|
1733
|
-
const guard = record.status === "destroyed" ? void 0 : options.leases;
|
|
1734
|
-
const held = guard === void 0 ? void 0 : heldGenerations(after, guard.now());
|
|
1735
|
-
const below = record.status === "destroyed" || cutoff === null ? [] : gens.filter((g) => g < cutoff).sort((a, b) => b - a);
|
|
1736
|
-
const candidates = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
1737
|
-
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
1738
|
-
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
1739
|
-
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
1740
|
-
[]
|
|
1741
|
-
) : protect !== void 0 ? (
|
|
1742
|
-
// The row's list is the window: everything below the pointer it does not name is garbage.
|
|
1743
|
-
below.filter((g) => !protect.has(g))
|
|
1744
|
-
) : (
|
|
1745
|
-
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
1746
|
-
below.slice(keep)
|
|
1747
|
-
);
|
|
1748
|
-
const leasedSpared = new Set(candidates.filter((g) => held?.has(g) === true));
|
|
1749
|
-
const toDelete = candidates.filter((g) => !leasedSpared.has(g));
|
|
1750
|
-
const done = [];
|
|
1751
|
-
for (const generation of toDelete) {
|
|
1752
|
-
const verdict = await stillCollectable(generation);
|
|
1753
|
-
if (verdict === "stop") return { deleted: done, kept: [], complete: false };
|
|
1754
|
-
if (verdict === "skip") {
|
|
1755
|
-
leasedSpared.add(generation);
|
|
1756
|
-
continue;
|
|
1757
|
-
}
|
|
1758
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
1759
|
-
done.push(generation);
|
|
1760
|
-
}
|
|
1761
|
-
const deleted = new Set(done);
|
|
1762
|
-
return {
|
|
1763
|
-
deleted: done,
|
|
1764
|
-
kept: below.filter((g) => !deleted.has(g) && !leasedSpared.has(g)).reverse(),
|
|
1765
|
-
complete: true
|
|
1766
|
-
};
|
|
1767
|
-
}
|
|
1768
|
-
async function collectAfterLoad(ref, deps, options) {
|
|
1769
|
-
const { generation, keep, byName, currentGone, kept, proveCurrent, leases } = options;
|
|
1770
|
-
if (keep >= generation) return [];
|
|
1771
|
-
const list = kept?.list;
|
|
1772
|
-
if (list === void 0 || currentGone === true)
|
|
1773
|
-
return reconcile(ref, deps, { keep, kept, leases });
|
|
1774
|
-
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
1775
|
-
if (byName && !periodic) {
|
|
1776
|
-
if (keep >= 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
1777
|
-
return reconcile(ref, deps, { keep, kept, leases });
|
|
1778
|
-
}
|
|
1779
|
-
return deleteEvicted(ref, deps, { generation, evict: kept.evict, leases });
|
|
1780
|
-
}
|
|
1781
|
-
return (await listingPass(ref, deps, { keep, protect: list, quiet: true, leases })).deleted;
|
|
1782
|
-
}
|
|
1783
|
-
async function reconcile(ref, deps, options) {
|
|
1784
|
-
const { keep, kept, leases } = options;
|
|
1785
|
-
const {
|
|
1786
|
-
deleted,
|
|
1787
|
-
kept: present,
|
|
1788
|
-
complete
|
|
1789
|
-
} = await listingPass(ref, deps, { keep, quiet: true, leases });
|
|
1790
|
-
const token = kept?.token;
|
|
1791
|
-
if (complete && token !== void 0 && keep <= MAX_KEPT_GENERATIONS && !sameList(kept?.list, present)) {
|
|
1792
|
-
try {
|
|
1793
|
-
await deps.registry.compareAndSwap(ref, token, { keptGens: present });
|
|
1794
|
-
} catch (err) {
|
|
1795
|
-
if (!isWriteConflictError(err)) throw err;
|
|
1796
|
-
}
|
|
1797
|
-
}
|
|
1798
|
-
return deleted;
|
|
1799
|
-
}
|
|
1800
|
-
function sameList(a, b) {
|
|
1801
|
-
return a !== void 0 && a.length === b.length && a.every((g, i) => g === b[i]);
|
|
1802
|
-
}
|
|
1803
|
-
async function objectIsThere(storage, key) {
|
|
1804
|
-
try {
|
|
1805
|
-
await storage.getTail(key, 0);
|
|
1806
|
-
return true;
|
|
1807
|
-
} catch {
|
|
1808
|
-
return false;
|
|
1809
|
-
}
|
|
1810
|
-
}
|
|
1811
|
-
async function deleteEvicted(ref, deps, options) {
|
|
1812
|
-
const deleted = [];
|
|
1813
|
-
for (const generation of options.evict) {
|
|
1814
|
-
const still = await deps.registry.get(ref);
|
|
1815
|
-
if (still === null || still.currentGen === null || still.currentGen < options.generation) break;
|
|
1816
|
-
const named = usableKeptGens(still);
|
|
1817
|
-
if (named === void 0 || generation >= still.currentGen || named.includes(generation)) break;
|
|
1818
|
-
if (options.leases !== void 0 && heldGenerations(still, options.leases.now()).has(generation)) {
|
|
1819
|
-
continue;
|
|
1820
|
-
}
|
|
1821
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
1822
|
-
deleted.push(generation);
|
|
1823
|
-
}
|
|
1824
|
-
return deleted;
|
|
1825
|
-
}
|
|
1826
|
-
|
|
1827
|
-
// src/core/reader-defaults.ts
|
|
1828
|
-
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
1829
|
-
var REFRESH_RETRY_MS = 500;
|
|
1830
|
-
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
1831
|
-
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
1832
|
-
|
|
1833
|
-
// src/core/cost.ts
|
|
1834
|
-
function deepFreeze(value) {
|
|
1835
|
-
if (value !== null && typeof value === "object" && !Object.isFrozen(value)) {
|
|
1836
|
-
Object.freeze(value);
|
|
1837
|
-
for (const inner of Object.values(value)) deepFreeze(inner);
|
|
1838
|
-
}
|
|
1839
|
-
return value;
|
|
1840
|
-
}
|
|
1841
|
-
var ELASTICACHE_REDIS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
1842
|
-
source: "ElastiCache for Redis OSS, us-east-1 on-demand, AWS price list 20260914063714",
|
|
1843
|
-
replicasPerShard: 2,
|
|
1844
|
-
reservedMemoryFraction: 0.25,
|
|
1845
|
-
nodeTypes: [
|
|
1846
|
-
{ name: "cache.t4g.micro", memoryGiB: 0.5, hourlyUSD: 0.016, maxShards: 1 },
|
|
1847
|
-
{ name: "cache.t4g.small", memoryGiB: 1.37, hourlyUSD: 0.032, maxShards: 1 },
|
|
1848
|
-
{ name: "cache.t4g.medium", memoryGiB: 3.09, hourlyUSD: 0.065, maxShards: 1 },
|
|
1849
|
-
{ name: "cache.m6g.large", memoryGiB: 6.38, hourlyUSD: 0.149 },
|
|
1850
|
-
{ name: "cache.r6g.large", memoryGiB: 13.07, hourlyUSD: 0.206 },
|
|
1851
|
-
{ name: "cache.r6g.xlarge", memoryGiB: 26.32, hourlyUSD: 0.411 },
|
|
1852
|
-
{ name: "cache.r6g.2xlarge", memoryGiB: 52.82, hourlyUSD: 0.821 },
|
|
1853
|
-
{ name: "cache.r6g.4xlarge", memoryGiB: 105.81, hourlyUSD: 1.642 },
|
|
1854
|
-
{ name: "cache.r6g.8xlarge", memoryGiB: 209.55, hourlyUSD: 3.284 },
|
|
1855
|
-
{ name: "cache.r6g.12xlarge", memoryGiB: 317.77, hourlyUSD: 4.925 },
|
|
1856
|
-
{ name: "cache.r6g.16xlarge", memoryGiB: 419.09, hourlyUSD: 6.567 },
|
|
1857
|
-
{ name: "cache.r6gd.xlarge", memoryGiB: 26.32, ssdGiB: 99.33, hourlyUSD: 0.781 },
|
|
1858
|
-
{ name: "cache.r6gd.2xlarge", memoryGiB: 52.82, ssdGiB: 199.07, hourlyUSD: 1.56 },
|
|
1859
|
-
{ name: "cache.r6gd.4xlarge", memoryGiB: 105.81, ssdGiB: 398.14, hourlyUSD: 3.12 },
|
|
1860
|
-
{ name: "cache.r6gd.8xlarge", memoryGiB: 209.55, ssdGiB: 796.28, hourlyUSD: 6.24 },
|
|
1861
|
-
{ name: "cache.r6gd.12xlarge", memoryGiB: 317.77, ssdGiB: 1194.42, hourlyUSD: 9.358 },
|
|
1862
|
-
{ name: "cache.r6gd.16xlarge", memoryGiB: 419.09, ssdGiB: 1592.56, hourlyUSD: 12.477 }
|
|
1863
|
-
]
|
|
1864
|
-
});
|
|
1865
|
-
var ONE_REDIS_HA_CLUSTER = deepFreeze({ monthlyUSD: 346 });
|
|
1866
|
-
var AWS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
1867
|
-
name: "aws-us-east-1-ondemand",
|
|
1868
|
-
storage: { getPerMillion: 0.4, putPerMillion: 5, storagePerGiBMonth: 0.023 },
|
|
1869
|
-
redis: { sizedToData: ELASTICACHE_REDIS_US_EAST_1_ONDEMAND }
|
|
1870
|
-
});
|
|
1871
|
-
var HOURS_PER_MONTH = 730;
|
|
1872
|
-
var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
|
|
1873
|
-
var GIB = 1024 ** 3;
|
|
1874
|
-
var STORE_LOAD_PUT_CLASS = 1 + 1 / LIST_COLLECTION_CADENCE;
|
|
1875
|
-
var STORE_LOAD_POINTER_READS = 2 + 2 / LIST_COLLECTION_CADENCE;
|
|
1876
|
-
var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
|
|
1877
|
-
var RETENTION_SWEEP_REQUESTS = deepFreeze({
|
|
1878
|
-
conditionalDelete: {
|
|
1879
|
-
retirement: { reads: 7, writes: 2, deletes: 1 },
|
|
1880
|
-
purge: { reads: 4, writes: 0, deletes: 2 }
|
|
1881
|
-
},
|
|
1882
|
-
tombstoning: {
|
|
1883
|
-
retirement: { reads: 6, writes: 2, deletes: 0 },
|
|
1884
|
-
purge: { reads: 3, writes: 1, deletes: 0 }
|
|
1885
|
-
}
|
|
1886
|
-
});
|
|
1887
|
-
var shown = (n) => String(Number(n.toFixed(2)));
|
|
1888
|
-
function requireFiniteNonNeg(n, field) {
|
|
1889
|
-
if (n === void 0 || !Number.isFinite(n) || n < 0) {
|
|
1890
|
-
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
1891
|
-
}
|
|
1892
|
-
return n;
|
|
1893
|
-
}
|
|
1894
|
-
function clamp01(n) {
|
|
1895
|
-
return n < 0 ? 0 : n > 1 ? 1 : n;
|
|
1896
|
-
}
|
|
1897
|
-
var TIE_USD = 1e-6;
|
|
1898
|
-
function usableGiB(sizing, nodeType) {
|
|
1899
|
-
return nodeType.memoryGiB * (1 - sizing.reservedMemoryFraction) + (nodeType.ssdGiB ?? 0);
|
|
1900
|
-
}
|
|
1901
|
-
function validateSizing(sizing) {
|
|
1902
|
-
const at = "pricing.redis.sizedToData";
|
|
1903
|
-
if (sizing === null || typeof sizing !== "object") {
|
|
1904
|
-
throw new ValidationError(`${at} must be a RedisSizing`);
|
|
1905
|
-
}
|
|
1906
|
-
if (typeof sizing.source !== "string" || sizing.source === "") {
|
|
1907
|
-
throw new ValidationError(`${at}.source must say where the prices come from`);
|
|
1908
|
-
}
|
|
1909
|
-
if (!Array.isArray(sizing.nodeTypes) || sizing.nodeTypes.length === 0) {
|
|
1910
|
-
throw new ValidationError(`${at}.nodeTypes must list at least one node type`);
|
|
1911
|
-
}
|
|
1912
|
-
if (!Number.isInteger(sizing.replicasPerShard) || sizing.replicasPerShard < 0) {
|
|
1913
|
-
throw new ValidationError(
|
|
1914
|
-
`${at}.replicasPerShard must be an integer >= 0; got ${sizing.replicasPerShard}`
|
|
1915
|
-
);
|
|
1916
|
-
}
|
|
1917
|
-
const reserved = requireFiniteNonNeg(
|
|
1918
|
-
sizing.reservedMemoryFraction,
|
|
1919
|
-
`${at}.reservedMemoryFraction`
|
|
1920
|
-
);
|
|
1921
|
-
if (reserved >= 1) {
|
|
1922
|
-
throw new ValidationError(`${at}.reservedMemoryFraction must be below 1; got ${reserved}`);
|
|
1923
|
-
}
|
|
1924
|
-
const names = /* @__PURE__ */ new Set();
|
|
1925
|
-
for (const n of sizing.nodeTypes) {
|
|
1926
|
-
if (n === null || typeof n !== "object" || typeof n.name !== "string" || n.name === "") {
|
|
1927
|
-
throw new ValidationError(`${at}.nodeTypes: every node type needs a name`);
|
|
1928
|
-
}
|
|
1929
|
-
const row = `${at}.nodeTypes[${JSON.stringify(n.name)}]`;
|
|
1930
|
-
if (names.has(n.name)) throw new ValidationError(`${row} is listed twice`);
|
|
1931
|
-
names.add(n.name);
|
|
1932
|
-
if (!(requireFiniteNonNeg(n.memoryGiB, `${row}.memoryGiB`) > 0)) {
|
|
1933
|
-
throw new ValidationError(`${row}.memoryGiB must be above 0`);
|
|
1934
|
-
}
|
|
1935
|
-
requireFiniteNonNeg(n.ssdGiB ?? 0, `${row}.ssdGiB`);
|
|
1936
|
-
requireFiniteNonNeg(n.hourlyUSD, `${row}.hourlyUSD`);
|
|
1937
|
-
if (n.maxShards !== void 0 && (!Number.isInteger(n.maxShards) || n.maxShards < 1)) {
|
|
1938
|
-
throw new ValidationError(`${row}.maxShards must be an integer >= 1; got ${n.maxShards}`);
|
|
1939
|
-
}
|
|
1940
|
-
if (!(usableGiB(sizing, n) > 0)) {
|
|
1941
|
-
throw new ValidationError(
|
|
1942
|
-
`${row} leaves no memory for data once ${reserved} of it is reserved`
|
|
1943
|
-
);
|
|
1944
|
-
}
|
|
1945
|
-
}
|
|
1946
|
-
}
|
|
1947
|
-
function cheapestCluster(sizing, dataGiB) {
|
|
1948
|
-
const fits = [];
|
|
1949
|
-
for (const nodeType of sizing.nodeTypes) {
|
|
1950
|
-
const usable = usableGiB(sizing, nodeType);
|
|
1951
|
-
let shards = Math.max(1, Math.ceil(dataGiB / usable));
|
|
1952
|
-
if (shards > 1 && (shards - 1) * usable >= dataGiB) shards -= 1;
|
|
1953
|
-
if (nodeType.maxShards !== void 0 && shards > nodeType.maxShards) continue;
|
|
1954
|
-
const nodes = shards * (1 + sizing.replicasPerShard);
|
|
1955
|
-
fits.push({
|
|
1956
|
-
nodeType,
|
|
1957
|
-
shards,
|
|
1958
|
-
nodes,
|
|
1959
|
-
monthlyUSD: nodes * nodeType.hourlyUSD * HOURS_PER_MONTH
|
|
1960
|
-
});
|
|
1961
|
-
}
|
|
1962
|
-
const size = bytesWords(dataGiB * GIB);
|
|
1963
|
-
if (fits.length === 0) {
|
|
1964
|
-
throw new ValidationError(
|
|
1965
|
-
`no node type in pricing.redis.sizedToData.nodeTypes holds ${size} within its maxShards`
|
|
1966
|
-
);
|
|
1967
|
-
}
|
|
1968
|
-
const cheapest = fits.reduce((m, c) => Math.min(m, c.monthlyUSD), Infinity);
|
|
1969
|
-
const [best] = fits.filter((c) => c.monthlyUSD <= cheapest + TIE_USD).sort(
|
|
1970
|
-
(a, b) => a.nodes - b.nodes || a.monthlyUSD - b.monthlyUSD || (a.nodeType.name < b.nodeType.name ? -1 : a.nodeType.name > b.nodeType.name ? 1 : 0)
|
|
1971
|
-
);
|
|
1972
|
-
if (best === void 0 || !Number.isFinite(best.monthlyUSD)) {
|
|
1973
|
-
throw new ValidationError(
|
|
1974
|
-
`pricing.redis.sizedToData prices ${size} at no finite cost; check its node types`
|
|
1975
|
-
);
|
|
1976
|
-
}
|
|
1977
|
-
return best;
|
|
1978
|
-
}
|
|
1979
|
-
function snapshotSizing(sizing) {
|
|
1980
|
-
if (sizing === null || typeof sizing !== "object") return sizing;
|
|
1981
|
-
const { source, nodeTypes, replicasPerShard, reservedMemoryFraction } = sizing;
|
|
1982
|
-
const row = (n) => {
|
|
1983
|
-
if (n === null || typeof n !== "object") return n;
|
|
1984
|
-
const { name, memoryGiB, ssdGiB, hourlyUSD, maxShards } = n;
|
|
1985
|
-
return {
|
|
1986
|
-
name,
|
|
1987
|
-
memoryGiB,
|
|
1988
|
-
hourlyUSD,
|
|
1989
|
-
...ssdGiB === void 0 ? {} : { ssdGiB },
|
|
1990
|
-
...maxShards === void 0 ? {} : { maxShards }
|
|
1991
|
-
};
|
|
1992
|
-
};
|
|
1993
|
-
return {
|
|
1994
|
-
source,
|
|
1995
|
-
replicasPerShard,
|
|
1996
|
-
reservedMemoryFraction,
|
|
1997
|
-
nodeTypes: Array.isArray(nodeTypes) ? Array.from(nodeTypes, row) : nodeTypes
|
|
1998
|
-
};
|
|
1999
|
-
}
|
|
2000
|
-
function resolveBaseline(redis, storedGiB) {
|
|
2001
|
-
if (redis === null || typeof redis !== "object") {
|
|
2002
|
-
throw new ValidationError("pricing.redis must be { monthlyUSD } or { sizedToData }");
|
|
2003
|
-
}
|
|
2004
|
-
const { monthlyUSD, sizedToData } = redis;
|
|
2005
|
-
const fixed = monthlyUSD !== void 0;
|
|
2006
|
-
const sized = sizedToData !== void 0;
|
|
2007
|
-
if (fixed === sized) {
|
|
2008
|
-
throw new ValidationError(
|
|
2009
|
-
"pricing.redis must be exactly one of { monthlyUSD } and { sizedToData }" + (fixed ? ", not both: spreading AWS_US_EAST_1_ONDEMAND.redis keeps its sizedToData, so pass redis: { monthlyUSD } alone" : "")
|
|
2010
|
-
);
|
|
2011
|
-
}
|
|
2012
|
-
if (sized) {
|
|
2013
|
-
const sizing = snapshotSizing(sizedToData);
|
|
2014
|
-
validateSizing(sizing);
|
|
2015
|
-
const cluster = cheapestCluster(sizing, storedGiB);
|
|
2016
|
-
return { monthlyUSD: cluster.monthlyUSD, sized: { sizing, cluster } };
|
|
2017
|
-
}
|
|
2018
|
-
return { monthlyUSD: requireFiniteNonNeg(monthlyUSD, "pricing.redis.monthlyUSD") };
|
|
2019
|
-
}
|
|
2020
|
-
function bytesWords(bytes) {
|
|
2021
|
-
const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
|
|
2022
|
-
let value = bytes;
|
|
2023
|
-
let unit = 0;
|
|
2024
|
-
const shown2 = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
|
|
2025
|
-
while (unit < units.length - 1 && shown2(value, unit) >= 1024) {
|
|
2026
|
-
value /= 1024;
|
|
2027
|
-
unit += 1;
|
|
2028
|
-
}
|
|
2029
|
-
if (unit > 0) return `${value.toFixed(2)} ${units[unit]}`;
|
|
2030
|
-
const n = Math.round(value);
|
|
2031
|
-
return `${n} byte${n === 1 ? "" : "s"}`;
|
|
2032
|
-
}
|
|
2033
|
-
function clusterWords(c) {
|
|
2034
|
-
const count = (n) => n.toLocaleString("en-US");
|
|
2035
|
-
const nodes = `${count(c.nodes)} ${c.nodeType.name} node${c.nodes === 1 ? "" : "s"}`;
|
|
2036
|
-
return c.shards === 1 ? `1 shard of ${nodes}` : `${count(c.shards)} shards, ${nodes}`;
|
|
2037
|
-
}
|
|
2038
|
-
function sizingBytes(spec) {
|
|
2039
|
-
if (spec.sizeBytes !== void 0) return requireFiniteNonNeg(spec.sizeBytes, "segment.sizeBytes");
|
|
2040
|
-
if (spec.cardinality !== void 0) {
|
|
2041
|
-
return requireFiniteNonNeg(spec.cardinality, "segment.cardinality") * 2;
|
|
2042
|
-
}
|
|
2043
|
-
return 0;
|
|
2044
|
-
}
|
|
2045
|
-
function buildReport(input) {
|
|
2046
|
-
const { pricing, grounded } = input;
|
|
2047
|
-
const { storage, redis } = pricing;
|
|
2048
|
-
const S = SECONDS_PER_MONTH;
|
|
2049
|
-
requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
|
|
2050
|
-
requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
|
|
2051
|
-
requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
|
|
2052
|
-
const pointerRead = requireFiniteNonNeg(
|
|
2053
|
-
storage.requestsPerPointerRead ?? 1,
|
|
2054
|
-
"pricing.storage.requestsPerPointerRead"
|
|
2055
|
-
);
|
|
2056
|
-
const sizedRead = requireFiniteNonNeg(
|
|
2057
|
-
storage.requestsPerSizedRead ?? 1,
|
|
2058
|
-
"pricing.storage.requestsPerSizedRead"
|
|
2059
|
-
);
|
|
2060
|
-
const storageBytes = requireFiniteNonNeg(input.storageBytes, "storageBytes");
|
|
2061
|
-
const storedGiB = storageBytes / GIB;
|
|
2062
|
-
const baseline = resolveBaseline(redis, storedGiB);
|
|
2063
|
-
const baselineUSD = baseline.monthlyUSD;
|
|
2064
|
-
if ((input.workload.cacheHitRate ?? 0) > 1) {
|
|
2065
|
-
throw new ValidationError(
|
|
2066
|
-
`cacheHitRate must be a fraction from 0 to 1; got ${String(input.workload.cacheHitRate)}`
|
|
2067
|
-
);
|
|
2068
|
-
}
|
|
2069
|
-
const cacheHitRate = clamp01(
|
|
2070
|
-
requireFiniteNonNeg(input.workload.cacheHitRate ?? 0, "cacheHitRate")
|
|
2071
|
-
);
|
|
2072
|
-
const readsPerSec = requireFiniteNonNeg(input.workload.readsPerSec ?? 0, "readsPerSec");
|
|
2073
|
-
const intersectsPerSec = requireFiniteNonNeg(
|
|
2074
|
-
input.workload.intersectsPerSec ?? 0,
|
|
2075
|
-
"intersectsPerSec"
|
|
2076
|
-
);
|
|
2077
|
-
const chunksPerIntersect = requireFiniteNonNeg(
|
|
2078
|
-
input.workload.chunksPerIntersect ?? 1,
|
|
2079
|
-
"chunksPerIntersect"
|
|
2080
|
-
);
|
|
2081
|
-
const loadsPerMonth = requireFiniteNonNeg(input.workload.loadsPerMonth ?? 0, "loadsPerMonth");
|
|
2082
|
-
const requestsPerLoad = requireFiniteNonNeg(
|
|
2083
|
-
input.workload.requestsPerLoad ?? 1,
|
|
2084
|
-
"requestsPerLoad"
|
|
2085
|
-
);
|
|
2086
|
-
const operandsPerIntersect = requireFiniteNonNeg(
|
|
2087
|
-
input.workload.operandsPerIntersect ?? 2,
|
|
2088
|
-
"operandsPerIntersect"
|
|
2089
|
-
);
|
|
2090
|
-
if (operandsPerIntersect < 1) {
|
|
2091
|
-
throw new ValidationError(
|
|
2092
|
-
`operandsPerIntersect must be at least 1, since an intersection reads its operands; got ${operandsPerIntersect}`
|
|
2093
|
-
);
|
|
2094
|
-
}
|
|
2095
|
-
const hotSegments = requireFiniteNonNeg(input.workload.hotSegments ?? 0, "hotSegments");
|
|
2096
|
-
const readerProcesses = requireFiniteNonNeg(
|
|
2097
|
-
input.workload.readerProcesses ?? 1,
|
|
2098
|
-
"readerProcesses"
|
|
2099
|
-
);
|
|
2100
|
-
if (readerProcesses < 1) {
|
|
2101
|
-
throw new ValidationError(`readerProcesses must be at least 1; got ${readerProcesses}`);
|
|
2102
|
-
}
|
|
2103
|
-
const genTtlMs = requireFiniteNonNeg(
|
|
2104
|
-
input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
|
|
2105
|
-
"genTtlMs"
|
|
2106
|
-
);
|
|
2107
|
-
const retirementsPerMonth = requireFiniteNonNeg(
|
|
2108
|
-
input.workload.retirementsPerMonth ?? 0,
|
|
2109
|
-
"retirementsPerMonth"
|
|
2110
|
-
);
|
|
2111
|
-
const purgesPerMonth = requireFiniteNonNeg(input.workload.purgesPerMonth ?? 0, "purgesPerMonth");
|
|
2112
|
-
const conditionalDelete = input.workload.conditionalDelete ?? true;
|
|
2113
|
-
if (typeof conditionalDelete !== "boolean") {
|
|
2114
|
-
throw new ValidationError(`conditionalDelete must be a boolean; got ${conditionalDelete}`);
|
|
2115
|
-
}
|
|
2116
|
-
const storageGetUSD = storage.getPerMillion / 1e6;
|
|
2117
|
-
const putUSD = storage.putPerMillion / 1e6;
|
|
2118
|
-
const missFraction = 1 - cacheHitRate;
|
|
2119
|
-
const readMisses = readsPerSec * S * missFraction;
|
|
2120
|
-
const intersects = intersectsPerSec * S;
|
|
2121
|
-
const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
|
|
2122
|
-
const storageUSD = storedGiB * storage.storagePerGiBMonth;
|
|
2123
|
-
const readsUSD = readMisses * storageGetUSD;
|
|
2124
|
-
const getsPerColdOperand = pointerRead + sizedRead;
|
|
2125
|
-
const intersectGets = chunksPerIntersect + getsPerColdOperand * operandsPerIntersect;
|
|
2126
|
-
const intersectsUSD = intersects * intersectGets * storageGetUSD;
|
|
2127
|
-
const loadGets = STORE_LOAD_POINTER_READS * pointerRead + STORE_LOAD_EXISTENCE_CHECKS;
|
|
2128
|
-
const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
|
|
2129
|
-
const refreshUSD = refreshes * pointerRead * storageGetUSD;
|
|
2130
|
-
const sweep = RETENTION_SWEEP_REQUESTS[conditionalDelete ? "conditionalDelete" : "tombstoning"];
|
|
2131
|
-
const retentionUSD = retirementsPerMonth * (sweep.retirement.reads * storageGetUSD + sweep.retirement.writes * putUSD) + purgesPerMonth * (sweep.purge.reads * storageGetUSD + sweep.purge.writes * putUSD);
|
|
2132
|
-
const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD + retentionUSD;
|
|
2133
|
-
const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
|
|
2134
|
-
const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
|
|
2135
|
-
const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
|
|
2136
|
-
const dominant = Math.max(
|
|
2137
|
-
readsUSD,
|
|
2138
|
-
intersectsUSD,
|
|
2139
|
-
storageUSD,
|
|
2140
|
-
loadsUSD,
|
|
2141
|
-
refreshUSD,
|
|
2142
|
-
retentionUSD
|
|
2143
|
-
);
|
|
2144
|
-
let driver = "storage";
|
|
2145
|
-
if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
|
|
2146
|
-
else if (dominant === intersectsUSD && intersectsUSD > 0) {
|
|
2147
|
-
driver = "intersections (pointer, index and chunk GETs)";
|
|
2148
|
-
} else if (dominant === loadsUSD && loadsUSD > 0)
|
|
2149
|
-
driver = "loads (object, listing and pointer requests)";
|
|
2150
|
-
else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
|
|
2151
|
-
else if (dominant === retentionUSD && retentionUSD > 0)
|
|
2152
|
-
driver = "the retention sweep (registry reads and writes)";
|
|
2153
|
-
const sized = baseline.sized;
|
|
2154
|
-
const noBytes = Math.round(storageBytes) === 0;
|
|
2155
|
-
const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
|
|
2156
|
-
const rationale = verdict === "lose-zone" ? `pay-per-use total $${total.toFixed(2)}/mo exceeds the ${redisWords ?? `$${baselineUSD}/mo flat baseline`}; dominated by ${driver}` : verdict === "win-big" ? `$${total.toFixed(2)}/mo \u2014 \u226410% of the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}` : `$${total.toFixed(2)}/mo, under the ${redisWords ?? `$${baselineUSD}/mo baseline`}; dominated by ${driver}`;
|
|
2157
|
-
const dataTiering = sized !== void 0 && (sized.cluster.nodeType.ssdGiB ?? 0) > 0;
|
|
2158
|
-
const redisNote = sized === void 0 ? `Redis priced at the fixed $${baselineUSD}/mo given, whatever the data size.` : (noBytes ? "Redis priced at the cheapest cluster in the catalogue, as no bytes are counted, because nothing is stored or nothing was measured" : `Redis priced at the cheapest cluster in the catalogue that holds the ${bytesWords(storageBytes)} stored`) + `: ${clusterWords(sized.cluster)}, each shard a primary and ${sized.sizing.replicasPerShard} replica(s), ${Number((sized.sizing.reservedMemoryFraction * 100).toFixed(1))}% of memory reserved (${sized.sizing.source}).` + (noBytes ? " Pass pricing.redis: { monthlyUSD } to compare against a Redis of your own." : " Compressed bytes are a floor on the memory Redis needs: a native Redis bitmap is sized by its highest id, so sparse ids need more.") + " Redis bought another way \u2014 fewer replicas, another engine, reserved nodes \u2014 can cost less than the catalogue prices it." + (dataTiering ? ` ${sized.cluster.nodeType.name} is a data-tiering node, which keeps the values read least recently on its SSD. It is priced as if the data read often fits in its memory: AWS recommends data tiering for workloads that regularly read up to 20% of their data.` : "") + (input.sizesFromCardinality ? " Some segment sizes came from cardinality, at 2 bytes an id, which is more than a dense bitmap needs, so this Redis may be larger than the data calls for: pass sizeBytes to size it to the compressed bytes." : "");
|
|
2159
|
-
const notes = [
|
|
2160
|
-
"Same-region egress treated as free; internet egress is not modeled.",
|
|
2161
|
-
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
2162
|
-
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${shown(STORE_LOAD_PUT_CLASS)} PUT-class and ${shown(loadGets)} GETs that store.load() adds (the pointer, a check that the next generation number is free, a check that the current generation is there, and a listing every ${LIST_COLLECTION_CADENCE}th load).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
|
|
2163
|
-
...intersects > 0 ? [
|
|
2164
|
-
`Intersections priced cold: ${intersectGets} GETs each, ${getsPerColdOperand} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk range request(s); cacheHitRate does not apply.`
|
|
2165
|
-
] : [],
|
|
2166
|
-
hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
|
|
2167
|
-
hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
|
|
2168
|
-
...retirementsPerMonth > 0 || purgesPerMonth > 0 ? [
|
|
2169
|
-
`Retention sweep modeled: ${retirementsPerMonth}/mo retirements at ${sweep.retirement.reads} reads, ${sweep.retirement.writes} writes and ${sweep.retirement.deletes} delete(s) each, ${purgesPerMonth}/mo purges at ${sweep.purge.reads} reads, ${sweep.purge.writes} writes and ${sweep.purge.deletes} delete(s) each; reads are priced at the GET rate and writes at the PUT rate, deletes at nothing, as S3 bills none` + (conditionalDelete ? "." : "; a registry that only tombstones is read again, two reads per purged segment, by every later full sweep, which is not priced.")
|
|
2170
|
-
] : [],
|
|
2171
|
-
...input.extraNotes ?? [],
|
|
2172
|
-
// Last, so the notes a report already carried keep their places.
|
|
2173
|
-
redisNote
|
|
2174
|
-
].filter((n) => n !== null);
|
|
2175
|
-
return {
|
|
2176
|
-
monthlyUSD: {
|
|
2177
|
-
byOp: {
|
|
2178
|
-
reads: readsUSD,
|
|
2179
|
-
intersects: intersectsUSD,
|
|
2180
|
-
storage: storageUSD,
|
|
2181
|
-
loads: loadsUSD,
|
|
2182
|
-
pointerRefresh: refreshUSD,
|
|
2183
|
-
retention: retentionUSD
|
|
2184
|
-
},
|
|
2185
|
-
total
|
|
2186
|
-
},
|
|
2187
|
-
redisBaseline: sized === void 0 ? { basis: "fixed", monthlyUSD: baselineUSD } : {
|
|
2188
|
-
basis: "sized-to-data",
|
|
2189
|
-
monthlyUSD: baselineUSD,
|
|
2190
|
-
cluster: {
|
|
2191
|
-
nodeType: sized.cluster.nodeType.name,
|
|
2192
|
-
shards: sized.cluster.shards,
|
|
2193
|
-
nodes: sized.cluster.nodes,
|
|
2194
|
-
dataTiering
|
|
2195
|
-
}
|
|
2196
|
-
},
|
|
2197
|
-
redisCrossover: { readsPerSec: readsCross },
|
|
2198
|
-
verdict,
|
|
2199
|
-
rationale,
|
|
2200
|
-
assumptions: { cacheHitRate, pricingName: pricing.name, grounded, notes }
|
|
2201
|
-
};
|
|
2202
|
-
}
|
|
2203
|
-
function checkModelInputs(op, pricing, workload) {
|
|
2204
|
-
const storage = pricing !== null && typeof pricing === "object" ? pricing.storage : void 0;
|
|
2205
|
-
if (pricing !== void 0 && (storage === null || typeof storage !== "object")) {
|
|
2206
|
-
throw new ValidationError(`${op}: pricing must be a profile with a storage price list`);
|
|
2207
|
-
}
|
|
2208
|
-
if (workload !== void 0 && workload !== null && (typeof workload !== "object" || Array.isArray(workload))) {
|
|
2209
|
-
throw new ValidationError(`${op}: workload must be an object`);
|
|
2210
|
-
}
|
|
2211
|
-
}
|
|
2212
|
-
function estimateCost(input) {
|
|
2213
|
-
if (input === null || typeof input !== "object") {
|
|
2214
|
-
throw new ValidationError("estimateCost: input must be an object such as { segments }");
|
|
2215
|
-
}
|
|
2216
|
-
if (!Array.isArray(input.segments)) {
|
|
2217
|
-
throw new ValidationError(
|
|
2218
|
-
"estimateCost: segments must be an array of { sizeBytes } or { cardinality }"
|
|
2219
|
-
);
|
|
2220
|
-
}
|
|
2221
|
-
for (let i = 0; i < input.segments.length; i++) {
|
|
2222
|
-
const spec = input.segments[i];
|
|
2223
|
-
if (spec === null || typeof spec !== "object") {
|
|
2224
|
-
throw new ValidationError(`estimateCost: segments[${i}] must be an object`);
|
|
2225
|
-
}
|
|
2226
|
-
}
|
|
2227
|
-
checkModelInputs("estimateCost", input.pricing, input.workload);
|
|
2228
|
-
const pricing = input.pricing ?? AWS_US_EAST_1_ONDEMAND;
|
|
2229
|
-
const workload = input.workload ?? {};
|
|
2230
|
-
let storageBytes = 0;
|
|
2231
|
-
let sizesFromCardinality = false;
|
|
2232
|
-
for (const spec of input.segments) {
|
|
2233
|
-
const count = Math.floor(requireFiniteNonNeg(spec.count ?? 1, "segment.count"));
|
|
2234
|
-
storageBytes += sizingBytes(spec) * count;
|
|
2235
|
-
if (spec.sizeBytes === void 0 && spec.cardinality !== void 0 && sizingBytes(spec) * count > 0) {
|
|
2236
|
-
sizesFromCardinality = true;
|
|
2237
|
-
}
|
|
2238
|
-
}
|
|
2239
|
-
return buildReport({ storageBytes, workload, pricing, grounded: false, sizesFromCardinality });
|
|
2240
|
-
}
|
|
2241
|
-
function groundedReport(input) {
|
|
2242
|
-
checkModelInputs("costReport", input.pricing, input.workload);
|
|
2243
|
-
return buildReport({
|
|
2244
|
-
storageBytes: input.storageBytes,
|
|
2245
|
-
workload: input.workload ?? {},
|
|
2246
|
-
pricing: input.pricing ?? AWS_US_EAST_1_ONDEMAND,
|
|
2247
|
-
grounded: input.grounded ?? true,
|
|
2248
|
-
extraNotes: input.extraNotes
|
|
2249
|
-
});
|
|
2250
|
-
}
|
|
2251
|
-
|
|
2252
1675
|
// src/core/due-index.ts
|
|
2253
1676
|
var DUE_BUCKET_MS = 864e5;
|
|
2254
1677
|
function dueBucket(expiresAt) {
|
|
@@ -2371,6 +1794,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2371
1794
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
2372
1795
|
const segments = [];
|
|
2373
1796
|
const failed = [];
|
|
1797
|
+
const skipped = [];
|
|
2374
1798
|
let totalIds = 0;
|
|
2375
1799
|
if (typeof reader?.segment !== "function") {
|
|
2376
1800
|
throw new ValidationError(
|
|
@@ -2447,10 +1871,21 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2447
1871
|
};
|
|
2448
1872
|
for await (const rec of registry.list(options.namespace)) {
|
|
2449
1873
|
if (options.namespace === void 0 && isReservedRow(rec)) continue;
|
|
2450
|
-
if (rec.status === "destroyed")
|
|
1874
|
+
if (rec.status === "destroyed") {
|
|
1875
|
+
skipped.push({ segment: rec.segment, namespace: rec.namespace, reason: "destroyed" });
|
|
1876
|
+
continue;
|
|
1877
|
+
}
|
|
2451
1878
|
await exportRef({ segment: rec.segment, namespace: rec.namespace });
|
|
2452
1879
|
}
|
|
2453
|
-
return {
|
|
1880
|
+
return {
|
|
1881
|
+
version: 1,
|
|
1882
|
+
format,
|
|
1883
|
+
totalSegments: segments.length,
|
|
1884
|
+
totalIds,
|
|
1885
|
+
segments,
|
|
1886
|
+
failed,
|
|
1887
|
+
skipped
|
|
1888
|
+
};
|
|
2454
1889
|
}
|
|
2455
1890
|
|
|
2456
1891
|
// src/core/discover.ts
|
|
@@ -2564,7 +1999,7 @@ var MemoryRegistryDriver = class {
|
|
|
2564
1999
|
if (!existing || existing.token !== expected) {
|
|
2565
2000
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2566
2001
|
}
|
|
2567
|
-
const token = this.nextToken(incarnationOf(existing.token)
|
|
2002
|
+
const token = this.nextToken(incarnationOf(existing.token));
|
|
2568
2003
|
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
2569
2004
|
return { token };
|
|
2570
2005
|
}
|
|
@@ -2585,8 +2020,14 @@ var MemoryRegistryDriver = class {
|
|
|
2585
2020
|
};
|
|
2586
2021
|
var MemoryStorageDriver = class {
|
|
2587
2022
|
objects = /* @__PURE__ */ new Map();
|
|
2023
|
+
stored = 0;
|
|
2588
2024
|
capabilities() {
|
|
2589
|
-
return {
|
|
2025
|
+
return {
|
|
2026
|
+
rangeRead: true,
|
|
2027
|
+
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2028
|
+
conditionalPut: true,
|
|
2029
|
+
conditionalDelete: true
|
|
2030
|
+
};
|
|
2590
2031
|
}
|
|
2591
2032
|
async putImmutable(key, write) {
|
|
2592
2033
|
const k = genObjectKey(key);
|
|
@@ -2598,14 +2039,15 @@ var MemoryStorageDriver = class {
|
|
|
2598
2039
|
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
2599
2040
|
);
|
|
2600
2041
|
}
|
|
2601
|
-
this.
|
|
2042
|
+
this.stored += 1;
|
|
2043
|
+
this.objects.set(k, { body, version: String(this.stored) });
|
|
2602
2044
|
return { size: body.length, sha256: createHash("sha256").update(body).digest("hex") };
|
|
2603
2045
|
}
|
|
2604
2046
|
async getRange(key, offset, length) {
|
|
2605
2047
|
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
2606
2048
|
throw new ValidationError(`invalid range offset=${offset} length=${length}`);
|
|
2607
2049
|
}
|
|
2608
|
-
const body = this.require(key);
|
|
2050
|
+
const { body } = this.require(key);
|
|
2609
2051
|
if (offset + length > body.length) {
|
|
2610
2052
|
throw new ValidationError(
|
|
2611
2053
|
`range [${offset}, ${offset + length}) out of bounds for ${body.length}B`
|
|
@@ -2614,15 +2056,23 @@ var MemoryStorageDriver = class {
|
|
|
2614
2056
|
return body.slice(offset, offset + length);
|
|
2615
2057
|
}
|
|
2616
2058
|
async getTail(key, maxBytes) {
|
|
2617
|
-
const body = this.require(key);
|
|
2059
|
+
const { body, version } = this.require(key);
|
|
2618
2060
|
const take = Math.min(Math.max(maxBytes, 0), body.length);
|
|
2619
|
-
return { bytes: body.slice(body.length - take), size: body.length };
|
|
2620
|
-
}
|
|
2621
|
-
async delete(key) {
|
|
2622
|
-
this.objects.delete(genObjectKey(key));
|
|
2061
|
+
return { bytes: body.slice(body.length - take), size: body.length, version };
|
|
2623
2062
|
}
|
|
2624
|
-
async
|
|
2625
|
-
|
|
2063
|
+
async delete(key, options) {
|
|
2064
|
+
const k = genObjectKey(key);
|
|
2065
|
+
const ifVersion = options?.ifVersion;
|
|
2066
|
+
const stored = this.objects.get(k);
|
|
2067
|
+
if (ifVersion !== void 0 && stored !== void 0 && stored.version !== ifVersion) {
|
|
2068
|
+
throw new WriteConflictError(
|
|
2069
|
+
`generation ${key.segment}.${key.generation} is another object than the version given; not deleted`
|
|
2070
|
+
);
|
|
2071
|
+
}
|
|
2072
|
+
this.objects.delete(k);
|
|
2073
|
+
}
|
|
2074
|
+
async *list(ref) {
|
|
2075
|
+
validateSegmentRef(ref);
|
|
2626
2076
|
const prefix = `${segmentKey(ref)} `;
|
|
2627
2077
|
for (const k of this.objects.keys()) {
|
|
2628
2078
|
if (k.startsWith(prefix)) {
|
|
@@ -2635,11 +2085,11 @@ var MemoryStorageDriver = class {
|
|
|
2635
2085
|
}
|
|
2636
2086
|
}
|
|
2637
2087
|
require(key) {
|
|
2638
|
-
const
|
|
2639
|
-
if (
|
|
2088
|
+
const stored = this.objects.get(genObjectKey(key));
|
|
2089
|
+
if (stored === void 0) {
|
|
2640
2090
|
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
2641
2091
|
}
|
|
2642
|
-
return
|
|
2092
|
+
return stored;
|
|
2643
2093
|
}
|
|
2644
2094
|
};
|
|
2645
2095
|
function genObjectKey(key) {
|
|
@@ -2898,7 +2348,12 @@ var LocalFsStorageDriver = class {
|
|
|
2898
2348
|
root;
|
|
2899
2349
|
exactCase;
|
|
2900
2350
|
capabilities() {
|
|
2901
|
-
return {
|
|
2351
|
+
return {
|
|
2352
|
+
rangeRead: true,
|
|
2353
|
+
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2354
|
+
conditionalPut: false,
|
|
2355
|
+
conditionalDelete: false
|
|
2356
|
+
};
|
|
2902
2357
|
}
|
|
2903
2358
|
async putImmutable(key, write) {
|
|
2904
2359
|
const finalPath = storageObjectPath(this.root, key);
|
|
@@ -2975,7 +2430,12 @@ var LocalFsStorageDriver = class {
|
|
|
2975
2430
|
await handle.close();
|
|
2976
2431
|
}
|
|
2977
2432
|
}
|
|
2978
|
-
|
|
2433
|
+
/**
|
|
2434
|
+
* Idempotent. Ignores `options.ifVersion`, as `conditionalDelete: false` says: see the note on the class for why it
|
|
2435
|
+
* cannot apply one. The parameter is declared so that a wrapper of this driver can hand the port's options on.
|
|
2436
|
+
*/
|
|
2437
|
+
async delete(key, options) {
|
|
2438
|
+
void options;
|
|
2979
2439
|
const path = storageObjectPath(this.root, key);
|
|
2980
2440
|
if (await this.exactCase.differs(path)) return;
|
|
2981
2441
|
await unlink(path).catch((err) => {
|
|
@@ -3074,7 +2534,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3074
2534
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
3075
2535
|
}
|
|
3076
2536
|
const token = newIncarnationToken(this.entropy, current?.record);
|
|
3077
|
-
await this.writeRow(path,
|
|
2537
|
+
await this.writeRow(path, recordFromNew(ref, checked, this.now(), token));
|
|
3078
2538
|
return { token };
|
|
3079
2539
|
});
|
|
3080
2540
|
}
|
|
@@ -3092,11 +2552,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3092
2552
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3093
2553
|
}
|
|
3094
2554
|
const token = nextRegistryToken(current.record, this.entropy);
|
|
3095
|
-
await this.writeRow(
|
|
3096
|
-
path,
|
|
3097
|
-
false,
|
|
3098
|
-
applyRegistryPatch(current.record, checked, this.now(), token)
|
|
3099
|
-
);
|
|
2555
|
+
await this.writeRow(path, applyRegistryPatch(current.record, checked, this.now(), token));
|
|
3100
2556
|
return { token };
|
|
3101
2557
|
});
|
|
3102
2558
|
}
|
|
@@ -3131,12 +2587,7 @@ var LocalFsRegistryDriver = class {
|
|
|
3131
2587
|
} else if (current === null || current.deleted) {
|
|
3132
2588
|
return;
|
|
3133
2589
|
}
|
|
3134
|
-
|
|
3135
|
-
await this.unlinkRow(path);
|
|
3136
|
-
return;
|
|
3137
|
-
}
|
|
3138
|
-
const token = nextRegistryToken(current.record, this.entropy);
|
|
3139
|
-
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
2590
|
+
await this.unlinkRow(path);
|
|
3140
2591
|
});
|
|
3141
2592
|
}
|
|
3142
2593
|
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
@@ -3186,10 +2637,10 @@ var LocalFsRegistryDriver = class {
|
|
|
3186
2637
|
await handle.close();
|
|
3187
2638
|
}
|
|
3188
2639
|
}
|
|
3189
|
-
async writeRow(path,
|
|
2640
|
+
async writeRow(path, record) {
|
|
3190
2641
|
await this.exactCase.refuseVariant(path);
|
|
3191
2642
|
await mkdir2(dirname3(path), { recursive: true });
|
|
3192
|
-
const out = Buffer.from(serializeRegistryEnvelope({ deleted, record }), "utf8");
|
|
2643
|
+
const out = Buffer.from(serializeRegistryEnvelope({ deleted: false, record }), "utf8");
|
|
3193
2644
|
if (out.length > DEFAULT_MAX_ROW_BYTES) {
|
|
3194
2645
|
throw new ValidationError(
|
|
3195
2646
|
`registry row ${out.length}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`
|
|
@@ -3263,6 +2714,14 @@ var LocalFsStorage = class {
|
|
|
3263
2714
|
registry;
|
|
3264
2715
|
};
|
|
3265
2716
|
|
|
2717
|
+
// src/core/reader-defaults.ts
|
|
2718
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
2719
|
+
var REFRESH_RETRY_MS = 500;
|
|
2720
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
2721
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
2722
|
+
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
2723
|
+
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
2724
|
+
|
|
3266
2725
|
// src/core/cooperative.ts
|
|
3267
2726
|
var YIELD_EVERY = 1024;
|
|
3268
2727
|
function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
@@ -3431,14 +2890,6 @@ function indexCapacity(indexLength) {
|
|
|
3431
2890
|
function magicMatches(bytes, offset) {
|
|
3432
2891
|
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
3433
2892
|
}
|
|
3434
|
-
var sizePart = (size) => `${size}:`;
|
|
3435
|
-
var fingerprintFor = (size, footerCrc) => `${sizePart(size)}${footerCrc}`;
|
|
3436
|
-
function compactFingerprint(fingerprint) {
|
|
3437
|
-
const colon = fingerprint.indexOf(":");
|
|
3438
|
-
const size = Number(fingerprint.slice(0, colon));
|
|
3439
|
-
const crc = Number(fingerprint.slice(colon + 1));
|
|
3440
|
-
return `${size.toString(36)}.${crc.toString(36)}`;
|
|
3441
|
-
}
|
|
3442
2893
|
function checkSize(tail, size) {
|
|
3443
2894
|
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3444
2895
|
throw new IntegrityError(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
@@ -3515,7 +2966,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3515
2966
|
get fingerprint() {
|
|
3516
2967
|
return fingerprintFor(this.objectSize, this.footerCrc);
|
|
3517
2968
|
}
|
|
3518
|
-
/** Total object bytes (from the one-GET tail read)
|
|
2969
|
+
/** Total object bytes (from the one-GET tail read): the `sizeBytes` a segment's `stat()` reports. */
|
|
3519
2970
|
get sizeBytes() {
|
|
3520
2971
|
return this.objectSize;
|
|
3521
2972
|
}
|
|
@@ -4424,7 +3875,16 @@ function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
|
4424
3875
|
var NONCE_BYTES = 12;
|
|
4425
3876
|
var TAG_BYTES = 16;
|
|
4426
3877
|
var COUNT_BYTES = 8;
|
|
3878
|
+
var SIZE_BYTES = 8;
|
|
3879
|
+
var CRC_BYTES = 4;
|
|
3880
|
+
var FIXED_BYTES = COUNT_BYTES + SIZE_BYTES + CRC_BYTES;
|
|
4427
3881
|
var MAX_CARDINALITY = 2 ** 32;
|
|
3882
|
+
function sizeOfFingerprint(fingerprint) {
|
|
3883
|
+
const parts = fingerprintParts(fingerprint);
|
|
3884
|
+
if (parts === void 0)
|
|
3885
|
+
throw new IntegrityError("registry record summary: a malformed fingerprint");
|
|
3886
|
+
return parts.size;
|
|
3887
|
+
}
|
|
4428
3888
|
var BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
4429
3889
|
function toBase64(bytes) {
|
|
4430
3890
|
let out = "";
|
|
@@ -4451,16 +3911,21 @@ function fromBase64(text) {
|
|
|
4451
3911
|
}
|
|
4452
3912
|
return out;
|
|
4453
3913
|
}
|
|
4454
|
-
function clearSummary(generation, cardinality, metadata) {
|
|
4455
|
-
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata };
|
|
3914
|
+
function clearSummary(generation, cardinality, fingerprint, metadata) {
|
|
3915
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality, fingerprint } : { generation, cardinality, fingerprint, metadata };
|
|
4456
3916
|
}
|
|
4457
|
-
function sealSummary(aead, ref, generation, cardinality, metadata) {
|
|
3917
|
+
function sealSummary(aead, ref, generation, cardinality, fingerprint, metadata) {
|
|
4458
3918
|
const meta = metadataBytes(metadata, (message) => {
|
|
4459
3919
|
throw new IntegrityError(`summary metadata: ${message}`);
|
|
4460
3920
|
});
|
|
4461
|
-
const
|
|
4462
|
-
|
|
4463
|
-
|
|
3921
|
+
const parts = fingerprintParts(fingerprint);
|
|
3922
|
+
if (parts === void 0) throw new IntegrityError("summary: a malformed fingerprint");
|
|
3923
|
+
const plain = new Uint8Array(FIXED_BYTES + (meta?.length ?? 0));
|
|
3924
|
+
const view = new DataView(plain.buffer);
|
|
3925
|
+
view.setBigUint64(0, BigInt(cardinality), true);
|
|
3926
|
+
view.setBigUint64(COUNT_BYTES, BigInt(parts.size), true);
|
|
3927
|
+
view.setUint32(COUNT_BYTES + SIZE_BYTES, parts.footerCrc, true);
|
|
3928
|
+
if (meta !== void 0) plain.set(meta, FIXED_BYTES);
|
|
4464
3929
|
const { nonce, ciphertext, tag } = aead.seal(plain, aadFor(ref, generation, "summary"));
|
|
4465
3930
|
const framed2 = new Uint8Array(nonce.length + ciphertext.length + tag.length);
|
|
4466
3931
|
framed2.set(nonce, 0);
|
|
@@ -4474,8 +3939,10 @@ function openSummary(aead, ref, summary) {
|
|
|
4474
3939
|
};
|
|
4475
3940
|
const framed2 = fromBase64(summary.sealed);
|
|
4476
3941
|
const framing = NONCE_BYTES + TAG_BYTES;
|
|
4477
|
-
if (framed2 === void 0 || framed2.length < framing +
|
|
4478
|
-
fail(
|
|
3942
|
+
if (framed2 === void 0 || framed2.length < framing + FIXED_BYTES || framed2.length > framing + FIXED_BYTES + MAX_METADATA_BYTES) {
|
|
3943
|
+
fail(
|
|
3944
|
+
"the sealed summary is not a nonce, a count, a fingerprint, up to 1 KiB of metadata and a tag"
|
|
3945
|
+
);
|
|
4479
3946
|
}
|
|
4480
3947
|
const bytes = framed2;
|
|
4481
3948
|
let plain;
|
|
@@ -4494,14 +3961,18 @@ function openSummary(aead, ref, summary) {
|
|
|
4494
3961
|
"the sealed summary does not open: tampered bytes, or a summary moved from another generation"
|
|
4495
3962
|
);
|
|
4496
3963
|
}
|
|
4497
|
-
if (plain.length <
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
);
|
|
3964
|
+
if (plain.length < FIXED_BYTES)
|
|
3965
|
+
fail("the sealed summary is shorter than its count and fingerprint");
|
|
3966
|
+
const view = new DataView(plain.buffer, plain.byteOffset, plain.byteLength);
|
|
3967
|
+
const count = view.getBigUint64(0, true);
|
|
4502
3968
|
if (count > BigInt(MAX_CARDINALITY)) fail(`cardinality ${count} is outside 0 to 2^32`);
|
|
4503
|
-
const
|
|
4504
|
-
|
|
3969
|
+
const size = view.getBigUint64(COUNT_BYTES, true);
|
|
3970
|
+
const fingerprint = fingerprintFor(Number(size), view.getUint32(COUNT_BYTES + SIZE_BYTES, true));
|
|
3971
|
+
if (size > BigInt(Number.MAX_SAFE_INTEGER) || fingerprintParts(fingerprint) === void 0) {
|
|
3972
|
+
fail(`the object size ${size} is not one a .crbm object can have`);
|
|
3973
|
+
}
|
|
3974
|
+
const metadata = plain.length === FIXED_BYTES ? void 0 : metadataFromBytes(plain.subarray(FIXED_BYTES), fail);
|
|
3975
|
+
return { cardinality: Number(count), metadata, fingerprint };
|
|
4505
3976
|
}
|
|
4506
3977
|
function usableSummary(ref, row, aead) {
|
|
4507
3978
|
const summary = row.summary;
|
|
@@ -4519,7 +3990,12 @@ function usableSummary(ref, row, aead) {
|
|
|
4519
3990
|
}
|
|
4520
3991
|
}
|
|
4521
3992
|
if (keyed) return void 0;
|
|
4522
|
-
|
|
3993
|
+
if (fingerprintParts(summary.fingerprint) === void 0) return void 0;
|
|
3994
|
+
return {
|
|
3995
|
+
cardinality: summary.cardinality,
|
|
3996
|
+
metadata: summary.metadata,
|
|
3997
|
+
fingerprint: summary.fingerprint
|
|
3998
|
+
};
|
|
4523
3999
|
}
|
|
4524
4000
|
function canonicalOrNone(metadata) {
|
|
4525
4001
|
if (metadata === void 0) return void 0;
|
|
@@ -4536,20 +4012,35 @@ function metadataToCarry(object, authenticated) {
|
|
|
4536
4012
|
return empty(object) && authenticated !== void 0 && !empty(authenticated.metadata) ? authenticated.metadata : object;
|
|
4537
4013
|
}
|
|
4538
4014
|
function summaryOf(ref, generation, description, aead) {
|
|
4539
|
-
|
|
4015
|
+
const { cardinality, fingerprint, metadata } = description;
|
|
4016
|
+
return aead === void 0 ? clearSummary(generation, cardinality, fingerprint, metadata) : sealSummary(aead, ref, generation, cardinality, fingerprint, metadata);
|
|
4540
4017
|
}
|
|
4541
4018
|
|
|
4542
4019
|
// src/core/crbm-storage-source.ts
|
|
4543
|
-
function storageBlobReader(driver, key) {
|
|
4020
|
+
function storageBlobReader(driver, key, noteTail) {
|
|
4544
4021
|
return {
|
|
4545
4022
|
getRange(offset, length) {
|
|
4546
4023
|
return driver.getRange(key, offset, length);
|
|
4547
4024
|
},
|
|
4548
|
-
getTail(maxBytes) {
|
|
4549
|
-
|
|
4025
|
+
getTail: noteTail === void 0 ? (maxBytes) => driver.getTail(key, maxBytes) : async (maxBytes) => {
|
|
4026
|
+
const tail = await driver.getTail(key, maxBytes);
|
|
4027
|
+
noteTail(tail);
|
|
4028
|
+
return tail;
|
|
4550
4029
|
}
|
|
4551
4030
|
};
|
|
4552
4031
|
}
|
|
4032
|
+
function resolvedOf(record) {
|
|
4033
|
+
if (record === null || record.status === "destroyed" || record.currentGen === null) return null;
|
|
4034
|
+
return {
|
|
4035
|
+
generation: record.currentGen,
|
|
4036
|
+
lineage: pointerIdOf(record),
|
|
4037
|
+
wrappedDeks: record.wrappedDeks,
|
|
4038
|
+
summary: record.summary
|
|
4039
|
+
};
|
|
4040
|
+
}
|
|
4041
|
+
function weightOf(resolved) {
|
|
4042
|
+
return SNAPSHOT_BASE_BYTES + (resolved.summary === void 0 ? 0 : JSON.stringify(resolved.summary).length) + (resolved.wrappedDeks === void 0 ? 0 : JSON.stringify(resolved.wrappedDeks).length);
|
|
4043
|
+
}
|
|
4553
4044
|
var notThePinned = (ref, generation) => new NotFoundError(
|
|
4554
4045
|
`segment "${ref.segment}" generation ${generation} is no longer the object this handle pinned`
|
|
4555
4046
|
);
|
|
@@ -4575,6 +4066,12 @@ function summaryOfGeneration(generation, description) {
|
|
|
4575
4066
|
const { cardinality, metadata } = description;
|
|
4576
4067
|
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
4577
4068
|
}
|
|
4069
|
+
function sizedSummary(reader) {
|
|
4070
|
+
return {
|
|
4071
|
+
...summaryOfGeneration(reader.generation, describe(reader)),
|
|
4072
|
+
sizeBytes: reader.sizeBytes
|
|
4073
|
+
};
|
|
4074
|
+
}
|
|
4578
4075
|
var sameKeys = (a, b) => a !== void 0 && JSON.stringify(a) === JSON.stringify(b);
|
|
4579
4076
|
function once(fn) {
|
|
4580
4077
|
let pending;
|
|
@@ -4603,25 +4100,25 @@ function describe(reader) {
|
|
|
4603
4100
|
return { cardinality: reader.count(), metadata: reader.metadata };
|
|
4604
4101
|
}
|
|
4605
4102
|
var Snapshot = class _Snapshot {
|
|
4606
|
-
constructor(target, open4, prior) {
|
|
4103
|
+
constructor(resolution, target, open4, prior) {
|
|
4104
|
+
this.resolution = resolution;
|
|
4607
4105
|
this.target = target;
|
|
4608
4106
|
this.open = open4;
|
|
4609
4107
|
this.prior = prior;
|
|
4610
4108
|
}
|
|
4109
|
+
resolution;
|
|
4611
4110
|
target;
|
|
4612
4111
|
open;
|
|
4613
4112
|
prior;
|
|
4614
|
-
/** When the snapshot's refresh clock started. A refresh that failed transiently moves it back, so it lapses sooner. */
|
|
4615
|
-
installedAtMs = 0;
|
|
4616
4113
|
/** Set by the cache, to hear of a reader as it is opened. */
|
|
4617
4114
|
onReader;
|
|
4618
4115
|
opened;
|
|
4619
4116
|
outcome;
|
|
4620
|
-
static live(target, open4, prior) {
|
|
4621
|
-
return new _Snapshot(target, open4, prior);
|
|
4117
|
+
static live(resolution, target, open4, prior) {
|
|
4118
|
+
return new _Snapshot(resolution, target, open4, prior);
|
|
4622
4119
|
}
|
|
4623
4120
|
static eager(reader) {
|
|
4624
|
-
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
4121
|
+
const snap = new _Snapshot(void 0, void 0, void 0, void 0);
|
|
4625
4122
|
snap.hold(reader);
|
|
4626
4123
|
return snap;
|
|
4627
4124
|
}
|
|
@@ -4667,11 +4164,18 @@ var Snapshot = class _Snapshot {
|
|
|
4667
4164
|
get settled() {
|
|
4668
4165
|
return this.outcome;
|
|
4669
4166
|
}
|
|
4670
|
-
/**
|
|
4167
|
+
/**
|
|
4168
|
+
* The prior snapshot's reader, if it is of what this snapshot resolved: the same generation and `pointerId`, and the
|
|
4169
|
+
* object the row's summary names, where it names one. A registry renews `pointerId` with every write that changes the
|
|
4170
|
+
* summary, so the last holds whenever the first two do; it is checked anyway, since it costs no request, and a reader
|
|
4171
|
+
* kept across a write that should have renewed it would serve another object's chunks.
|
|
4172
|
+
*/
|
|
4671
4173
|
async reuse(live) {
|
|
4672
4174
|
const prior = this.prior;
|
|
4673
4175
|
const reader = prior === void 0 ? null : await prior.catch(() => null);
|
|
4674
|
-
|
|
4176
|
+
if (reader === null || !readerIsOf(reader, live)) return null;
|
|
4177
|
+
const named = await live.summary().catch(() => void 0);
|
|
4178
|
+
return named === void 0 || named.fingerprint === reader.fingerprint ? reader : null;
|
|
4675
4179
|
}
|
|
4676
4180
|
/** Keep the prior snapshot's reader when the refresh found its generation, so a count does not drop it. */
|
|
4677
4181
|
async adopt(live) {
|
|
@@ -4730,6 +4234,11 @@ var CrbmStorageChunkSource = class {
|
|
|
4730
4234
|
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
4731
4235
|
clock: clock ?? { now: () => 0 }
|
|
4732
4236
|
});
|
|
4237
|
+
this.resolutions = this.timedRefresh ? new BoundedLru({
|
|
4238
|
+
maxEntries: RESOLUTIONS_PER_OPEN_SEGMENT * (maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS),
|
|
4239
|
+
maxBytes: (maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES) / RESOLUTION_BYTES_DIVISOR,
|
|
4240
|
+
clock: { now: () => 0 }
|
|
4241
|
+
}) : void 0;
|
|
4733
4242
|
this.distrusted = new BoundedLru({
|
|
4734
4243
|
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
4735
4244
|
clock: { now: () => 0 }
|
|
@@ -4754,6 +4263,21 @@ var CrbmStorageChunkSource = class {
|
|
|
4754
4263
|
* steady-state memory bound; re-opening an evicted segment is one cheap tail GET.
|
|
4755
4264
|
*/
|
|
4756
4265
|
snapshots;
|
|
4266
|
+
/**
|
|
4267
|
+
* Each segment's resolution, kept apart from its reader, by segment: what a read of its row found, for the TTL from
|
|
4268
|
+
* the instant that read was sent, whether or not the reader cache still holds the segment's reader. So letting a reader
|
|
4269
|
+
* go moves nothing: the next read builds a snapshot on the resolution, reads no row, and opens the object when it needs
|
|
4270
|
+
* the index or a chunk the chunk cache does not hold (a version, a count and a stat from the row's summary need
|
|
4271
|
+
* neither). It holds the resolved fields only ({@link Resolved}): the wrapped keys, never an unwrapped one, so a
|
|
4272
|
+
* snapshot built on it unwraps the key again through the keystore. It never holds a resolution that found no
|
|
4273
|
+
* generation: the next read of such a segment reads the row.
|
|
4274
|
+
*
|
|
4275
|
+
* Only a source with a timed refresh has one: without it, letting a reader go is one of the few things that ever moves
|
|
4276
|
+
* a read on, and keeping the resolution would keep it forever. **Bounded** by count and by bytes, both derived from
|
|
4277
|
+
* the reader cache's: {@link RESOLUTIONS_PER_OPEN_SEGMENT} times its count bound, and its byte bound over
|
|
4278
|
+
* {@link RESOLUTION_BYTES_DIVISOR}. An entry heavier than the byte bound is kept alone, as the reader cache keeps one.
|
|
4279
|
+
*/
|
|
4280
|
+
resolutions;
|
|
4757
4281
|
/**
|
|
4758
4282
|
* Pins ({@link heldKey}) whose object has been found replaced: another object is under its generation's key. Kept,
|
|
4759
4283
|
* and bounded as the readers are, so that a pin's later chunk reads and any reopen fail at once rather than pay for
|
|
@@ -4783,6 +4307,13 @@ var CrbmStorageChunkSource = class {
|
|
|
4783
4307
|
* the disagreement again.
|
|
4784
4308
|
*/
|
|
4785
4309
|
distrusted;
|
|
4310
|
+
/**
|
|
4311
|
+
* The readers held to the row's summary: by a live open ({@link openLive}), or by a live pin that shared one. A pin
|
|
4312
|
+
* that finds one memoised under its version shares it as it is: the version names the row's `pointerId`, and every
|
|
4313
|
+
* write of the summary renews it, so the summary it was held to is the one the row has now. Any other memoised reader
|
|
4314
|
+
* (a `pinAt`'s, held to the fingerprint its caller named) is held to the row's summary before a live pin shares it.
|
|
4315
|
+
*/
|
|
4316
|
+
rowChecked = /* @__PURE__ */ new WeakSet();
|
|
4786
4317
|
registry;
|
|
4787
4318
|
keystore;
|
|
4788
4319
|
requireEncryption;
|
|
@@ -4793,8 +4324,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4793
4324
|
currentGenTtlMs;
|
|
4794
4325
|
/**
|
|
4795
4326
|
* Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
|
|
4796
|
-
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes
|
|
4797
|
-
*
|
|
4327
|
+
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes and the TTL
|
|
4328
|
+
* lapse, so it is decided once.
|
|
4798
4329
|
*/
|
|
4799
4330
|
timedRefresh;
|
|
4800
4331
|
/**
|
|
@@ -4807,77 +4338,135 @@ var CrbmStorageChunkSource = class {
|
|
|
4807
4338
|
return this.keys.of(ref);
|
|
4808
4339
|
}
|
|
4809
4340
|
/**
|
|
4810
|
-
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window
|
|
4811
|
-
*
|
|
4341
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window: the snapshot the reader cache holds,
|
|
4342
|
+
* while its resolution is inside the TTL. With a timed refresh, a segment whose reader the cache let go is served a
|
|
4343
|
+
* snapshot built on its kept resolution, while that is inside the TTL: no row is read, and the reader opens only when a
|
|
4344
|
+
* read needs one. Otherwise the pointer is read now. The snapshot holds the resolved target, and its reader once a read
|
|
4345
|
+
* has asked for one.
|
|
4812
4346
|
*/
|
|
4813
4347
|
liveSnapshot(ref) {
|
|
4814
4348
|
const key = this.keyOf(ref);
|
|
4815
4349
|
const existing = this.snapshots.get(key);
|
|
4816
|
-
if (existing
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
);
|
|
4350
|
+
if (existing !== void 0 && !this.expired(existing.resolution)) return existing;
|
|
4351
|
+
const kept = this.resolutions?.get(key);
|
|
4352
|
+
if (kept !== void 0 && kept !== existing?.resolution && !this.expired(kept)) {
|
|
4353
|
+
return this.install(key, this.snapshotOn(ref, kept, existing));
|
|
4821
4354
|
}
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
Snapshot.live(
|
|
4826
|
-
this.refreshedTarget(ref, existing, () => this.retrySoon(snap)),
|
|
4827
|
-
(live) => this.openLive(ref, live),
|
|
4828
|
-
existing.openedReader
|
|
4829
|
-
)
|
|
4830
|
-
);
|
|
4831
|
-
return snap;
|
|
4355
|
+
const resolution = this.resolveNow(ref, existing?.resolution);
|
|
4356
|
+
this.remember(key, resolution);
|
|
4357
|
+
return this.install(key, this.snapshotOn(ref, resolution, existing));
|
|
4832
4358
|
}
|
|
4833
|
-
/**
|
|
4834
|
-
|
|
4835
|
-
|
|
4359
|
+
/**
|
|
4360
|
+
* Read the segment's pointer now. The read is sent before this returns, and the TTL counts from this instant: a read
|
|
4361
|
+
* that answers late does not lengthen the life of what it found. The row read is always fresh, so what a snapshot on
|
|
4362
|
+
* it says of the generation (its summary included) is what the row says now, never `prior`'s.
|
|
4363
|
+
*
|
|
4364
|
+
* A read that fails with a transient fault answers what `prior` found, when it found a generation, and is asked again
|
|
4365
|
+
* soon ({@link retrySoon}): `prior` is the resolution of the snapshot the reader cache still holds, so an outage of the
|
|
4366
|
+
* registry keeps that snapshot's generation, and its reader and unwrapped key where it has them; one resolved without
|
|
4367
|
+
* an open, for a count or a generation lookup, unwraps the key from `prior`'s wrapped keys when a read first needs
|
|
4368
|
+
* it. The outage ends shortly after the registry answers.
|
|
4369
|
+
* That answer is the snapshot's alone ({@link Resolution.rodeOut}): it is not kept as the segment's resolution, and a
|
|
4370
|
+
* snapshot built on it after the reader cache let that one go fails with the fault. With no `prior` the fault
|
|
4371
|
+
* fails the read, as a cold resolve's does. If `prior` found none or failed, the pointer is read once more rather than
|
|
4372
|
+
* a dead resolution re-armed (else the segment reads empty for a whole TTL window). Anything else — an access denial, a corrupt row, a
|
|
4373
|
+
* registry that answers NotFound — fails the read exactly as a cold resolve of the segment would, and the resolution
|
|
4374
|
+
* and the snapshot on it are forgotten, so a reader and the key it unwrapped do not outlive a refresh that could not
|
|
4375
|
+
* be trusted.
|
|
4376
|
+
*/
|
|
4377
|
+
resolveNow(ref, prior) {
|
|
4378
|
+
const sentAtMs = this.now();
|
|
4379
|
+
const sent = this.resolve(ref);
|
|
4380
|
+
if (prior === void 0) return { resolved: sent, sentAtMs };
|
|
4381
|
+
const resolution = {
|
|
4382
|
+
sentAtMs,
|
|
4383
|
+
resolved: sent.catch(async (err) => {
|
|
4384
|
+
if (!isTransientError(err)) throw err;
|
|
4385
|
+
const before = await prior.resolved.catch(() => null);
|
|
4386
|
+
if (before === null) return this.resolve(ref);
|
|
4387
|
+
this.retrySoon(resolution);
|
|
4388
|
+
const key = this.keyOf(ref);
|
|
4389
|
+
if (this.resolutions?.peek(key) === resolution) this.resolutions.delete(key);
|
|
4390
|
+
resolution.rodeOut = { from: prior, fault: err };
|
|
4391
|
+
prior.rodeOut = void 0;
|
|
4392
|
+
return before;
|
|
4393
|
+
})
|
|
4394
|
+
};
|
|
4395
|
+
return resolution;
|
|
4836
4396
|
}
|
|
4837
4397
|
/**
|
|
4838
|
-
* A refresh failed transiently and the prior
|
|
4839
|
-
*
|
|
4840
|
-
*
|
|
4398
|
+
* A refresh failed transiently and the prior resolution keeps serving: make it lapse after {@link REFRESH_RETRY_MS}
|
|
4399
|
+
* (never longer than the TTL) rather than after a whole TTL, so the next read after that asks the registry again. It
|
|
4400
|
+
* costs no read of its own; it only changes when the next refresh happens.
|
|
4841
4401
|
*/
|
|
4842
|
-
retrySoon(
|
|
4843
|
-
|
|
4402
|
+
retrySoon(resolution) {
|
|
4403
|
+
resolution.sentAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
4844
4404
|
}
|
|
4845
4405
|
/**
|
|
4846
|
-
*
|
|
4847
|
-
*
|
|
4848
|
-
*
|
|
4406
|
+
* Keep `resolution` as the segment's, from the instant its read was sent, so the reads that come while it is in flight
|
|
4407
|
+
* share it, and a read after the reader cache let the segment go builds on it. Its weight is reported once it lands,
|
|
4408
|
+
* and one that finds no generation, or fails, is forgotten. Both are identity-guarded, as the reader cache's are, and
|
|
4409
|
+
* nothing is installed when the read lands: an `invalidate` while it is in flight, or a newer resolution, is never
|
|
4410
|
+
* undone by its answer.
|
|
4849
4411
|
*/
|
|
4850
|
-
|
|
4851
|
-
const
|
|
4852
|
-
|
|
4853
|
-
|
|
4854
|
-
|
|
4855
|
-
if (
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
}
|
|
4412
|
+
remember(key, resolution) {
|
|
4413
|
+
const cache = this.resolutions;
|
|
4414
|
+
if (cache === void 0) return;
|
|
4415
|
+
cache.set(key, resolution);
|
|
4416
|
+
const forgetIfStale = () => {
|
|
4417
|
+
if (cache.peek(key) === resolution) cache.delete(key);
|
|
4418
|
+
};
|
|
4419
|
+
resolution.resolved.then((resolved) => {
|
|
4420
|
+
if (resolved === null) forgetIfStale();
|
|
4421
|
+
else if (cache.peek(key) === resolution) cache.setWeight(key, weightOf(resolved));
|
|
4422
|
+
}, forgetIfStale);
|
|
4861
4423
|
}
|
|
4862
4424
|
/**
|
|
4863
|
-
*
|
|
4864
|
-
*
|
|
4865
|
-
*
|
|
4425
|
+
* A live snapshot on `resolution`. `before` is the snapshot it replaces, while the reader cache still holds one: its
|
|
4426
|
+
* reader is kept if the resolution finds the same generation, `pointerId` and object ({@link Snapshot.reuse}), and its
|
|
4427
|
+
* unwrapped key if the same wrapped keys. A snapshot built after the reader cache let the segment go has neither, so
|
|
4428
|
+
* its open asks the keystore again: a key is held only while a reader of the segment is.
|
|
4429
|
+
*/
|
|
4430
|
+
snapshotOn(ref, resolution, before) {
|
|
4431
|
+
return Snapshot.live(
|
|
4432
|
+
resolution,
|
|
4433
|
+
this.targetOn(ref, resolution, before),
|
|
4434
|
+
(live) => this.openLive(ref, live),
|
|
4435
|
+
before?.openedReader
|
|
4436
|
+
);
|
|
4437
|
+
}
|
|
4438
|
+
/** What the snapshot on `resolution` that replaces `before` reads: see {@link snapshotOn}. */
|
|
4439
|
+
targetOn(ref, resolution, before) {
|
|
4440
|
+
const prior = before?.target;
|
|
4441
|
+
const replaces = before?.resolution;
|
|
4442
|
+
return resolution.resolved.then(async (resolved) => {
|
|
4443
|
+
if (resolved === null) return null;
|
|
4444
|
+
const rodeOut = resolution.rodeOut;
|
|
4445
|
+
if (rodeOut !== void 0 && replaces !== rodeOut.from) throw rodeOut.fault;
|
|
4446
|
+
return this.liveOf(ref, resolved, prior === void 0 ? null : await prior.catch(() => null));
|
|
4447
|
+
});
|
|
4448
|
+
}
|
|
4449
|
+
/**
|
|
4450
|
+
* Memoize a snapshot as the segment's, forgetting it later if it resolves to null / throws (identity-guarded, so a
|
|
4451
|
+
* stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot **synchronously** is what lets
|
|
4452
|
+
* concurrent callers coalesce onto one in-flight resolve.
|
|
4866
4453
|
*/
|
|
4867
4454
|
install(key, snap) {
|
|
4868
|
-
snap.installedAtMs = this.now();
|
|
4869
4455
|
this.snapshots.set(key, snap);
|
|
4870
4456
|
const forgetIfStale = () => {
|
|
4871
4457
|
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
4872
4458
|
};
|
|
4873
4459
|
snap.onReader = (reader) => {
|
|
4874
|
-
reader.then(
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4460
|
+
reader.then(
|
|
4461
|
+
(r) => {
|
|
4462
|
+
if (r === null) {
|
|
4463
|
+
forgetIfStale();
|
|
4464
|
+
return;
|
|
4465
|
+
}
|
|
4466
|
+
if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedBytes);
|
|
4467
|
+
},
|
|
4468
|
+
() => this.dropStale(key, snap)
|
|
4469
|
+
);
|
|
4881
4470
|
};
|
|
4882
4471
|
if (snap.target !== void 0) {
|
|
4883
4472
|
snap.target.then(async (live) => {
|
|
@@ -4897,53 +4486,46 @@ var CrbmStorageChunkSource = class {
|
|
|
4897
4486
|
now() {
|
|
4898
4487
|
return this.clock ? this.clock.now() : 0;
|
|
4899
4488
|
}
|
|
4900
|
-
/**
|
|
4901
|
-
* How often this source re-reads a segment's pointer while the segment is being read, in ms: its TTL, or 0 when
|
|
4902
|
-
* it has no timed refresh (no clock, no registry, or a TTL of 0). The grounded cost report prices the refresh at
|
|
4903
|
-
* this.
|
|
4904
|
-
*/
|
|
4905
|
-
get pointerRefreshMs() {
|
|
4906
|
-
return this.timedRefresh ? this.currentGenTtlMs : 0;
|
|
4907
|
-
}
|
|
4908
4489
|
/**
|
|
4909
4490
|
* Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
|
|
4910
|
-
* would
|
|
4911
|
-
*
|
|
4912
|
-
*
|
|
4491
|
+
* would. In this order: an invalidation has happened since `epoch` (what the store that erased stops on, at once);
|
|
4492
|
+
* the TTL of the snapshot's resolution has lapsed; or that resolution is no longer the segment's: another read
|
|
4493
|
+
* installed a newer one (a refresh, or a heal), or the resolution cache let it go. With no resolution cache (no timed
|
|
4494
|
+
* refresh) the last test is of the snapshot itself, so there the reader cache letting it go is a move, as it is one of
|
|
4495
|
+
* the few things that ever moves such a store's read on. With one, it is not. Else nothing can have moved it, and the
|
|
4496
|
+
* check costs a compare and a lookup in each cache.
|
|
4913
4497
|
*/
|
|
4914
4498
|
mayHaveMoved(key, snap, epoch) {
|
|
4915
|
-
|
|
4499
|
+
const resolution = snap.resolution;
|
|
4500
|
+
if (this.invalidations !== epoch || this.expired(resolution)) return true;
|
|
4501
|
+
const held = this.snapshots.get(key);
|
|
4502
|
+
return this.resolutions === void 0 ? held !== snap : this.resolutions.get(key) !== resolution;
|
|
4916
4503
|
}
|
|
4917
|
-
expired(
|
|
4918
|
-
return this.timedRefresh && this.now() -
|
|
4504
|
+
expired(resolution) {
|
|
4505
|
+
return this.timedRefresh && this.now() - resolution.sentAtMs >= this.currentGenTtlMs;
|
|
4919
4506
|
}
|
|
4920
4507
|
/**
|
|
4921
|
-
* The segment's current generation number
|
|
4922
|
-
*
|
|
4923
|
-
*
|
|
4508
|
+
* The segment's current generation number, as its snapshot's resolution found it: no backend read within the TTL
|
|
4509
|
+
* window, and no open of the object, ever. `null` if the segment has no committed generation. A generation swept from
|
|
4510
|
+
* under the resolution is met by the read that fetches from it, which heals ({@link withFreshSnapshot}). Since it
|
|
4511
|
+
* opens nothing, the only error it meets is the resolution's own, and it fails with that error at once: there is
|
|
4512
|
+
* nothing to heal. The reads that can open the object (`listChunkKeys`, `summary`, `currentVersion` and the rest)
|
|
4513
|
+
* resolve once more on any `NotFoundError`, a registry's among them, since theirs can be a swept generation.
|
|
4924
4514
|
*
|
|
4925
|
-
*
|
|
4926
|
-
*
|
|
4927
|
-
*
|
|
4928
|
-
*
|
|
4929
|
-
* operand of every `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with
|
|
4930
|
-
* no backend call at all. Routing it through the generic helper cost ~115 ns/op on that path (a second async
|
|
4931
|
-
* frame, a per-call closure, and an `await` on a plain number) for a race that fires only during a
|
|
4932
|
-
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
4515
|
+
* It is spelled out rather than delegated because a lookup like this runs once per operand of every
|
|
4516
|
+
* `has`/`count`/`iterate`/`intersect`, almost always served from the cached snapshot with no backend call at all.
|
|
4517
|
+
* Routing it through the generic helper cost ~115 ns/op on that path (a second async frame, a per-call closure, and
|
|
4518
|
+
* an `await` on a plain number).
|
|
4933
4519
|
*/
|
|
4934
4520
|
async currentGeneration(ref) {
|
|
4935
4521
|
const snap = this.liveSnapshot(ref);
|
|
4936
|
-
|
|
4937
|
-
|
|
4938
|
-
|
|
4939
|
-
if (!isNotFoundError(err)) throw err;
|
|
4940
|
-
this.dropStale(segmentKey(ref), snap);
|
|
4941
|
-
return (await this.resolvedReader(ref))?.generation ?? null;
|
|
4942
|
-
}
|
|
4522
|
+
const settled2 = snap.settled;
|
|
4523
|
+
if (settled2 !== void 0) return settled2.reader?.generation ?? null;
|
|
4524
|
+
return (await snap.target)?.target.generation ?? null;
|
|
4943
4525
|
}
|
|
4944
4526
|
/**
|
|
4945
|
-
* Forget everything derived from `ref`: the resolved snapshot and the open reader behind it (and with
|
|
4946
|
-
* reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
4527
|
+
* Forget everything derived from `ref`: its resolution, the resolved snapshot and the open reader behind it (and with
|
|
4528
|
+
* the reader, the DEK it unwrapped at open time), and what its pins' objects were found to be. The next read
|
|
4947
4529
|
* resolves the segment again.
|
|
4948
4530
|
*
|
|
4949
4531
|
* Needed because the TTL refresh only ever answers "has the pointer moved?", and every *destructive* verb —
|
|
@@ -4956,6 +4538,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4956
4538
|
this.invalidations += 1;
|
|
4957
4539
|
const key = segmentKey(ref);
|
|
4958
4540
|
this.snapshots.delete(key);
|
|
4541
|
+
this.resolutions?.delete(key);
|
|
4959
4542
|
const pinnedPrefix = `${key}@`;
|
|
4960
4543
|
this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
4961
4544
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
@@ -4963,33 +4546,52 @@ var CrbmStorageChunkSource = class {
|
|
|
4963
4546
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
4964
4547
|
}
|
|
4965
4548
|
/**
|
|
4966
|
-
* `<generation>#<object>` for a registry-less source, or `<generation>:<
|
|
4967
|
-
* the segment's snapshot resolved, and the object
|
|
4968
|
-
* base 36
|
|
4969
|
-
*
|
|
4970
|
-
* string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
4971
|
-
* one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
4972
|
-
* within one row once its object was deleted, or an object put back under its key from
|
|
4973
|
-
*
|
|
4974
|
-
*
|
|
4975
|
-
*
|
|
4549
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<pointerId>#<object>` with one: the generation
|
|
4550
|
+
* the segment's snapshot resolved, and the object a reader of it opens, by that object's size and footer checksum in
|
|
4551
|
+
* base 36. The row's `pointerId` moves only with a write that names a field a read resolves through, at any value, so
|
|
4552
|
+
* a lease or a policy write (a `setRetention`) leaves the version, the open reader and the decoded chunks as they were.
|
|
4553
|
+
* Two objects do not share a version string, as far as their sizes and footer checksums tell them apart, whichever way
|
|
4554
|
+
* a number came to name a second one: a name purged and loaded again (a new row, or with no registry the same bare
|
|
4555
|
+
* number), a number taken again within one row once its object was deleted, or an object put back under its key from
|
|
4556
|
+
* outside the library. The version is opaque: compare two for equality.
|
|
4557
|
+
*
|
|
4558
|
+
* Where the row's summary names the object, the version is answered from the resolution, with no open: every live
|
|
4559
|
+
* open of the generation is held to that object ({@link openLive}), refused as a move otherwise, so the one object any
|
|
4560
|
+
* reader of this resolution can name is the one the summary names, and a version named from it is the one that
|
|
4561
|
+
* reader's would be. So a `has()` whose chunk is in the chunk cache opens nothing, after the reader cache let
|
|
4562
|
+
* the segment go included (a read that needs the index opens the object for it, and on an encrypted segment the
|
|
4563
|
+
* key is unwrapped to read the row's sealed summary); an object under the number that is not the row's is met by the read that fetches from it,
|
|
4564
|
+
* which refuses it and heals. Where the row has no summary it can use, or there is no registry, nothing names the
|
|
4565
|
+
* object but the object: the reader is opened, and the version names what it opened, however long after its row read
|
|
4566
|
+
* that is.
|
|
4976
4567
|
*
|
|
4977
|
-
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
4978
|
-
*
|
|
4979
|
-
*
|
|
4980
|
-
*
|
|
4981
|
-
*
|
|
4568
|
+
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its chunk
|
|
4569
|
+
* cache, so an open it makes heals a swept generation as a read does: unhealed, a cold read racing a publish and a
|
|
4570
|
+
* `keep: 0` sweep failed the whole operation with `NotFoundError`, where the same race on a chunk fetch heals. Spelled
|
|
4571
|
+
* out rather than delegated for the reason `currentGeneration` gives: it is almost always served from the cached
|
|
4572
|
+
* snapshot with no backend call at all. One retry, then propagate.
|
|
4982
4573
|
*/
|
|
4983
4574
|
async currentVersion(ref) {
|
|
4984
4575
|
const snap = this.liveSnapshot(ref);
|
|
4985
|
-
|
|
4576
|
+
const settled2 = snap.settled;
|
|
4577
|
+
if (settled2 !== void 0) {
|
|
4578
|
+
return settled2.reader === null ? null : versionOfReader(settled2.reader);
|
|
4579
|
+
}
|
|
4986
4580
|
try {
|
|
4987
|
-
|
|
4581
|
+
return await this.versionOn(snap);
|
|
4988
4582
|
} catch (err) {
|
|
4989
4583
|
if (!isNotFoundError(err)) throw err;
|
|
4990
4584
|
this.dropStale(segmentKey(ref), snap);
|
|
4991
|
-
|
|
4585
|
+
return await this.versionOn(this.liveSnapshot(ref));
|
|
4992
4586
|
}
|
|
4587
|
+
}
|
|
4588
|
+
/** {@link currentVersion} of one snapshot: named by the row's summary when it can be, else by the reader, opened. */
|
|
4589
|
+
async versionOn(snap) {
|
|
4590
|
+
const live = await snap.target;
|
|
4591
|
+
if (live === null) return null;
|
|
4592
|
+
const named = await live.version();
|
|
4593
|
+
if (named !== void 0) return named;
|
|
4594
|
+
const reader = await snap.reader;
|
|
4993
4595
|
return reader === null ? null : versionOfReader(reader);
|
|
4994
4596
|
}
|
|
4995
4597
|
/**
|
|
@@ -5050,11 +4652,10 @@ var CrbmStorageChunkSource = class {
|
|
|
5050
4652
|
if (record.currentGen === null || generation > record.currentGen) {
|
|
5051
4653
|
throw gone("it is not a published generation of this segment");
|
|
5052
4654
|
}
|
|
5053
|
-
|
|
4655
|
+
const lineage = pointerIdOf(record);
|
|
5054
4656
|
if (lease !== void 0) {
|
|
5055
4657
|
const taken = await lease.take(record);
|
|
5056
4658
|
if (taken === "moved") throw gone("the segment moved while its lease was taken");
|
|
5057
|
-
lineage = taken.token;
|
|
5058
4659
|
}
|
|
5059
4660
|
version = versionOf(generation, lineage);
|
|
5060
4661
|
const key = this.pinnedKey(ref, version);
|
|
@@ -5092,12 +4693,11 @@ var CrbmStorageChunkSource = class {
|
|
|
5092
4693
|
return { generation, version, fingerprint };
|
|
5093
4694
|
}
|
|
5094
4695
|
async pinOnce(ref, lease) {
|
|
5095
|
-
|
|
4696
|
+
const live = await this.resolveLive(ref);
|
|
5096
4697
|
if (live === null) return null;
|
|
5097
4698
|
if (lease !== void 0 && live.row !== void 0) {
|
|
5098
4699
|
const taken = await lease.take(live.row);
|
|
5099
4700
|
if (taken === "moved") throw new PinMoved();
|
|
5100
|
-
live = { ...live, target: { ...live.target, lineage: taken.token } };
|
|
5101
4701
|
}
|
|
5102
4702
|
const target = live.target;
|
|
5103
4703
|
const version = versionOf(target.generation, target.lineage);
|
|
@@ -5117,8 +4717,17 @@ var CrbmStorageChunkSource = class {
|
|
|
5117
4717
|
entry ??= this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5118
4718
|
reader = await entry.reader;
|
|
5119
4719
|
if (reader === null) return null;
|
|
5120
|
-
|
|
5121
|
-
const
|
|
4720
|
+
let named;
|
|
4721
|
+
const replaced = async (r) => {
|
|
4722
|
+
if (this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0)
|
|
4723
|
+
return true;
|
|
4724
|
+
if (this.rowChecked.has(r)) return false;
|
|
4725
|
+
const fingerprint = await (named ??= live.summary().then((d) => d?.fingerprint));
|
|
4726
|
+
if (fingerprint !== void 0 && r.fingerprint !== fingerprint) return true;
|
|
4727
|
+
this.rowChecked.add(r);
|
|
4728
|
+
return false;
|
|
4729
|
+
};
|
|
4730
|
+
const known = !opened && await replaced(reader);
|
|
5122
4731
|
if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
|
|
5123
4732
|
this.install(key, Snapshot.eager(Promise.resolve(reader)));
|
|
5124
4733
|
}
|
|
@@ -5127,7 +4736,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5127
4736
|
let fresh = current !== void 0 && current !== entry ? current : this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5128
4737
|
reader = await fresh.reader;
|
|
5129
4738
|
if (reader === null) return null;
|
|
5130
|
-
if (fresh === current && replaced(reader)) {
|
|
4739
|
+
if (fresh === current && await replaced(reader)) {
|
|
5131
4740
|
fresh = this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
5132
4741
|
reader = await fresh.reader;
|
|
5133
4742
|
if (reader === null) return null;
|
|
@@ -5144,7 +4753,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5144
4753
|
}
|
|
5145
4754
|
/**
|
|
5146
4755
|
* Where a pinned reader of one object is memoised. With a registry that is its version, which names the object:
|
|
5147
|
-
* a name purged and loaded again gets a new row, and with it a new
|
|
4756
|
+
* a name purged and loaded again gets a new row, and with it a new `pointerId`. Without one the version is the bare
|
|
5148
4757
|
* generation number, which two incarnations share, so the object's fingerprint is added to tell them apart.
|
|
5149
4758
|
*/
|
|
5150
4759
|
pinnedKey(ref, version, fingerprint) {
|
|
@@ -5180,9 +4789,10 @@ var CrbmStorageChunkSource = class {
|
|
|
5180
4789
|
* **And the object is checked**, when the caller says which one it pinned. A generation number is not an
|
|
5181
4790
|
* identity (invariant 1): once a name is purged and loaded again, its new segment starts again at generation 0,
|
|
5182
4791
|
* so a pin of the old one would open the new one's object as its own, and read it beside the chunks it had
|
|
5183
|
-
* already cached from the old one. The row's
|
|
5184
|
-
*
|
|
5185
|
-
*
|
|
4792
|
+
* already cached from the old one. The row's `pointerId` cannot tell them apart, since it moves with every write
|
|
4793
|
+
* that changes what the row resolves to, a publish included, while the pinned object stays what it was; the
|
|
4794
|
+
* object's fingerprint can, and a pin whose object has been replaced fails, as one whose generation has been swept
|
|
4795
|
+
* does. The memo is keyed so that two pins of one generation number in two
|
|
5186
4796
|
* incarnations never share a reader, with a registry or without one ({@link pinnedKey}).
|
|
5187
4797
|
*/
|
|
5188
4798
|
async readerAt(ref, generation, version, fingerprint) {
|
|
@@ -5224,7 +4834,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5224
4834
|
if (record === null || record.status === "destroyed") return null;
|
|
5225
4835
|
return this.openForTarget(ref, {
|
|
5226
4836
|
generation,
|
|
5227
|
-
lineage: record
|
|
4837
|
+
lineage: pointerIdOf(record),
|
|
5228
4838
|
wrappedDeks: record.wrappedDeks
|
|
5229
4839
|
});
|
|
5230
4840
|
}
|
|
@@ -5305,7 +4915,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5305
4915
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5306
4916
|
return reader === null ? null : reader.cardinalities();
|
|
5307
4917
|
}
|
|
5308
|
-
/**
|
|
4918
|
+
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
5309
4919
|
async sizeOfAt(ref, generation, held) {
|
|
5310
4920
|
validateUserRef(ref);
|
|
5311
4921
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
@@ -5333,19 +4943,29 @@ var CrbmStorageChunkSource = class {
|
|
|
5333
4943
|
return false;
|
|
5334
4944
|
}
|
|
5335
4945
|
/**
|
|
5336
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
5337
|
-
* fresher snapshot is never clobbered. The identity guard is the whole point: `install`
|
|
5338
|
-
* wholesale, so comparing anything else would drop a good
|
|
4946
|
+
* Evict a snapshot we just failed to read from, and the resolution it was built on — each matched by identity, so a
|
|
4947
|
+
* concurrent call's fresher snapshot or resolution is never clobbered. The identity guard is the whole point: `install`
|
|
4948
|
+
* and `remember` replace an entry wholesale, so comparing anything else would drop a good one on the floor. The
|
|
4949
|
+
* resolution goes too, or the read's retry would build its snapshot on it again and reopen the generation it just
|
|
4950
|
+
* found gone; and so does a snapshot the reader cache holds on that same resolution, which another read built once the
|
|
4951
|
+
* cache had let `snap` go, or the retry would open that one.
|
|
5339
4952
|
*/
|
|
5340
4953
|
dropStale(key, snap) {
|
|
5341
|
-
|
|
4954
|
+
const resolution = snap.resolution;
|
|
4955
|
+
const held = this.snapshots.peek(key);
|
|
4956
|
+
if (held === snap || resolution !== void 0 && held?.resolution === resolution) {
|
|
4957
|
+
this.snapshots.delete(key);
|
|
4958
|
+
}
|
|
4959
|
+
if (resolution !== void 0 && this.resolutions?.peek(key) === resolution) {
|
|
4960
|
+
this.resolutions.delete(key);
|
|
4961
|
+
}
|
|
5342
4962
|
}
|
|
5343
4963
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
5344
4964
|
async openForTarget(ref, target) {
|
|
5345
4965
|
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
5346
4966
|
return this.openGeneration(ref, target, crypto);
|
|
5347
4967
|
}
|
|
5348
|
-
openGeneration(ref, target, crypto) {
|
|
4968
|
+
openGeneration(ref, target, crypto, named) {
|
|
5349
4969
|
const genKey = {
|
|
5350
4970
|
namespace: ref.namespace,
|
|
5351
4971
|
segment: ref.segment,
|
|
@@ -5355,17 +4975,23 @@ var CrbmStorageChunkSource = class {
|
|
|
5355
4975
|
this.driver,
|
|
5356
4976
|
genKey,
|
|
5357
4977
|
{ ...this.readerOptions, crypto, lineage: target.lineage },
|
|
5358
|
-
this.keepChunkBytesUpTo
|
|
4978
|
+
this.keepChunkBytesUpTo,
|
|
4979
|
+
named
|
|
5359
4980
|
);
|
|
5360
4981
|
}
|
|
5361
4982
|
/**
|
|
5362
|
-
* Open the generation a resolution found, and hold the row's summary against the object
|
|
5363
|
-
* the
|
|
5364
|
-
*
|
|
4983
|
+
* Open the generation a resolution found, and hold the row's summary against the object: the object must be the one
|
|
4984
|
+
* the summary names by its fingerprint, and is refused before its index is opened otherwise, as another object under
|
|
4985
|
+
* the generation's number ({@link NotFoundError}, so the read resolves the row again, as for a swept generation); and
|
|
4986
|
+
* it should hold the same count and metadata, or the summary is not believed again in this process
|
|
4987
|
+
* ({@link distrusted}). Neither check costs a request. A row with no summary it can use names no object, and its
|
|
4988
|
+
* generation's object is opened as it is.
|
|
5365
4989
|
*/
|
|
5366
4990
|
async openLive(ref, live) {
|
|
5367
|
-
const
|
|
4991
|
+
const crypto = await live.crypto();
|
|
5368
4992
|
const summary = await live.summary();
|
|
4993
|
+
const reader = await this.openGeneration(ref, live.target, crypto, summary?.fingerprint);
|
|
4994
|
+
this.rowChecked.add(reader);
|
|
5369
4995
|
if (summary !== void 0 && !summaryAgrees(summary, describe(reader))) {
|
|
5370
4996
|
this.distrusted.set(this.distrustKey(ref, live.target.generation, live.target.lineage), true);
|
|
5371
4997
|
}
|
|
@@ -5382,43 +5008,70 @@ var CrbmStorageChunkSource = class {
|
|
|
5382
5008
|
return live.summary();
|
|
5383
5009
|
}
|
|
5384
5010
|
/**
|
|
5385
|
-
*
|
|
5386
|
-
* `list`
|
|
5387
|
-
*
|
|
5388
|
-
* asks.
|
|
5011
|
+
* What the segment resolves to now: the fields of the registry's authoritative record a read resolves through (one
|
|
5012
|
+
* strong read), or the highest generation a `list` of the bucket finds when there is no registry (a registry-less
|
|
5013
|
+
* source can only read cleartext: there is nowhere a wrapped DEK could live). `null` for no generation.
|
|
5389
5014
|
*/
|
|
5390
|
-
async
|
|
5391
|
-
if (this.registry !== void 0)
|
|
5392
|
-
const record = await this.registry.get(ref);
|
|
5393
|
-
if (record === null) return null;
|
|
5394
|
-
if (record.status === "destroyed") return null;
|
|
5395
|
-
if (record.currentGen === null) return null;
|
|
5396
|
-
const generation = record.currentGen;
|
|
5397
|
-
const keys = record.wrappedDeks;
|
|
5398
|
-
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
5399
|
-
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
5400
|
-
return {
|
|
5401
|
-
row: record,
|
|
5402
|
-
target: { generation, lineage: record.token, wrappedDeks: record.wrappedDeks },
|
|
5403
|
-
crypto,
|
|
5404
|
-
unwrap,
|
|
5405
|
-
// The same checks as a read of the object makes (`requireEncryption`, a keystore for an encrypted row)
|
|
5406
|
-
// come first, then the row's summary, used only if it names this generation in the shape the keys call for.
|
|
5407
|
-
summary: once(async () => usableSummary(ref, record, (await crypto())?.aead)),
|
|
5408
|
-
bytes: SNAPSHOT_BASE_BYTES + (record.summary === void 0 ? 0 : JSON.stringify(record.summary).length)
|
|
5409
|
-
};
|
|
5410
|
-
}
|
|
5015
|
+
async resolve(ref) {
|
|
5016
|
+
if (this.registry !== void 0) return resolvedOf(await this.registry.get(ref));
|
|
5411
5017
|
let maxGen = -1;
|
|
5412
5018
|
for await (const key of this.driver.list(ref)) {
|
|
5413
5019
|
if (key.generation > maxGen) maxGen = key.generation;
|
|
5414
5020
|
}
|
|
5415
|
-
|
|
5021
|
+
return maxGen < 0 ? null : { generation: maxGen };
|
|
5022
|
+
}
|
|
5023
|
+
/**
|
|
5024
|
+
* The segment resolved afresh for a pin, with the row it read, which a lease is written against. Never a kept
|
|
5025
|
+
* resolution, and kept nowhere: the generation current right now is the whole promise of a pin.
|
|
5026
|
+
*/
|
|
5027
|
+
async resolveLive(ref) {
|
|
5028
|
+
if (this.registry === void 0) {
|
|
5029
|
+
const resolved2 = await this.resolve(ref);
|
|
5030
|
+
return resolved2 === null ? null : this.liveOf(ref, resolved2, null);
|
|
5031
|
+
}
|
|
5032
|
+
const row = await this.registry.get(ref);
|
|
5033
|
+
const resolved = resolvedOf(row);
|
|
5034
|
+
return resolved === null || row === null ? null : this.liveOf(ref, resolved, null, row);
|
|
5035
|
+
}
|
|
5036
|
+
/**
|
|
5037
|
+
* What a snapshot, or a pin, makes of a resolution: its decryption context and what the row's summary says of the
|
|
5038
|
+
* generation, each made only when asked for. A snapshot that replaces `prior` with the same wrapped keys keeps the key
|
|
5039
|
+
* `prior` unwrapped, so a count polling an encrypted segment asks the keystore once, not once each `genTtlMs`.
|
|
5040
|
+
* Different keys (a shred, a re-created name) unwrap afresh, and so does a snapshot with no prior: the resolution
|
|
5041
|
+
* holds the wrapped keys, never an unwrapped one.
|
|
5042
|
+
*/
|
|
5043
|
+
liveOf(ref, resolved, prior, row) {
|
|
5044
|
+
if (this.registry === void 0) {
|
|
5045
|
+
return {
|
|
5046
|
+
target: resolved,
|
|
5047
|
+
crypto: () => Promise.resolve(void 0),
|
|
5048
|
+
unwrap: void 0,
|
|
5049
|
+
summary: () => Promise.resolve(void 0),
|
|
5050
|
+
version: () => Promise.resolve(void 0),
|
|
5051
|
+
bytes: weightOf(resolved)
|
|
5052
|
+
};
|
|
5053
|
+
}
|
|
5054
|
+
const { generation, lineage, wrappedDeks: keys, summary } = resolved;
|
|
5055
|
+
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
5056
|
+
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
5057
|
+
const described = once(
|
|
5058
|
+
async () => usableSummary(
|
|
5059
|
+
ref,
|
|
5060
|
+
{ status: "active", currentGen: generation, summary, wrappedDeks: keys },
|
|
5061
|
+
(await crypto())?.aead
|
|
5062
|
+
)
|
|
5063
|
+
);
|
|
5416
5064
|
return {
|
|
5417
|
-
target:
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5065
|
+
target: resolved,
|
|
5066
|
+
...row === void 0 ? {} : { row },
|
|
5067
|
+
crypto,
|
|
5068
|
+
unwrap,
|
|
5069
|
+
summary: described,
|
|
5070
|
+
version: once(async () => {
|
|
5071
|
+
const named = (await described())?.fingerprint;
|
|
5072
|
+
return named === void 0 ? void 0 : `${versionOf(generation, lineage)}#${compactFingerprint(named)}`;
|
|
5073
|
+
}),
|
|
5074
|
+
bytes: weightOf(resolved)
|
|
5422
5075
|
};
|
|
5423
5076
|
}
|
|
5424
5077
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
@@ -5606,24 +5259,37 @@ var CrbmStorageChunkSource = class {
|
|
|
5606
5259
|
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
5607
5260
|
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
5608
5261
|
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
5609
|
-
* With no summary it can use (
|
|
5610
|
-
* generation and answers from the object. A row that names an object that is gone
|
|
5611
|
-
* of the object throws: the number is true of the generation the row
|
|
5262
|
+
* With no summary it can use (a row written without one, one for another generation, one that does not open), it
|
|
5263
|
+
* opens the generation and answers from the object. A row that names an object that is gone, or another object
|
|
5264
|
+
* under the number, still answers, while a read of the object throws: the number is true of the generation the row
|
|
5265
|
+
* names.
|
|
5612
5266
|
*
|
|
5613
5267
|
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
5614
5268
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
5615
5269
|
*/
|
|
5616
5270
|
async summary(ref) {
|
|
5617
5271
|
validateUserRef(ref);
|
|
5272
|
+
return this.described(
|
|
5273
|
+
ref,
|
|
5274
|
+
(generation, row) => summaryOfGeneration(generation, row),
|
|
5275
|
+
(reader) => summaryOfGeneration(reader.generation, describe(reader))
|
|
5276
|
+
);
|
|
5277
|
+
}
|
|
5278
|
+
/**
|
|
5279
|
+
* What one resolution of the segment says of its generation: from the row's summary when this process may use it,
|
|
5280
|
+
* else from the generation's object, opened. `null` for no generation. One heal, as a read of the object has: a
|
|
5281
|
+
* generation swept from under the snapshot, or replaced under its number, is resolved again once.
|
|
5282
|
+
*/
|
|
5283
|
+
async described(ref, fromRow, fromObject) {
|
|
5618
5284
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
5619
5285
|
const snap = this.liveSnapshot(ref);
|
|
5620
5286
|
try {
|
|
5621
5287
|
const live = await snap.target;
|
|
5622
5288
|
if (live === null) return null;
|
|
5623
5289
|
const row = await this.trustedSummary(ref, live);
|
|
5624
|
-
if (row !== void 0) return
|
|
5290
|
+
if (row !== void 0) return fromRow(live.target.generation, row);
|
|
5625
5291
|
const reader = await snap.reader;
|
|
5626
|
-
return reader === null ? null :
|
|
5292
|
+
return reader === null ? null : fromObject(reader);
|
|
5627
5293
|
} catch (err) {
|
|
5628
5294
|
if (attempt === 1) throw err;
|
|
5629
5295
|
const checks = isNotFoundError(err) || isIntegrityError(err) || isValidationError(err);
|
|
@@ -5641,6 +5307,32 @@ var CrbmStorageChunkSource = class {
|
|
|
5641
5307
|
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5642
5308
|
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
5643
5309
|
}
|
|
5310
|
+
/**
|
|
5311
|
+
* The current generation's number, id count and metadata, with its object's size, all from one resolution. The answer
|
|
5312
|
+
* comes from the row when it can, as {@link summary}'s does: the row's summary of the generation it names, whose
|
|
5313
|
+
* fingerprint carries the object's size. That is one registry read when the snapshot is cold, none when it is warm,
|
|
5314
|
+
* and no read of the object; on an encrypted segment the key is unwrapped to open the summary, as for a count. With no
|
|
5315
|
+
* summary it can use, it opens the generation and answers from its footer and index, with no payload read: a tail read
|
|
5316
|
+
* more when cold. A row that names an object that is gone, or another object, still answers from the row: the figures
|
|
5317
|
+
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
5318
|
+
*/
|
|
5319
|
+
async stat(ref) {
|
|
5320
|
+
validateUserRef(ref);
|
|
5321
|
+
return this.described(
|
|
5322
|
+
ref,
|
|
5323
|
+
(generation, row) => ({
|
|
5324
|
+
...summaryOfGeneration(generation, row),
|
|
5325
|
+
sizeBytes: sizeOfFingerprint(row.fingerprint)
|
|
5326
|
+
}),
|
|
5327
|
+
(reader) => sizedSummary(reader)
|
|
5328
|
+
);
|
|
5329
|
+
}
|
|
5330
|
+
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5331
|
+
async statAt(ref, generation, held) {
|
|
5332
|
+
validateUserRef(ref);
|
|
5333
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
5334
|
+
return reader === null ? null : sizedSummary(reader);
|
|
5335
|
+
}
|
|
5644
5336
|
/**
|
|
5645
5337
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
5646
5338
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -5648,8 +5340,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5648
5340
|
* caller-visible error, so instead we drop the stale snapshot,
|
|
5649
5341
|
* re-resolve `currentGen`, and retry once — the read then serves the newer (committed, immutable) generation,
|
|
5650
5342
|
* a monotonic move forward — within one incarnation of the row. A name that was retired and re-created is a
|
|
5651
|
-
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
5652
|
-
* as its lineage rather than trusting the generation number (
|
|
5343
|
+
* different segment and can resolve to a LOWER generation, which is why the snapshot carries the row's
|
|
5344
|
+
* `pointerId` as its lineage rather than trusting the generation number (invariants 1 and 2).
|
|
5653
5345
|
*
|
|
5654
5346
|
* **Resolution and open are inside the retry, not before it.** Resolving `currentGen` and opening that
|
|
5655
5347
|
* generation's object are two backend round trips with a gap between them, so the object can be swept after
|
|
@@ -5779,15 +5471,30 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
5779
5471
|
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
5780
5472
|
};
|
|
5781
5473
|
}
|
|
5782
|
-
async function
|
|
5474
|
+
async function provenObject(storage, key, fingerprint) {
|
|
5475
|
+
let version;
|
|
5783
5476
|
try {
|
|
5784
|
-
|
|
5477
|
+
const blob = storageBlobReader(storage, key, (tail) => {
|
|
5478
|
+
version ??= tail.version;
|
|
5479
|
+
});
|
|
5480
|
+
return await CrbmReader.sameObject(blob, fingerprint) ? { version } : null;
|
|
5785
5481
|
} catch {
|
|
5786
|
-
return
|
|
5482
|
+
return null;
|
|
5787
5483
|
}
|
|
5788
5484
|
}
|
|
5789
|
-
function
|
|
5790
|
-
return
|
|
5485
|
+
function sameObjectUnder(storage, key, fingerprint) {
|
|
5486
|
+
return CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
5487
|
+
}
|
|
5488
|
+
function objectIsEncrypted(storage, key, noteVersion) {
|
|
5489
|
+
return footerSaysEncrypted(
|
|
5490
|
+
storageBlobReader(
|
|
5491
|
+
storage,
|
|
5492
|
+
key,
|
|
5493
|
+
noteVersion === void 0 ? void 0 : (tail) => {
|
|
5494
|
+
if (tail.version !== void 0) noteVersion(tail.version);
|
|
5495
|
+
}
|
|
5496
|
+
)
|
|
5497
|
+
);
|
|
5791
5498
|
}
|
|
5792
5499
|
function rememberingTail(blob) {
|
|
5793
5500
|
let first;
|
|
@@ -5861,7 +5568,7 @@ function refuseCleartextOntoKey(key, record, cleartext) {
|
|
|
5861
5568
|
`segment "${key.segment}" is encrypted, and generation ${key.generation} was written in cleartext: another writer created the segment with a key while this one was writing. Nothing was published. Re-run the write with the keystore that holds the segment's key.`
|
|
5862
5569
|
);
|
|
5863
5570
|
}
|
|
5864
|
-
var
|
|
5571
|
+
var UNANSWERED_RESENDS2 = 3;
|
|
5865
5572
|
var UNANSWERED_RESEND_BASE_MS = 500;
|
|
5866
5573
|
function leasesPatch(record, options) {
|
|
5867
5574
|
const held = record.leases;
|
|
@@ -5903,7 +5610,8 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5903
5610
|
if (landed)
|
|
5904
5611
|
return {
|
|
5905
5612
|
published: true,
|
|
5906
|
-
kept: { ...nextKept(failedOn, key.generation, options.keep), token: void 0 }
|
|
5613
|
+
kept: { ...nextKept(failedOn, key.generation, options.keep), token: void 0 },
|
|
5614
|
+
token: record?.token
|
|
5907
5615
|
};
|
|
5908
5616
|
if (record !== null && record.status === "active" && record.currentGen === key.generation) {
|
|
5909
5617
|
return REFUSED;
|
|
@@ -5971,7 +5679,7 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5971
5679
|
return REFUSED;
|
|
5972
5680
|
} else if (record.currentGen === key.generation) {
|
|
5973
5681
|
if (reused) continue;
|
|
5974
|
-
return { published: true };
|
|
5682
|
+
return { published: true, token: record.token };
|
|
5975
5683
|
} else {
|
|
5976
5684
|
if (options.wrappedDeks !== void 0 && options.wrappedDeks.length > 0) {
|
|
5977
5685
|
throw new ValidationError(
|
|
@@ -5994,7 +5702,8 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
5994
5702
|
}
|
|
5995
5703
|
return {
|
|
5996
5704
|
published: true,
|
|
5997
|
-
kept: { ...next ?? { list: void 0, evict: [] }, token: wrote }
|
|
5705
|
+
kept: { ...next ?? { list: void 0, evict: [] }, token: wrote },
|
|
5706
|
+
token: wrote
|
|
5998
5707
|
};
|
|
5999
5708
|
} catch (err) {
|
|
6000
5709
|
if (isWriteConflictError(err)) {
|
|
@@ -6014,7 +5723,7 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
6014
5723
|
}
|
|
6015
5724
|
if (sameRow(now, record)) {
|
|
6016
5725
|
const sleep = options.clock?.sleep;
|
|
6017
|
-
if (sleep === void 0 || resends >=
|
|
5726
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS2) throw outcomeUnknown(key, err);
|
|
6018
5727
|
const bound = UNANSWERED_RESEND_BASE_MS * 2 ** resends;
|
|
6019
5728
|
resends += 1;
|
|
6020
5729
|
await sleep.call(options.clock, Math.floor((options.rng?.next() ?? 1) * bound));
|
|
@@ -6029,11 +5738,10 @@ async function publishGenerationKept(registry, key, options = {}) {
|
|
|
6029
5738
|
const final = await registry.get(key);
|
|
6030
5739
|
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation) {
|
|
6031
5740
|
if (final.currentGen !== key.generation) return REFUSED;
|
|
6032
|
-
if (!sawUnanswered) return { published: true };
|
|
5741
|
+
if (!sawUnanswered) return { published: true, token: final.token };
|
|
6033
5742
|
try {
|
|
6034
|
-
|
|
6035
|
-
|
|
6036
|
-
};
|
|
5743
|
+
const published = await landedHere(final, lastRecord ?? null, key, options.holdsOwnObject);
|
|
5744
|
+
return published ? { published, token: final.token } : REFUSED;
|
|
6037
5745
|
} catch (proofErr) {
|
|
6038
5746
|
throw outcomeUnknown(key, proofErr);
|
|
6039
5747
|
}
|
|
@@ -6133,22 +5841,24 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6133
5841
|
const summary = summaryOf(
|
|
6134
5842
|
key,
|
|
6135
5843
|
key.generation,
|
|
6136
|
-
{ cardinality, metadata: options.metadata },
|
|
5844
|
+
{ cardinality, metadata: options.metadata, fingerprint },
|
|
6137
5845
|
crypto?.aead
|
|
6138
5846
|
);
|
|
6139
5847
|
if (options.registry !== void 0 && options.publish !== false) {
|
|
6140
|
-
const
|
|
5848
|
+
const published = await publishGenerationKept(options.registry, key, {
|
|
6141
5849
|
wrappedDeks: newWrapped,
|
|
6142
5850
|
summary,
|
|
6143
5851
|
cleartext: crypto === void 0,
|
|
6144
5852
|
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
6145
5853
|
clock: options.clock
|
|
6146
5854
|
});
|
|
5855
|
+
const becameCurrent = published.published;
|
|
6147
5856
|
if (becameCurrent) {
|
|
6148
5857
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
6149
5858
|
kind: "segment.publish",
|
|
6150
5859
|
namespace: key.namespace,
|
|
6151
5860
|
segment: key.segment,
|
|
5861
|
+
...incarnationField(published.token),
|
|
6152
5862
|
generation: key.generation
|
|
6153
5863
|
});
|
|
6154
5864
|
}
|
|
@@ -6261,15 +5971,72 @@ async function bucketIds(ids, codec, clock) {
|
|
|
6261
5971
|
await flushPending();
|
|
6262
5972
|
return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
|
|
6263
5973
|
}
|
|
6264
|
-
|
|
6265
|
-
|
|
5974
|
+
var openedVersions = /* @__PURE__ */ new WeakMap();
|
|
5975
|
+
function openGenerationReader(storage, key, crypto, options = {}, named) {
|
|
5976
|
+
return openNoting(storage, key, crypto, options, named);
|
|
5977
|
+
}
|
|
5978
|
+
function openGenerationReaderNoting(storage, key, crypto, noteVersion) {
|
|
5979
|
+
return openNoting(storage, key, crypto, {}, void 0, noteVersion);
|
|
6266
5980
|
}
|
|
6267
|
-
async function
|
|
6268
|
-
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
5981
|
+
async function openNoting(storage, key, crypto, options, named, noteVersion) {
|
|
5982
|
+
let version;
|
|
5983
|
+
const reader = await openChecked(
|
|
5984
|
+
storage,
|
|
5985
|
+
key,
|
|
5986
|
+
{ ...options, crypto },
|
|
5987
|
+
void 0,
|
|
5988
|
+
named,
|
|
5989
|
+
(tail) => {
|
|
5990
|
+
if (version !== void 0 || tail.version === void 0) return;
|
|
5991
|
+
version = tail.version;
|
|
5992
|
+
noteVersion?.(version);
|
|
5993
|
+
}
|
|
6272
5994
|
);
|
|
5995
|
+
if (version !== void 0) openedVersions.set(reader, version);
|
|
5996
|
+
return reader;
|
|
5997
|
+
}
|
|
5998
|
+
function objectVersionOf(reader) {
|
|
5999
|
+
return openedVersions.get(reader);
|
|
6000
|
+
}
|
|
6001
|
+
async function openNamedGeneration(storage, key, crypto, fingerprint) {
|
|
6002
|
+
return openHeldTo(storage, key, { crypto }, void 0, fingerprint);
|
|
6003
|
+
}
|
|
6004
|
+
async function openChecked(storage, key, options, keepChunkBytesUpTo, named, noteTail) {
|
|
6005
|
+
const reader = await openHeldTo(storage, key, options, keepChunkBytesUpTo, named, noteTail);
|
|
6006
|
+
if (reader === void 0) {
|
|
6007
|
+
throw new NotFoundError(
|
|
6008
|
+
`segment "${key.segment}" generation ${key.generation} is another object than its registry row names`
|
|
6009
|
+
);
|
|
6010
|
+
}
|
|
6011
|
+
return reader;
|
|
6012
|
+
}
|
|
6013
|
+
async function openHeldTo(storage, key, options, keepChunkBytesUpTo, named, noteTail) {
|
|
6014
|
+
const raw = storageBlobReader(storage, key, noteTail);
|
|
6015
|
+
let reader;
|
|
6016
|
+
if (named === void 0) {
|
|
6017
|
+
reader = await openCrbmReaderKeeping(raw, options, keepChunkBytesUpTo);
|
|
6018
|
+
} else {
|
|
6019
|
+
let tail = await raw.getTail(
|
|
6020
|
+
Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES)
|
|
6021
|
+
);
|
|
6022
|
+
const blob = {
|
|
6023
|
+
getRange: (offset, length) => raw.getRange(offset, length),
|
|
6024
|
+
getTail: async (maxBytes) => {
|
|
6025
|
+
const held = tail;
|
|
6026
|
+
if (held !== void 0 && (maxBytes <= held.bytes.length || held.bytes.length === held.size)) {
|
|
6027
|
+
const take = Math.min(maxBytes, held.bytes.length);
|
|
6028
|
+
return { bytes: held.bytes.subarray(held.bytes.length - take), size: held.size };
|
|
6029
|
+
}
|
|
6030
|
+
return raw.getTail(maxBytes);
|
|
6031
|
+
}
|
|
6032
|
+
};
|
|
6033
|
+
try {
|
|
6034
|
+
if (!await CrbmReader.sameObject(blob, named)) return void 0;
|
|
6035
|
+
reader = await openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo);
|
|
6036
|
+
} finally {
|
|
6037
|
+
tail = void 0;
|
|
6038
|
+
}
|
|
6039
|
+
}
|
|
6273
6040
|
if (reader.generation !== key.generation) {
|
|
6274
6041
|
throw new IntegrityError(
|
|
6275
6042
|
`segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
|
|
@@ -6311,10 +6078,6 @@ var PinnedStorageChunkSource = class {
|
|
|
6311
6078
|
}
|
|
6312
6079
|
inner;
|
|
6313
6080
|
pins;
|
|
6314
|
-
/** The unpinned segments refresh as the store's own source does; a pinned one never does. */
|
|
6315
|
-
get pointerRefreshMs() {
|
|
6316
|
-
return this.inner.pointerRefreshMs;
|
|
6317
|
-
}
|
|
6318
6081
|
/**
|
|
6319
6082
|
* The pin for this segment, or `undefined` if it is not pinned here.
|
|
6320
6083
|
*
|
|
@@ -6339,76 +6102,250 @@ var PinnedStorageChunkSource = class {
|
|
|
6339
6102
|
if (pin === void 0) return this.inner.getChunks(ref, keys, options);
|
|
6340
6103
|
return this.pinnedChunks(ref, pin, keys, options);
|
|
6341
6104
|
}
|
|
6342
|
-
async *pinnedChunks(ref, pin, keys, options) {
|
|
6343
|
-
if (pin.generation === null) {
|
|
6344
|
-
for (const key of keys) yield { key, bytes: null, version: null };
|
|
6345
|
-
return;
|
|
6105
|
+
async *pinnedChunks(ref, pin, keys, options) {
|
|
6106
|
+
if (pin.generation === null) {
|
|
6107
|
+
for (const key of keys) yield { key, bytes: null, version: null };
|
|
6108
|
+
return;
|
|
6109
|
+
}
|
|
6110
|
+
const version = await this.currentVersion(ref);
|
|
6111
|
+
const pull = new ItemPull(
|
|
6112
|
+
this.inner.getChunksAt(ref, pin.generation, keys, heldBy(pin), options)
|
|
6113
|
+
);
|
|
6114
|
+
try {
|
|
6115
|
+
while (await pull.advance()) yield pull.take((chunk) => ({ ...chunk, version }));
|
|
6116
|
+
} finally {
|
|
6117
|
+
await pull.close();
|
|
6118
|
+
}
|
|
6119
|
+
}
|
|
6120
|
+
listChunkKeys(ref) {
|
|
6121
|
+
const pin = this.pinFor(ref);
|
|
6122
|
+
if (pin === void 0) return this.inner.listChunkKeys(ref);
|
|
6123
|
+
return pin.generation === null ? Promise.resolve([]) : this.inner.listChunkKeysAt(ref, pin.generation, heldBy(pin));
|
|
6124
|
+
}
|
|
6125
|
+
sizeOf(ref) {
|
|
6126
|
+
const pin = this.pinFor(ref);
|
|
6127
|
+
if (pin === void 0) return this.inner.sizeOf(ref);
|
|
6128
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.sizeOfAt(ref, pin.generation, heldBy(pin));
|
|
6129
|
+
}
|
|
6130
|
+
cardinalities(ref) {
|
|
6131
|
+
const pin = this.pinFor(ref);
|
|
6132
|
+
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
6133
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
6134
|
+
}
|
|
6135
|
+
summary(ref) {
|
|
6136
|
+
const pin = this.pinFor(ref);
|
|
6137
|
+
if (pin === void 0) return this.inner.summary(ref);
|
|
6138
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
6139
|
+
}
|
|
6140
|
+
stat(ref) {
|
|
6141
|
+
const pin = this.pinFor(ref);
|
|
6142
|
+
if (pin === void 0) return this.inner.stat(ref);
|
|
6143
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.statAt(ref, pin.generation, heldBy(pin));
|
|
6144
|
+
}
|
|
6145
|
+
currentGeneration(ref) {
|
|
6146
|
+
const pin = this.pinFor(ref);
|
|
6147
|
+
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
6148
|
+
}
|
|
6149
|
+
/**
|
|
6150
|
+
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
6151
|
+
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
6152
|
+
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
6153
|
+
* and a lapsed `cache.genTtlMs`, a reader-cache eviction on a store with no timed refresh, the resolution
|
|
6154
|
+
* cache letting the segment's resolution go on a store with one, a sweep that heals the read forward, or an
|
|
6155
|
+
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
6156
|
+
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
6157
|
+
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
6158
|
+
* read fetches exactly its own generation, so the entries it fills are always that generation's.
|
|
6159
|
+
*
|
|
6160
|
+
* The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
|
|
6161
|
+
* live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
|
|
6162
|
+
* though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
|
|
6163
|
+
* the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
|
|
6164
|
+
* key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
|
|
6165
|
+
*/
|
|
6166
|
+
currentVersion(ref) {
|
|
6167
|
+
const pin = this.pinFor(ref);
|
|
6168
|
+
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
6169
|
+
if (pin.version === null) return Promise.resolve(null);
|
|
6170
|
+
return Promise.resolve(
|
|
6171
|
+
`pin ${pin.version}${pin.fingerprint == null ? "" : `#${pin.fingerprint}`}`
|
|
6172
|
+
);
|
|
6173
|
+
}
|
|
6174
|
+
exists(ref) {
|
|
6175
|
+
return this.inner.exists(ref);
|
|
6176
|
+
}
|
|
6177
|
+
invalidate(ref) {
|
|
6178
|
+
this.inner.invalidate(ref);
|
|
6179
|
+
}
|
|
6180
|
+
};
|
|
6181
|
+
|
|
6182
|
+
// src/core/generation-gc.ts
|
|
6183
|
+
async function nextGeneration(ref, deps) {
|
|
6184
|
+
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
6185
|
+
}
|
|
6186
|
+
async function aboveEverything(ref, deps, record) {
|
|
6187
|
+
let highest = record?.currentGen ?? -1;
|
|
6188
|
+
for await (const key of deps.storage.list(ref)) {
|
|
6189
|
+
if (key.generation > highest) highest = key.generation;
|
|
6190
|
+
}
|
|
6191
|
+
return numberAbove(highest);
|
|
6192
|
+
}
|
|
6193
|
+
function numberAbove(highest) {
|
|
6194
|
+
if (highest >= Number.MAX_SAFE_INTEGER) {
|
|
6195
|
+
throw new IntegrityError("the segment has no generation number left above its highest");
|
|
6196
|
+
}
|
|
6197
|
+
return highest + 1;
|
|
6198
|
+
}
|
|
6199
|
+
async function nextLoadGeneration(ref, deps, row) {
|
|
6200
|
+
const generation = numberAbove(row?.currentGen ?? -1);
|
|
6201
|
+
try {
|
|
6202
|
+
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
6203
|
+
} catch (err) {
|
|
6204
|
+
if (isNotFoundError(err)) return { generation, checked: true };
|
|
6205
|
+
}
|
|
6206
|
+
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
6207
|
+
}
|
|
6208
|
+
var LIST_COLLECTION_CADENCE = 16;
|
|
6209
|
+
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
6210
|
+
return (await listingPass(ref, deps, { keep: options.keep })).deleted;
|
|
6211
|
+
}
|
|
6212
|
+
async function listingPass(ref, deps, options) {
|
|
6213
|
+
const keep = options.keep ?? 1;
|
|
6214
|
+
const quiet = options.quiet === true;
|
|
6215
|
+
if (!Number.isInteger(keep) || keep < 0) {
|
|
6216
|
+
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
6217
|
+
}
|
|
6218
|
+
const record = await deps.registry.get(ref);
|
|
6219
|
+
if (record === null) return { deleted: [], kept: [], complete: true };
|
|
6220
|
+
const current = record.currentGen;
|
|
6221
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6222
|
+
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
6223
|
+
const gens = [...seen];
|
|
6224
|
+
const after = await deps.registry.get(ref);
|
|
6225
|
+
if (after === null) {
|
|
6226
|
+
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
6227
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
6228
|
+
}
|
|
6229
|
+
if (record.status === "destroyed" && after.token !== record.token) {
|
|
6230
|
+
if (quiet) return { deleted: [], kept: [], complete: false };
|
|
6231
|
+
throw new WriteConflictError(
|
|
6232
|
+
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
6233
|
+
);
|
|
6234
|
+
}
|
|
6235
|
+
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
6236
|
+
const named = record.status === "destroyed" ? void 0 : usableKeptGens(after);
|
|
6237
|
+
const protect = options.protect === void 0 || named === void 0 ? void 0 : /* @__PURE__ */ new Set([...options.protect, ...named]);
|
|
6238
|
+
const stillCollectable = async (generation) => {
|
|
6239
|
+
const still = await deps.registry.get(ref);
|
|
6240
|
+
if (still === null) {
|
|
6241
|
+
if (quiet) return "stop";
|
|
6242
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
6243
|
+
}
|
|
6244
|
+
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
6245
|
+
const now = protect === void 0 ? void 0 : usableKeptGens(still);
|
|
6246
|
+
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
6247
|
+
if (!ok || !unnamed) {
|
|
6248
|
+
if (quiet) return "stop";
|
|
6249
|
+
throw new WriteConflictError(
|
|
6250
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
6251
|
+
);
|
|
6252
|
+
}
|
|
6253
|
+
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
6254
|
+
return "delete";
|
|
6255
|
+
};
|
|
6256
|
+
const guard = record.status === "destroyed" ? void 0 : options.leases;
|
|
6257
|
+
const held = guard === void 0 ? void 0 : heldGenerations(after, guard.now());
|
|
6258
|
+
const below = record.status === "destroyed" || cutoff === null ? [] : gens.filter((g) => g < cutoff).sort((a, b) => b - a);
|
|
6259
|
+
const candidates = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
6260
|
+
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
6261
|
+
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
6262
|
+
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
6263
|
+
[]
|
|
6264
|
+
) : protect !== void 0 ? (
|
|
6265
|
+
// The row's list is the window: everything below the pointer it does not name is garbage.
|
|
6266
|
+
below.filter((g) => !protect.has(g))
|
|
6267
|
+
) : (
|
|
6268
|
+
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
6269
|
+
below.slice(keep)
|
|
6270
|
+
);
|
|
6271
|
+
const leasedSpared = new Set(candidates.filter((g) => held?.has(g) === true));
|
|
6272
|
+
const toDelete = candidates.filter((g) => !leasedSpared.has(g));
|
|
6273
|
+
const done = [];
|
|
6274
|
+
for (const generation of toDelete) {
|
|
6275
|
+
const verdict = await stillCollectable(generation);
|
|
6276
|
+
if (verdict === "stop") return { deleted: done, kept: [], complete: false };
|
|
6277
|
+
if (verdict === "skip") {
|
|
6278
|
+
leasedSpared.add(generation);
|
|
6279
|
+
continue;
|
|
6280
|
+
}
|
|
6281
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
6282
|
+
done.push(generation);
|
|
6283
|
+
}
|
|
6284
|
+
const deleted = new Set(done);
|
|
6285
|
+
return {
|
|
6286
|
+
deleted: done,
|
|
6287
|
+
kept: below.filter((g) => !deleted.has(g) && !leasedSpared.has(g)).reverse(),
|
|
6288
|
+
complete: true
|
|
6289
|
+
};
|
|
6290
|
+
}
|
|
6291
|
+
async function collectAfterLoad(ref, deps, options) {
|
|
6292
|
+
const { generation, keep, byName, currentGone, kept, proveCurrent, leases } = options;
|
|
6293
|
+
if (keep >= generation) return [];
|
|
6294
|
+
const list = kept?.list;
|
|
6295
|
+
if (list === void 0 || currentGone === true)
|
|
6296
|
+
return reconcile(ref, deps, { keep, kept, leases });
|
|
6297
|
+
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
6298
|
+
if (byName && !periodic) {
|
|
6299
|
+
if (keep >= 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
6300
|
+
return reconcile(ref, deps, { keep, kept, leases });
|
|
6346
6301
|
}
|
|
6347
|
-
|
|
6348
|
-
|
|
6349
|
-
|
|
6350
|
-
|
|
6302
|
+
return deleteEvicted(ref, deps, { generation, evict: kept.evict, leases });
|
|
6303
|
+
}
|
|
6304
|
+
return (await listingPass(ref, deps, { keep, protect: list, quiet: true, leases })).deleted;
|
|
6305
|
+
}
|
|
6306
|
+
async function reconcile(ref, deps, options) {
|
|
6307
|
+
const { keep, kept, leases } = options;
|
|
6308
|
+
const {
|
|
6309
|
+
deleted,
|
|
6310
|
+
kept: present,
|
|
6311
|
+
complete
|
|
6312
|
+
} = await listingPass(ref, deps, { keep, quiet: true, leases });
|
|
6313
|
+
const token = kept?.token;
|
|
6314
|
+
if (complete && token !== void 0 && keep <= MAX_KEPT_GENERATIONS && !sameList(kept?.list, present)) {
|
|
6351
6315
|
try {
|
|
6352
|
-
|
|
6353
|
-
}
|
|
6354
|
-
|
|
6316
|
+
await deps.registry.compareAndSwap(ref, token, { keptGens: present });
|
|
6317
|
+
} catch (err) {
|
|
6318
|
+
if (!isWriteConflictError(err)) throw err;
|
|
6355
6319
|
}
|
|
6356
6320
|
}
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
return
|
|
6366
|
-
}
|
|
6367
|
-
|
|
6368
|
-
const pin = this.pinFor(ref);
|
|
6369
|
-
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
6370
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
6371
|
-
}
|
|
6372
|
-
summary(ref) {
|
|
6373
|
-
const pin = this.pinFor(ref);
|
|
6374
|
-
if (pin === void 0) return this.inner.summary(ref);
|
|
6375
|
-
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
6376
|
-
}
|
|
6377
|
-
currentGeneration(ref) {
|
|
6378
|
-
const pin = this.pinFor(ref);
|
|
6379
|
-
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
6380
|
-
}
|
|
6381
|
-
/**
|
|
6382
|
-
* A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks
|
|
6383
|
-
* are cached under keys that no live read writes. A live read keys its fetches by the version it resolved when
|
|
6384
|
-
* it began, but is served whatever generation the live source holds when each fetch lands — after a publish
|
|
6385
|
-
* and a lapsed `cache.genTtlMs`, a reader-cache eviction, a sweep that heals the read forward, or an
|
|
6386
|
-
* invalidation, a different one — so an entry under a live version can hold another generation's chunk.
|
|
6387
|
-
* Invariant 3 lets that live call see it, and later live reads resolve the current version and never look the
|
|
6388
|
-
* entry up; a pin sharing the key would be handed it, and return a read that mixed two generations. A pinned
|
|
6389
|
-
* read fetches exactly its own generation, so the entries it fills are always that generation's.
|
|
6390
|
-
*
|
|
6391
|
-
* The pin still shares the store's chunk cache, and its memory ceiling; what it gives up is a hit on a chunk a
|
|
6392
|
-
* live read of the same version cached, which costs a pin one GET per such chunk. Live reads make no call for it,
|
|
6393
|
-
* though a pin's entries share the cache's bound with theirs, so under a small `cache.maxChunks` each can evict
|
|
6394
|
-
* the other. The version, not the bare generation, is still what the key carries: sharing on a generation-only
|
|
6395
|
-
* key was how a pinned read could resurrect an id that `eraseIdFromSegment` had reported physically gone.
|
|
6396
|
-
*/
|
|
6397
|
-
currentVersion(ref) {
|
|
6398
|
-
const pin = this.pinFor(ref);
|
|
6399
|
-
if (pin === void 0) return this.inner.currentVersion(ref);
|
|
6400
|
-
if (pin.version === null) return Promise.resolve(null);
|
|
6401
|
-
return Promise.resolve(
|
|
6402
|
-
`pin ${pin.version}${pin.fingerprint == null ? "" : `#${pin.fingerprint}`}`
|
|
6403
|
-
);
|
|
6404
|
-
}
|
|
6405
|
-
exists(ref) {
|
|
6406
|
-
return this.inner.exists(ref);
|
|
6321
|
+
return deleted;
|
|
6322
|
+
}
|
|
6323
|
+
function sameList(a, b) {
|
|
6324
|
+
return a !== void 0 && a.length === b.length && a.every((g, i) => g === b[i]);
|
|
6325
|
+
}
|
|
6326
|
+
async function objectIsThere(storage, key) {
|
|
6327
|
+
try {
|
|
6328
|
+
await storage.getTail(key, 0);
|
|
6329
|
+
return true;
|
|
6330
|
+
} catch {
|
|
6331
|
+
return false;
|
|
6407
6332
|
}
|
|
6408
|
-
|
|
6409
|
-
|
|
6333
|
+
}
|
|
6334
|
+
async function deleteEvicted(ref, deps, options) {
|
|
6335
|
+
const deleted = [];
|
|
6336
|
+
for (const generation of options.evict) {
|
|
6337
|
+
const still = await deps.registry.get(ref);
|
|
6338
|
+
if (still === null || still.currentGen === null || still.currentGen < options.generation) break;
|
|
6339
|
+
const named = usableKeptGens(still);
|
|
6340
|
+
if (named === void 0 || generation >= still.currentGen || named.includes(generation)) break;
|
|
6341
|
+
if (options.leases !== void 0 && heldGenerations(still, options.leases.now()).has(generation)) {
|
|
6342
|
+
continue;
|
|
6343
|
+
}
|
|
6344
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
6345
|
+
deleted.push(generation);
|
|
6410
6346
|
}
|
|
6411
|
-
|
|
6347
|
+
return deleted;
|
|
6348
|
+
}
|
|
6412
6349
|
|
|
6413
6350
|
// src/core/retry.ts
|
|
6414
6351
|
var DEFAULT_RETRY_POLICY = {
|
|
@@ -6482,10 +6419,10 @@ async function withRetry(op, policy, deps) {
|
|
|
6482
6419
|
// src/core/load.ts
|
|
6483
6420
|
function checkedGuard(guard, where) {
|
|
6484
6421
|
if (guard === void 0 || guard === null) return void 0;
|
|
6485
|
-
const
|
|
6422
|
+
const shown = (v) => typeof v === "number" ? String(v) : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
6486
6423
|
if (typeof guard !== "object" || Array.isArray(guard)) {
|
|
6487
6424
|
throw new ValidationError(
|
|
6488
|
-
`${where}guard must be an object of bounds, such as { minRetained: 0.5 }; got ${
|
|
6425
|
+
`${where}guard must be an object of bounds, such as { minRetained: 0.5 }; got ${shown(guard)}`
|
|
6489
6426
|
);
|
|
6490
6427
|
}
|
|
6491
6428
|
const bounds = guard;
|
|
@@ -6500,17 +6437,17 @@ function checkedGuard(guard, where) {
|
|
|
6500
6437
|
const { minCardinality, minRetained, maxGrowth } = guard;
|
|
6501
6438
|
if (minCardinality !== void 0 && (!Number.isInteger(minCardinality) || minCardinality < 0)) {
|
|
6502
6439
|
throw new ValidationError(
|
|
6503
|
-
`${where}guard.minCardinality must be a non-negative integer; got ${
|
|
6440
|
+
`${where}guard.minCardinality must be a non-negative integer; got ${shown(minCardinality)}`
|
|
6504
6441
|
);
|
|
6505
6442
|
}
|
|
6506
6443
|
if (minRetained !== void 0 && (!Number.isFinite(minRetained) || minRetained < 0 || minRetained > 1)) {
|
|
6507
6444
|
throw new ValidationError(
|
|
6508
|
-
`${where}guard.minRetained must be a fraction in 0..1; got ${
|
|
6445
|
+
`${where}guard.minRetained must be a fraction in 0..1; got ${shown(minRetained)}`
|
|
6509
6446
|
);
|
|
6510
6447
|
}
|
|
6511
6448
|
if (maxGrowth !== void 0 && (!Number.isFinite(maxGrowth) || maxGrowth !== 0 && maxGrowth < 1)) {
|
|
6512
6449
|
throw new ValidationError(
|
|
6513
|
-
`${where}guard.maxGrowth must be 0 (no bound) or a factor of at least 1; got ${
|
|
6450
|
+
`${where}guard.maxGrowth must be 0 (no bound) or a factor of at least 1; got ${shown(maxGrowth)}`
|
|
6514
6451
|
);
|
|
6515
6452
|
}
|
|
6516
6453
|
return Object.freeze({
|
|
@@ -6686,6 +6623,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6686
6623
|
kind: "segment.load-refused",
|
|
6687
6624
|
segment: ref.segment,
|
|
6688
6625
|
namespace: ref.namespace,
|
|
6626
|
+
...incarnationField(fromToken),
|
|
6689
6627
|
generation: generation2,
|
|
6690
6628
|
reason: "superseded",
|
|
6691
6629
|
cardinality: 0
|
|
@@ -6711,9 +6649,9 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6711
6649
|
return;
|
|
6712
6650
|
}
|
|
6713
6651
|
const keyed = now !== null && now.wrappedDeks !== void 0 && now.wrappedDeks.length > 0;
|
|
6714
|
-
if (
|
|
6715
|
-
|
|
6716
|
-
}
|
|
6652
|
+
if (now !== null && !(keyed && !written.encrypted)) return;
|
|
6653
|
+
const own = await provenObject(deps.storage, key, written.fingerprint);
|
|
6654
|
+
if (own !== null) await deps.storage.delete(key, { ifVersion: own.version });
|
|
6717
6655
|
};
|
|
6718
6656
|
let unanswered = false;
|
|
6719
6657
|
const tidy = () => reclaim().catch(() => void 0);
|
|
@@ -6723,6 +6661,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6723
6661
|
kind: "segment.load-refused",
|
|
6724
6662
|
segment: ref.segment,
|
|
6725
6663
|
namespace: ref.namespace,
|
|
6664
|
+
...incarnationField(fromToken),
|
|
6726
6665
|
generation,
|
|
6727
6666
|
reason,
|
|
6728
6667
|
cardinality: written.cardinality,
|
|
@@ -6781,6 +6720,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6781
6720
|
kind: "segment.publish",
|
|
6782
6721
|
segment: ref.segment,
|
|
6783
6722
|
namespace: ref.namespace,
|
|
6723
|
+
...incarnationField(published.token),
|
|
6784
6724
|
generation
|
|
6785
6725
|
});
|
|
6786
6726
|
const currentObjectGone = needsBefore && before === null;
|
|
@@ -8950,7 +8890,11 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8950
8890
|
const summary = reader === void 0 ? void 0 : summaryOf(
|
|
8951
8891
|
ref,
|
|
8952
8892
|
toGeneration,
|
|
8953
|
-
{
|
|
8893
|
+
{
|
|
8894
|
+
cardinality: reader.count(),
|
|
8895
|
+
metadata: reader.metadata,
|
|
8896
|
+
fingerprint: reader.fingerprint
|
|
8897
|
+
},
|
|
8954
8898
|
aead
|
|
8955
8899
|
);
|
|
8956
8900
|
const churn = leaseChurn(deps);
|
|
@@ -9005,7 +8949,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9005
8949
|
break;
|
|
9006
8950
|
} catch (err) {
|
|
9007
8951
|
if (!isWriteConflictError(err)) break;
|
|
9008
|
-
const moved = ["currentGen", "summary", "keptGens"];
|
|
8952
|
+
const moved = ["currentGen", "summary", "keptGens", "pointerId"];
|
|
9009
8953
|
let now;
|
|
9010
8954
|
try {
|
|
9011
8955
|
const seen = await deps.registry.get(ref);
|
|
@@ -9015,7 +8959,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9015
8959
|
} catch {
|
|
9016
8960
|
break;
|
|
9017
8961
|
}
|
|
9018
|
-
if (now === null || now.currentGen !== toGeneration || !onlyLeasesDiffer(record, now, moved))
|
|
8962
|
+
if (now === null || now.currentGen !== toGeneration || now.pointerId !== token || !onlyLeasesDiffer(record, now, moved))
|
|
9019
8963
|
break;
|
|
9020
8964
|
undoAgainst = now.token;
|
|
9021
8965
|
}
|
|
@@ -9028,6 +8972,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9028
8972
|
kind: "segment.rollback",
|
|
9029
8973
|
segment: ref.segment,
|
|
9030
8974
|
namespace: ref.namespace,
|
|
8975
|
+
...incarnationField(record.token),
|
|
9031
8976
|
fromGeneration: record.currentGen,
|
|
9032
8977
|
generation: toGeneration
|
|
9033
8978
|
});
|
|
@@ -9046,12 +8991,27 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
9046
8991
|
const first = {};
|
|
9047
8992
|
const result = await eraseOnce(ref, id, deps, options, first);
|
|
9048
8993
|
if (result.reason === "destroyed" && first.tombstoned === false) {
|
|
9049
|
-
const
|
|
8994
|
+
const second = {};
|
|
8995
|
+
const again = await eraseOnce(ref, id, deps, options, second);
|
|
9050
8996
|
if (again.reason === "destroyed") return result;
|
|
9051
8997
|
if (!again.erased) return again;
|
|
9052
8998
|
const collected = [.../* @__PURE__ */ new Set([...result.collected, ...again.collected])].sort((a, b) => a - b);
|
|
9053
8999
|
const newest = Math.max(result.fromGeneration ?? -1, again.fromGeneration ?? -1);
|
|
9054
|
-
|
|
9000
|
+
const merged = { ...again, collected, ...newest >= 0 ? { fromGeneration: newest } : {} };
|
|
9001
|
+
return recordCollect(ref, merged, second, options);
|
|
9002
|
+
}
|
|
9003
|
+
return recordCollect(ref, result, first, options);
|
|
9004
|
+
}
|
|
9005
|
+
function recordCollect(ref, result, pass, options) {
|
|
9006
|
+
if (pass.collectedOn !== void 0) {
|
|
9007
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9008
|
+
kind: "segment.collect",
|
|
9009
|
+
namespace: ref.namespace,
|
|
9010
|
+
segment: ref.segment,
|
|
9011
|
+
...incarnationField(pass.collectedOn),
|
|
9012
|
+
...pass.found === true && result.fromGeneration !== void 0 ? { fromGeneration: result.fromGeneration } : {},
|
|
9013
|
+
collected: [...result.collected]
|
|
9014
|
+
});
|
|
9055
9015
|
}
|
|
9056
9016
|
return result;
|
|
9057
9017
|
}
|
|
@@ -9062,6 +9022,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9062
9022
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
9063
9023
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
9064
9024
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9025
|
+
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
9065
9026
|
const read = (op) => retryRead(op, deps.readRetry);
|
|
9066
9027
|
const record = await deps.registry.get(ref);
|
|
9067
9028
|
if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
|
|
@@ -9097,30 +9058,62 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9097
9058
|
const rowVerdict = (row) => {
|
|
9098
9059
|
if (row === null) return "absent";
|
|
9099
9060
|
if (row.status === "destroyed") return "destroyed";
|
|
9061
|
+
const same = row.token === premiseToken || onlyLeasesDiffer(premise, row);
|
|
9062
|
+
if (pointerless) return row.currentGen === null && same ? null : "superseded";
|
|
9100
9063
|
if (row.currentGen === null) return "no-generation";
|
|
9101
|
-
return row.currentGen === from &&
|
|
9064
|
+
return row.currentGen === from && same ? null : "superseded";
|
|
9102
9065
|
};
|
|
9103
|
-
const
|
|
9066
|
+
const sealed = /* @__PURE__ */ new Set();
|
|
9067
|
+
const versions = /* @__PURE__ */ new Map();
|
|
9104
9068
|
const holds = async (generation2) => {
|
|
9105
9069
|
const key2 = { ...base, generation: generation2 };
|
|
9106
|
-
const
|
|
9070
|
+
const crypto = cryptoAt(generation2);
|
|
9107
9071
|
try {
|
|
9108
|
-
let
|
|
9072
|
+
let reader;
|
|
9073
|
+
const index = { refused: false };
|
|
9074
|
+
let opened;
|
|
9109
9075
|
try {
|
|
9110
|
-
|
|
9076
|
+
reader = await read(
|
|
9077
|
+
() => openGenerationReaderNoting(
|
|
9078
|
+
deps.storage,
|
|
9079
|
+
key2,
|
|
9080
|
+
crypto === void 0 ? void 0 : noticingIndex(crypto, index),
|
|
9081
|
+
(version) => opened = version
|
|
9082
|
+
)
|
|
9083
|
+
);
|
|
9111
9084
|
} catch (err) {
|
|
9112
|
-
if (
|
|
9113
|
-
|
|
9085
|
+
if (crypto === void 0) {
|
|
9086
|
+
let footer;
|
|
9087
|
+
const encrypted = !isTransientError(err) && await read(
|
|
9088
|
+
() => objectIsEncrypted(deps.storage, key2, (version) => footer = version)
|
|
9089
|
+
);
|
|
9090
|
+
if (encrypted) {
|
|
9114
9091
|
if (tombstoned) return false;
|
|
9115
|
-
|
|
9092
|
+
sealed.add(generation2);
|
|
9093
|
+
versions.set(generation2, footer);
|
|
9116
9094
|
return true;
|
|
9117
9095
|
}
|
|
9118
9096
|
throw err;
|
|
9119
9097
|
}
|
|
9098
|
+
if (isIntegrityError(err) && index.refused) {
|
|
9099
|
+
sealed.add(generation2);
|
|
9100
|
+
versions.set(generation2, opened);
|
|
9101
|
+
return true;
|
|
9102
|
+
}
|
|
9120
9103
|
if (!isIntegrityError(err) || await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
9121
9104
|
throw err;
|
|
9122
9105
|
}
|
|
9123
|
-
|
|
9106
|
+
reader = await read(() => openGenerationReader(deps.storage, key2, void 0));
|
|
9107
|
+
}
|
|
9108
|
+
versions.set(generation2, objectVersionOf(reader));
|
|
9109
|
+
let bytes;
|
|
9110
|
+
try {
|
|
9111
|
+
bytes = await read(() => reader.getChunk(chunkKey));
|
|
9112
|
+
} catch (err) {
|
|
9113
|
+
if (!isTransientError(err) && !await read(() => sameObjectUnder(deps.storage, key2, reader.fingerprint))) {
|
|
9114
|
+
return null;
|
|
9115
|
+
}
|
|
9116
|
+
throw err;
|
|
9124
9117
|
}
|
|
9125
9118
|
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
9126
9119
|
} catch (err) {
|
|
@@ -9128,6 +9121,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9128
9121
|
throw err;
|
|
9129
9122
|
}
|
|
9130
9123
|
};
|
|
9124
|
+
const found = (generation2, held) => held === true && !sealed.has(generation2);
|
|
9131
9125
|
const holdsEach = async (generations, stopAtHolder) => {
|
|
9132
9126
|
let firstHolder = Infinity;
|
|
9133
9127
|
const settled2 = await mapWithConcurrency(
|
|
@@ -9139,7 +9133,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9139
9133
|
}
|
|
9140
9134
|
try {
|
|
9141
9135
|
const held = await holds(generation2);
|
|
9142
|
-
if (held
|
|
9136
|
+
if (found(generation2, held) && index < firstHolder) firstHolder = index;
|
|
9143
9137
|
return { generation: generation2, held, fault: void 0 };
|
|
9144
9138
|
} catch (fault) {
|
|
9145
9139
|
return { generation: generation2, held: null, fault: { error: fault } };
|
|
@@ -9150,7 +9144,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9150
9144
|
for (const { generation: generation2, held, fault } of settled2) {
|
|
9151
9145
|
if (fault !== void 0) throw fault.error;
|
|
9152
9146
|
outcomes.push({ generation: generation2, held });
|
|
9153
|
-
if (stopAtHolder && held
|
|
9147
|
+
if (stopAtHolder && found(generation2, held)) break;
|
|
9154
9148
|
}
|
|
9155
9149
|
return outcomes;
|
|
9156
9150
|
};
|
|
@@ -9161,41 +9155,119 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9161
9155
|
return (await holdsEach(toRead, true)).find((outcome) => outcome.held === true)?.generation;
|
|
9162
9156
|
};
|
|
9163
9157
|
const cannotRemove = (generation2) => new WriteConflictError(
|
|
9164
|
-
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which still holds the id and is still in the bucket; re-run`
|
|
9158
|
+
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which ${sealed.has(generation2) ? "cannot be searched, so may hold the id," : "still holds the id"} and is still in the bucket; re-run`
|
|
9165
9159
|
);
|
|
9166
9160
|
const fenceInFlight = async () => {
|
|
9167
|
-
const
|
|
9161
|
+
const rng = deps.rng ?? deps.readRetry?.rng;
|
|
9162
|
+
const churn = leaseChurn({ clock: deps.clock, rng });
|
|
9163
|
+
let resends = 0;
|
|
9168
9164
|
let row = await deps.registry.get(ref);
|
|
9169
9165
|
for (; ; ) {
|
|
9170
9166
|
const verdict = rowVerdict(row);
|
|
9171
9167
|
if (verdict !== null) return verdict;
|
|
9172
9168
|
const current = row;
|
|
9173
|
-
|
|
9169
|
+
let failure;
|
|
9174
9170
|
try {
|
|
9175
|
-
const { token } = await deps.registry.compareAndSwap(
|
|
9176
|
-
|
|
9171
|
+
const { token } = await deps.registry.compareAndSwap(
|
|
9172
|
+
ref,
|
|
9173
|
+
current.token,
|
|
9174
|
+
renewPointer(current)
|
|
9175
|
+
);
|
|
9176
|
+
premise = { ...current, token, pointerId: token };
|
|
9177
9177
|
premiseToken = token;
|
|
9178
9178
|
return null;
|
|
9179
9179
|
} catch (err) {
|
|
9180
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9180
|
+
if (!isWriteConflictError(err) && !isTransientError(err)) throw err;
|
|
9181
|
+
failure = err;
|
|
9182
|
+
}
|
|
9183
|
+
const unanswered = isTransientError(failure);
|
|
9184
|
+
let now;
|
|
9185
|
+
try {
|
|
9186
|
+
now = await deps.registry.get(ref);
|
|
9187
|
+
} catch (readErr) {
|
|
9188
|
+
if (!unanswered || isIntegrityError(readErr)) throw readErr;
|
|
9189
|
+
throw failure;
|
|
9190
|
+
}
|
|
9191
|
+
if (now !== null && renewedSince(current, now)) {
|
|
9192
|
+
premise = now;
|
|
9193
|
+
premiseToken = now.token;
|
|
9194
|
+
return null;
|
|
9195
|
+
}
|
|
9196
|
+
if (unanswered && now !== null && now.token === current.token) {
|
|
9197
|
+
const sleep = deps.clock?.sleep;
|
|
9198
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS) throw failure;
|
|
9199
|
+
const bound = UNANSWERED_BASE_MS * 2 ** resends;
|
|
9200
|
+
resends += 1;
|
|
9201
|
+
await sleep.call(deps.clock, Math.floor((rng?.next() ?? 1) * bound));
|
|
9202
|
+
row = now;
|
|
9203
|
+
continue;
|
|
9181
9204
|
}
|
|
9182
|
-
const now = await deps.registry.get(ref);
|
|
9183
9205
|
const settled2 = await churn.settle(current, now, () => deps.registry.get(ref));
|
|
9184
9206
|
if (settled2 === void 0) return rowVerdict(now) ?? "superseded";
|
|
9185
9207
|
row = settled2.row;
|
|
9186
9208
|
}
|
|
9187
9209
|
};
|
|
9210
|
+
const deleteFenced = async (holders) => {
|
|
9211
|
+
const deleted = [];
|
|
9212
|
+
let moved = await fenceInFlight();
|
|
9213
|
+
for (const generation2 of holders) {
|
|
9214
|
+
if (moved !== null) break;
|
|
9215
|
+
moved = rowVerdict(await deps.registry.get(ref));
|
|
9216
|
+
if (moved !== null) break;
|
|
9217
|
+
try {
|
|
9218
|
+
await deps.storage.delete({ ...base, generation: generation2 }, { ifVersion: versions.get(generation2) });
|
|
9219
|
+
} catch (err) {
|
|
9220
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9221
|
+
moved = rowVerdict(await deps.registry.get(ref)) ?? "superseded";
|
|
9222
|
+
break;
|
|
9223
|
+
}
|
|
9224
|
+
deleted.push(generation2);
|
|
9225
|
+
}
|
|
9226
|
+
return { deleted, moved };
|
|
9227
|
+
};
|
|
9228
|
+
const collectedAll = (fromGeneration, collected2) => {
|
|
9229
|
+
seen.collectedOn = record.token;
|
|
9230
|
+
seen.found = true;
|
|
9231
|
+
return {
|
|
9232
|
+
...base,
|
|
9233
|
+
erased: true,
|
|
9234
|
+
fromGeneration,
|
|
9235
|
+
collected: [...collected2].sort((a, b) => a - b)
|
|
9236
|
+
};
|
|
9237
|
+
};
|
|
9238
|
+
const sealedOnly = (answer, collected2) => {
|
|
9239
|
+
if (collected2.length > 0) seen.collectedOn = record.token;
|
|
9240
|
+
return { ...answer, collected: [...collected2].sort((a, b) => a - b) };
|
|
9241
|
+
};
|
|
9188
9242
|
const unpublished = async () => {
|
|
9189
9243
|
const generations = [];
|
|
9190
9244
|
for await (const key2 of deps.storage.list(ref)) generations.push(key2.generation);
|
|
9191
9245
|
const newestFirst = generations.sort((a, b) => b - a);
|
|
9192
|
-
const
|
|
9193
|
-
|
|
9194
|
-
|
|
9195
|
-
|
|
9196
|
-
|
|
9197
|
-
|
|
9198
|
-
|
|
9246
|
+
const holders = [];
|
|
9247
|
+
const clean = /* @__PURE__ */ new Set();
|
|
9248
|
+
for (const { generation: generation2, held } of await holdsEach(newestFirst, false)) {
|
|
9249
|
+
if (held === true) holders.push(generation2);
|
|
9250
|
+
else if (held === false) clean.add(generation2);
|
|
9251
|
+
}
|
|
9252
|
+
const none = { ...base, erased: false, reason: "no-generation", collected: [] };
|
|
9253
|
+
if (holders.length === 0) return none;
|
|
9254
|
+
const newest = holders.find((generation2) => !sealed.has(generation2));
|
|
9255
|
+
const { deleted, moved } = await deleteFenced(holders);
|
|
9256
|
+
const collected2 = [...deleted].sort((a, b) => a - b);
|
|
9257
|
+
const left2 = await holderLeft(clean);
|
|
9258
|
+
if (left2 === void 0) {
|
|
9259
|
+
return newest === void 0 ? sealedOnly(none, collected2) : collectedAll(newest, collected2);
|
|
9260
|
+
}
|
|
9261
|
+
if (moved !== null) {
|
|
9262
|
+
return {
|
|
9263
|
+
...base,
|
|
9264
|
+
erased: false,
|
|
9265
|
+
reason: moved,
|
|
9266
|
+
...newest === void 0 ? {} : { fromGeneration: newest },
|
|
9267
|
+
collected: collected2
|
|
9268
|
+
};
|
|
9269
|
+
}
|
|
9270
|
+
throw cannotRemove(left2);
|
|
9199
9271
|
};
|
|
9200
9272
|
const underTombstone = async () => {
|
|
9201
9273
|
const generations = [];
|
|
@@ -9206,7 +9278,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9206
9278
|
const collected2 = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
9207
9279
|
const left2 = await holderLeft(/* @__PURE__ */ new Set());
|
|
9208
9280
|
if (left2 !== void 0) throw cannotRemove(left2);
|
|
9209
|
-
return
|
|
9281
|
+
return collectedAll(holder.generation, collected2);
|
|
9210
9282
|
};
|
|
9211
9283
|
const notInCurrent = async () => {
|
|
9212
9284
|
const notMember = {
|
|
@@ -9231,38 +9303,47 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9231
9303
|
if (held === true) holdersAbove.push(generation2);
|
|
9232
9304
|
else if (held === false) clean.add(generation2);
|
|
9233
9305
|
}
|
|
9306
|
+
const foundAbove = holdersAbove.find((generation2) => !sealed.has(generation2));
|
|
9234
9307
|
let holderBelow;
|
|
9235
|
-
|
|
9308
|
+
const sealedBelow = [];
|
|
9309
|
+
if (foundAbove === void 0) {
|
|
9236
9310
|
for (const { generation: generation2, held } of await holdsEach(
|
|
9237
9311
|
newestFirst.filter((g) => g < from),
|
|
9238
9312
|
true
|
|
9239
9313
|
)) {
|
|
9240
|
-
if (held
|
|
9314
|
+
if (found(generation2, held)) holderBelow = generation2;
|
|
9315
|
+
else if (held === true) sealedBelow.push(generation2);
|
|
9241
9316
|
else if (held === false) clean.add(generation2);
|
|
9242
9317
|
}
|
|
9243
9318
|
}
|
|
9244
|
-
const newest =
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9248
|
-
|
|
9249
|
-
|
|
9250
|
-
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
collected2.push(generation2);
|
|
9319
|
+
const newest = foundAbove ?? holderBelow;
|
|
9320
|
+
const fenced = newest === void 0 ? [...holdersAbove, ...sealedBelow] : holdersAbove;
|
|
9321
|
+
if (newest === void 0 && fenced.length === 0) return notMember;
|
|
9322
|
+
const collected2 = newest === void 0 ? [] : [...await gcOrphanGenerations(ref, deps, { keep: 0 })];
|
|
9323
|
+
let moved = null;
|
|
9324
|
+
if (fenced.length > 0) {
|
|
9325
|
+
const deleted = await deleteFenced(fenced);
|
|
9326
|
+
collected2.push(...deleted.deleted);
|
|
9327
|
+
moved = deleted.moved;
|
|
9254
9328
|
}
|
|
9255
9329
|
const left2 = await holderLeft(clean);
|
|
9256
|
-
if (left2 === void 0)
|
|
9330
|
+
if (left2 === void 0) {
|
|
9331
|
+
if (newest !== void 0) return collectedAll(newest, collected2);
|
|
9332
|
+
return sealedOnly(notMember, collected2);
|
|
9333
|
+
}
|
|
9257
9334
|
if (moved !== null) {
|
|
9258
|
-
return { ...base, erased: false, reason: moved, fromGeneration: newest, collected: collected2 };
|
|
9335
|
+
return { ...base, erased: false, reason: moved, fromGeneration: newest ?? from, collected: collected2 };
|
|
9259
9336
|
}
|
|
9260
9337
|
throw cannotRemove(left2);
|
|
9261
9338
|
};
|
|
9339
|
+
let rewrote;
|
|
9262
9340
|
const stage = async () => {
|
|
9263
9341
|
let written;
|
|
9342
|
+
const described = usableSummary(ref, record, aead);
|
|
9264
9343
|
try {
|
|
9265
|
-
const reader = await read(
|
|
9344
|
+
const reader = await read(
|
|
9345
|
+
() => openGenerationReader(deps.storage, fromKey, cryptoAt(from), {}, described?.fingerprint)
|
|
9346
|
+
);
|
|
9266
9347
|
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
9267
9348
|
if (bytes === null) return await notInCurrent();
|
|
9268
9349
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
@@ -9272,10 +9353,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9272
9353
|
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
9273
9354
|
const generation2 = await nextGeneration(ref, deps);
|
|
9274
9355
|
const key2 = { ...base, generation: generation2 };
|
|
9275
|
-
const metadata = metadataToCarry(
|
|
9276
|
-
reader.metadata,
|
|
9277
|
-
aead === void 0 ? void 0 : usableSummary(ref, record, aead)
|
|
9278
|
-
);
|
|
9356
|
+
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
9279
9357
|
const tally = await writeCrbmGenerationStream(
|
|
9280
9358
|
deps.storage,
|
|
9281
9359
|
key2,
|
|
@@ -9283,6 +9361,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9283
9361
|
{ crypto: cryptoAt(generation2), clock: deps.clock, metadata }
|
|
9284
9362
|
);
|
|
9285
9363
|
written = generation2;
|
|
9364
|
+
rewrote = { generation: generation2, fingerprint: tally.fingerprint };
|
|
9286
9365
|
const beforeVerify = await deps.registry.get(ref);
|
|
9287
9366
|
const early = rowVerdict(beforeVerify);
|
|
9288
9367
|
if (early !== null) return refused(early, written);
|
|
@@ -9292,7 +9371,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9292
9371
|
const summary2 = summaryOf(
|
|
9293
9372
|
ref,
|
|
9294
9373
|
generation2,
|
|
9295
|
-
{ cardinality: tally.cardinality, metadata },
|
|
9374
|
+
{ cardinality: tally.cardinality, metadata, fingerprint: tally.fingerprint },
|
|
9296
9375
|
aead
|
|
9297
9376
|
);
|
|
9298
9377
|
return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
|
|
@@ -9309,22 +9388,31 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9309
9388
|
throw err;
|
|
9310
9389
|
}
|
|
9311
9390
|
};
|
|
9312
|
-
const discardRefused = async (
|
|
9391
|
+
const discardRefused = async (own) => {
|
|
9392
|
+
const key2 = { ...base, generation: own.generation };
|
|
9313
9393
|
const row = await deps.registry.get(ref);
|
|
9314
9394
|
if (row !== null && row.status === "destroyed") {
|
|
9315
|
-
await deps.storage.delete(
|
|
9395
|
+
await deps.storage.delete(key2);
|
|
9316
9396
|
return row;
|
|
9317
9397
|
}
|
|
9318
9398
|
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
9319
|
-
if (row.currentGen <= from || row.currentGen >=
|
|
9320
|
-
await deps.storage
|
|
9399
|
+
if (row.currentGen <= from || row.currentGen >= own.generation) return row;
|
|
9400
|
+
const proven = await provenObject(deps.storage, key2, own.fingerprint);
|
|
9401
|
+
if (proven === null) return row;
|
|
9402
|
+
try {
|
|
9403
|
+
await deps.storage.delete(key2, { ifVersion: proven.version });
|
|
9404
|
+
} catch (err) {
|
|
9405
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9406
|
+
}
|
|
9321
9407
|
return row;
|
|
9322
9408
|
};
|
|
9323
9409
|
if (tombstoned) return await underTombstone();
|
|
9324
9410
|
if (pointerless) return await unpublished();
|
|
9325
9411
|
const staged = await stage();
|
|
9326
9412
|
if ("erased" in staged) {
|
|
9327
|
-
if (!staged.erased && staged.generation !== void 0
|
|
9413
|
+
if (!staged.erased && staged.generation !== void 0 && rewrote !== void 0) {
|
|
9414
|
+
await discardRefused(rewrote);
|
|
9415
|
+
}
|
|
9328
9416
|
return staged;
|
|
9329
9417
|
}
|
|
9330
9418
|
const { generation, key, fingerprint, summary } = staged;
|
|
@@ -9345,7 +9433,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9345
9433
|
rng: deps.rng ?? deps.readRetry?.rng
|
|
9346
9434
|
});
|
|
9347
9435
|
if (!published) {
|
|
9348
|
-
const now = await discardRefused(generation);
|
|
9436
|
+
const now = await discardRefused({ generation, fingerprint });
|
|
9349
9437
|
return {
|
|
9350
9438
|
...base,
|
|
9351
9439
|
erased: false,
|
|
@@ -9355,10 +9443,11 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9355
9443
|
collected: []
|
|
9356
9444
|
};
|
|
9357
9445
|
}
|
|
9358
|
-
|
|
9446
|
+
audit.onEvent({
|
|
9359
9447
|
kind: "segment.rewrite",
|
|
9360
9448
|
namespace: ref.namespace,
|
|
9361
9449
|
segment: ref.segment,
|
|
9450
|
+
...incarnationField(fromToken),
|
|
9362
9451
|
fromGeneration: from,
|
|
9363
9452
|
generation
|
|
9364
9453
|
});
|
|
@@ -9367,6 +9456,29 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9367
9456
|
if (left !== void 0) throw cannotRemove(left);
|
|
9368
9457
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
9369
9458
|
}
|
|
9459
|
+
function noticingIndex(crypto, index) {
|
|
9460
|
+
const indexAad = crypto.aadFor("index");
|
|
9461
|
+
return {
|
|
9462
|
+
aadFor: crypto.aadFor,
|
|
9463
|
+
aead: {
|
|
9464
|
+
seal: (plaintext, aad) => crypto.aead.seal(plaintext, aad),
|
|
9465
|
+
open: (sealed, aad) => {
|
|
9466
|
+
try {
|
|
9467
|
+
return crypto.aead.open(sealed, aad);
|
|
9468
|
+
} catch (err) {
|
|
9469
|
+
if (sameBytes(aad, indexAad)) index.refused = true;
|
|
9470
|
+
throw err;
|
|
9471
|
+
}
|
|
9472
|
+
}
|
|
9473
|
+
}
|
|
9474
|
+
};
|
|
9475
|
+
}
|
|
9476
|
+
function sameBytes(a, b) {
|
|
9477
|
+
return a.length === b.length && a.every((byte, i) => byte === b[i]);
|
|
9478
|
+
}
|
|
9479
|
+
function renewedSince(held, now) {
|
|
9480
|
+
return now.pointerId !== held.pointerId && onlyLeasesDiffer(held, now, ["pointerId"]);
|
|
9481
|
+
}
|
|
9370
9482
|
var REWRITE_RANGES_AHEAD = 4;
|
|
9371
9483
|
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
9372
9484
|
const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
@@ -9685,12 +9797,13 @@ async function destroySegment(ref, deps, options) {
|
|
|
9685
9797
|
`destroySegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental crypto-shred)`
|
|
9686
9798
|
);
|
|
9687
9799
|
}
|
|
9688
|
-
const result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9800
|
+
const { token, ...result } = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9689
9801
|
if (result.cryptoShredded) {
|
|
9690
9802
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9691
9803
|
kind: "segment.erase",
|
|
9692
9804
|
namespace: ref.namespace,
|
|
9693
|
-
segment: ref.segment
|
|
9805
|
+
segment: ref.segment,
|
|
9806
|
+
...incarnationField(token)
|
|
9694
9807
|
});
|
|
9695
9808
|
}
|
|
9696
9809
|
return result;
|
|
@@ -9721,8 +9834,9 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9721
9834
|
let segmentsShredded = 0;
|
|
9722
9835
|
const destroyed = await mapWithConcurrency(refs, ERASE_CONCURRENCY, async (ref) => {
|
|
9723
9836
|
let result;
|
|
9837
|
+
let token;
|
|
9724
9838
|
try {
|
|
9725
|
-
result = await shredSegment(ref, deps, options.allowCleartext ?? false);
|
|
9839
|
+
({ token, ...result } = await shredSegment(ref, deps, options.allowCleartext ?? false));
|
|
9726
9840
|
} catch (err) {
|
|
9727
9841
|
result = {
|
|
9728
9842
|
segment: ref.segment,
|
|
@@ -9734,7 +9848,12 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9734
9848
|
}
|
|
9735
9849
|
if (result.cryptoShredded) {
|
|
9736
9850
|
segmentsShredded += 1;
|
|
9737
|
-
audit.onEvent({
|
|
9851
|
+
audit.onEvent({
|
|
9852
|
+
kind: "segment.erase",
|
|
9853
|
+
namespace: ref.namespace,
|
|
9854
|
+
segment: ref.segment,
|
|
9855
|
+
...incarnationField(token)
|
|
9856
|
+
});
|
|
9738
9857
|
}
|
|
9739
9858
|
return result;
|
|
9740
9859
|
});
|
|
@@ -9783,13 +9902,13 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9783
9902
|
};
|
|
9784
9903
|
}
|
|
9785
9904
|
try {
|
|
9786
|
-
await deps.registry.create(ref, {
|
|
9905
|
+
const tombstone = await deps.registry.create(ref, {
|
|
9787
9906
|
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
9788
9907
|
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
9789
9908
|
currentGen: orphans[orphans.length - 1],
|
|
9790
9909
|
status: "destroyed"
|
|
9791
9910
|
});
|
|
9792
|
-
shred = { ...shred, destroyed: true, reason: void 0 };
|
|
9911
|
+
shred = { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
9793
9912
|
} catch (err) {
|
|
9794
9913
|
if (!isWriteConflictError(err)) throw err;
|
|
9795
9914
|
shred = await shredSegment(ref, deps, true, "dropSegment");
|
|
@@ -9799,7 +9918,8 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9799
9918
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9800
9919
|
kind: "segment.erase",
|
|
9801
9920
|
namespace: ref.namespace,
|
|
9802
|
-
segment: ref.segment
|
|
9921
|
+
segment: ref.segment,
|
|
9922
|
+
...incarnationField(shred.token)
|
|
9803
9923
|
});
|
|
9804
9924
|
}
|
|
9805
9925
|
const generationsDeleted = [];
|
|
@@ -9829,6 +9949,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
9829
9949
|
kind: "segment.dispose",
|
|
9830
9950
|
namespace: ref.namespace,
|
|
9831
9951
|
segment: ref.segment,
|
|
9952
|
+
...incarnationField(shred.token),
|
|
9832
9953
|
generationsDeleted: generationsDeleted.length
|
|
9833
9954
|
});
|
|
9834
9955
|
}
|
|
@@ -9866,7 +9987,13 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9866
9987
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "absent" };
|
|
9867
9988
|
}
|
|
9868
9989
|
if (record.status === "destroyed") {
|
|
9869
|
-
return {
|
|
9990
|
+
return {
|
|
9991
|
+
...base,
|
|
9992
|
+
destroyed: true,
|
|
9993
|
+
cryptoShredded: false,
|
|
9994
|
+
reason: "already",
|
|
9995
|
+
token: record.token
|
|
9996
|
+
};
|
|
9870
9997
|
}
|
|
9871
9998
|
const encrypted = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
9872
9999
|
if (!encrypted && !allowCleartext) {
|
|
@@ -9874,7 +10001,7 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9874
10001
|
}
|
|
9875
10002
|
const unmarked = sweep === void 0 ? withoutSweepMark(record.retention) : null;
|
|
9876
10003
|
try {
|
|
9877
|
-
await deps.registry.compareAndSwap(ref, record.token, {
|
|
10004
|
+
const tombstone = await deps.registry.compareAndSwap(ref, record.token, {
|
|
9878
10005
|
status: "destroyed",
|
|
9879
10006
|
wrappedDeks: void 0,
|
|
9880
10007
|
// ← the crypto-shred: the only copy of the DEK wrappings is gone
|
|
@@ -9888,7 +10015,7 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sw
|
|
|
9888
10015
|
// one a restored or hand-edited row carried.
|
|
9889
10016
|
...sweep !== void 0 ? { retention: sweep.stamp(record.retention) } : unmarked === null ? {} : { retention: unmarked }
|
|
9890
10017
|
});
|
|
9891
|
-
return { ...base, destroyed: true, cryptoShredded: encrypted };
|
|
10018
|
+
return { ...base, destroyed: true, cryptoShredded: encrypted, token: tombstone.token };
|
|
9892
10019
|
} catch (err) {
|
|
9893
10020
|
if (!isWriteConflictError(err)) throw err;
|
|
9894
10021
|
lost = record;
|
|
@@ -10303,17 +10430,7 @@ async function checkSummary(ref, live, deps) {
|
|
|
10303
10430
|
throw err;
|
|
10304
10431
|
}
|
|
10305
10432
|
}
|
|
10306
|
-
const
|
|
10307
|
-
if (described === void 0) return { kind: "ok", unchecked: keyed };
|
|
10308
|
-
const reader = await openGenerationReader(
|
|
10309
|
-
deps.storage,
|
|
10310
|
-
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
10311
|
-
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) }
|
|
10312
|
-
);
|
|
10313
|
-
let cardinality = 0;
|
|
10314
|
-
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
10315
|
-
if (summaryAgrees(described, { cardinality, metadata: reader.metadata })) return { kind: "ok" };
|
|
10316
|
-
return {
|
|
10433
|
+
const mismatch = {
|
|
10317
10434
|
kind: "issue",
|
|
10318
10435
|
issue: {
|
|
10319
10436
|
segment: ref.segment,
|
|
@@ -10322,6 +10439,24 @@ async function checkSummary(ref, live, deps) {
|
|
|
10322
10439
|
issue: "summary-mismatch"
|
|
10323
10440
|
}
|
|
10324
10441
|
};
|
|
10442
|
+
const described = usableSummary(ref, live, aead);
|
|
10443
|
+
if (described === void 0) {
|
|
10444
|
+
const summary = live.summary;
|
|
10445
|
+
const unnamed = !keyed && live.status === "active" && summary !== void 0 && !("sealed" in summary) && summary.generation === generation && fingerprintParts(summary.fingerprint) === void 0;
|
|
10446
|
+
return unnamed ? mismatch : { kind: "ok", unchecked: keyed };
|
|
10447
|
+
}
|
|
10448
|
+
const reader = await openNamedGeneration(
|
|
10449
|
+
deps.storage,
|
|
10450
|
+
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
10451
|
+
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) },
|
|
10452
|
+
described.fingerprint
|
|
10453
|
+
);
|
|
10454
|
+
let cardinality = 0;
|
|
10455
|
+
for (const n of reader?.cardinalities().values() ?? []) cardinality += n;
|
|
10456
|
+
if (reader !== void 0 && summaryAgrees(described, { cardinality, metadata: reader.metadata })) {
|
|
10457
|
+
return { kind: "ok" };
|
|
10458
|
+
}
|
|
10459
|
+
return mismatch;
|
|
10325
10460
|
}
|
|
10326
10461
|
async function runConsistencyCheck(deps, options = {}) {
|
|
10327
10462
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
@@ -10408,17 +10543,11 @@ var RetryingStorageChunkSource = class {
|
|
|
10408
10543
|
getChunks;
|
|
10409
10544
|
cardinalities;
|
|
10410
10545
|
summary;
|
|
10546
|
+
stat;
|
|
10411
10547
|
currentGeneration;
|
|
10412
10548
|
invalidate;
|
|
10413
10549
|
exists;
|
|
10414
10550
|
currentVersion;
|
|
10415
|
-
/**
|
|
10416
|
-
* Forwarded, not retried: a property, with no request behind it. Without it the grounded cost report priced a
|
|
10417
|
-
* retrying store's refresh at the default TTL whatever the store was configured with.
|
|
10418
|
-
*/
|
|
10419
|
-
get pointerRefreshMs() {
|
|
10420
|
-
return this.inner.pointerRefreshMs;
|
|
10421
|
-
}
|
|
10422
10551
|
constructor(inner, opts) {
|
|
10423
10552
|
this.inner = inner;
|
|
10424
10553
|
const r = toRetry(opts);
|
|
@@ -10446,6 +10575,10 @@ var RetryingStorageChunkSource = class {
|
|
|
10446
10575
|
if (innerSummary) {
|
|
10447
10576
|
this.summary = (ref) => withRetry(() => innerSummary.call(inner, ref), this.policy, this.deps);
|
|
10448
10577
|
}
|
|
10578
|
+
const innerStat = inner.stat;
|
|
10579
|
+
if (innerStat) {
|
|
10580
|
+
this.stat = (ref) => withRetry(() => innerStat.call(inner, ref), this.policy, this.deps);
|
|
10581
|
+
}
|
|
10449
10582
|
const innerCurrentGeneration = inner.currentGeneration;
|
|
10450
10583
|
if (innerCurrentGeneration) {
|
|
10451
10584
|
this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);
|
|
@@ -10473,7 +10606,6 @@ var RetryingStorageChunkSource = class {
|
|
|
10473
10606
|
}
|
|
10474
10607
|
};
|
|
10475
10608
|
export {
|
|
10476
|
-
AWS_US_EAST_1_ONDEMAND,
|
|
10477
10609
|
BoundedLru,
|
|
10478
10610
|
BudgetExceededError,
|
|
10479
10611
|
BufferReader,
|
|
@@ -10484,7 +10616,6 @@ export {
|
|
|
10484
10616
|
CrbmStorageChunkSource,
|
|
10485
10617
|
DEFAULT_BUDGET,
|
|
10486
10618
|
DEFAULT_RETRY_POLICY,
|
|
10487
|
-
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
10488
10619
|
InProcessKeystore,
|
|
10489
10620
|
IntegrityError,
|
|
10490
10621
|
KeyUnavailableError,
|
|
@@ -10499,7 +10630,6 @@ export {
|
|
|
10499
10630
|
NOOP_METRICS,
|
|
10500
10631
|
NodeAead,
|
|
10501
10632
|
NotFoundError,
|
|
10502
|
-
ONE_REDIS_HA_CLUSTER,
|
|
10503
10633
|
PinnedStorageChunkSource,
|
|
10504
10634
|
RecordingAuditSink,
|
|
10505
10635
|
RetryingStorageChunkSource,
|
|
@@ -10519,10 +10649,8 @@ export {
|
|
|
10519
10649
|
dropSegment,
|
|
10520
10650
|
eraseIdFromSegment,
|
|
10521
10651
|
eraseNamespace,
|
|
10522
|
-
estimateCost,
|
|
10523
10652
|
excludingReservedRows,
|
|
10524
10653
|
getSegmentRetention,
|
|
10525
|
-
groundedReport,
|
|
10526
10654
|
isBudgetExceededError,
|
|
10527
10655
|
isCapabilityError,
|
|
10528
10656
|
isCloudRoaringError,
|