@cloudbitmaps/core 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -12
- package/dist/{chunk-OSU7W2Y3.js → chunk-2B2QF4FU.js} +205 -756
- package/dist/chunk-2B2QF4FU.js.map +7 -0
- package/dist/chunk-45ML6YU4.js +641 -0
- package/dist/chunk-45ML6YU4.js.map +7 -0
- package/dist/chunk-BUDAVGBI.js +73 -0
- package/dist/chunk-BUDAVGBI.js.map +7 -0
- package/dist/chunk-WPER7R2R.js +121 -0
- package/dist/chunk-WPER7R2R.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +2 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +50 -0
- package/dist/core/codec.d.ts +13 -3
- package/dist/core/combine-many.d.ts +11 -0
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +15 -5
- package/dist/core/crbm/reader.d.ts +1 -1
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +5 -5
- package/dist/core/erase-id.d.ts +36 -13
- package/dist/core/erasure.d.ts +61 -27
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +49 -1
- package/dist/core/load.d.ts +23 -2
- package/dist/core/metrics.d.ts +21 -0
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +100 -7
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/retention-sweep.d.ts +60 -15
- package/dist/core/retention.d.ts +17 -6
- package/dist/core/retry.d.ts +1 -2
- package/dist/core/rollback.d.ts +22 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +3 -3
- package/dist/driver-kit.js +61 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +17 -2
- package/dist/drivers/crypto.d.ts +17 -2
- package/dist/export/index.d.ts +16 -2
- package/dist/index.d.ts +10 -9
- package/dist/index.js +1799 -1700
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +11 -3
- package/dist/chunk-OSU7W2Y3.js.map +0 -7
package/dist/index.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
|
+
import {
|
|
2
|
+
boundedAll,
|
|
3
|
+
mapWithConcurrency
|
|
4
|
+
} from "./chunk-BUDAVGBI.js";
|
|
5
|
+
import {
|
|
6
|
+
BufferReader,
|
|
7
|
+
KeptSegmentKeys,
|
|
8
|
+
chunkKeyUnder,
|
|
9
|
+
segmentKey,
|
|
10
|
+
segmentPrefix,
|
|
11
|
+
shardOf
|
|
12
|
+
} from "./chunk-WPER7R2R.js";
|
|
1
13
|
import {
|
|
2
14
|
BudgetExceededError,
|
|
3
15
|
CHUNK_COUNT,
|
|
4
16
|
CONTENTION_NEXT_STEP,
|
|
5
17
|
CapabilityError,
|
|
6
18
|
CloudRoaringError,
|
|
7
|
-
DEFAULT_NAMESPACE,
|
|
8
19
|
DEFAULT_RETRY_POLICY,
|
|
9
20
|
FOOTER_BYTES,
|
|
10
21
|
IntegrityError,
|
|
@@ -20,6 +31,7 @@ import {
|
|
|
20
31
|
MAX_REMAINDER,
|
|
21
32
|
MAX_STORED_KNOWN_GENERATIONS,
|
|
22
33
|
NotFoundError,
|
|
34
|
+
PORT_KEYS,
|
|
23
35
|
PREAMBLE_BYTES,
|
|
24
36
|
RESERVED_NAMESPACE_PREFIX,
|
|
25
37
|
StaleOperandError,
|
|
@@ -31,23 +43,14 @@ import {
|
|
|
31
43
|
ValidationError,
|
|
32
44
|
WriteConflictError,
|
|
33
45
|
aadFor,
|
|
34
|
-
applyRegistryPatch,
|
|
35
46
|
ascending,
|
|
36
47
|
assertRegistryCanWrite,
|
|
37
|
-
|
|
38
|
-
brandAsBackend,
|
|
39
|
-
checkNow,
|
|
48
|
+
checkRequireEncryption,
|
|
40
49
|
checkedRetryPolicy,
|
|
41
50
|
compactFingerprint,
|
|
42
51
|
copiedMetadata,
|
|
43
|
-
decodeNameFromPath,
|
|
44
52
|
describingKnown,
|
|
45
|
-
drawIncarnation,
|
|
46
|
-
drawWrite,
|
|
47
|
-
encodeNameForKey,
|
|
48
|
-
encodeNameForPath,
|
|
49
53
|
encodedNameLength,
|
|
50
|
-
entropyIsAvailable,
|
|
51
54
|
errorKind,
|
|
52
55
|
fallbackBound,
|
|
53
56
|
fingerprintFor,
|
|
@@ -65,6 +68,7 @@ import {
|
|
|
65
68
|
isReservedNamespace,
|
|
66
69
|
isStorageBackend,
|
|
67
70
|
isTransientError,
|
|
71
|
+
isUnsettledConflict,
|
|
68
72
|
isValidationError,
|
|
69
73
|
isWriteConflictError,
|
|
70
74
|
jitteredWait,
|
|
@@ -72,24 +76,22 @@ import {
|
|
|
72
76
|
keptWindow,
|
|
73
77
|
knownGensShapeFault,
|
|
74
78
|
leaseChurn,
|
|
75
|
-
mapWithConcurrency,
|
|
76
79
|
metadataBytes,
|
|
77
80
|
metadataFromBytes,
|
|
78
81
|
metadataToCarry,
|
|
79
|
-
namespacePathPart,
|
|
80
|
-
newIncarnationToken,
|
|
81
82
|
nextKnown,
|
|
82
|
-
nextRegistryToken,
|
|
83
83
|
onlyLeasesDiffer,
|
|
84
84
|
ownSummary,
|
|
85
|
-
parseRegistryEnvelope,
|
|
86
85
|
pointerIdOf,
|
|
86
|
+
pointerSetBy,
|
|
87
87
|
pruneKnown,
|
|
88
|
-
|
|
88
|
+
refusalOfId,
|
|
89
89
|
refuseUnknownOptions,
|
|
90
|
+
refuseUnknownRefKeys,
|
|
90
91
|
releaseLease,
|
|
91
92
|
renewPointer,
|
|
92
93
|
renewsPointer,
|
|
94
|
+
requireBool,
|
|
93
95
|
requireFiniteNumber,
|
|
94
96
|
requireInteger,
|
|
95
97
|
requirePositiveInteger,
|
|
@@ -97,7 +99,7 @@ import {
|
|
|
97
99
|
rollbackKnown,
|
|
98
100
|
sameIncarnation,
|
|
99
101
|
sameKeys,
|
|
100
|
-
|
|
102
|
+
sameKnownGens,
|
|
101
103
|
sizeOfFingerprint,
|
|
102
104
|
sizePart,
|
|
103
105
|
splitId,
|
|
@@ -109,23 +111,20 @@ import {
|
|
|
109
111
|
validateChunkKey,
|
|
110
112
|
validateChunkKeyOrder,
|
|
111
113
|
validateChunkRef,
|
|
112
|
-
validateNewRegistryRecord,
|
|
113
|
-
validateRegistryPatch,
|
|
114
114
|
validateSegmentRef,
|
|
115
115
|
validateUserNamespace,
|
|
116
116
|
validateUserRef,
|
|
117
|
-
webCryptoEntropy,
|
|
118
117
|
windowOf,
|
|
119
118
|
withRetry
|
|
120
|
-
} from "./chunk-
|
|
119
|
+
} from "./chunk-2B2QF4FU.js";
|
|
121
120
|
|
|
122
121
|
// src/core/budget.ts
|
|
123
122
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
124
123
|
function resolveBudget(opt, fallback) {
|
|
125
124
|
if (opt === false) return null;
|
|
126
|
-
if (opt !== void 0
|
|
127
|
-
if (typeof opt !== "object" || Array.isArray(opt)) {
|
|
128
|
-
const got = Array.isArray(opt) ? "an array" : `a ${typeof opt}`;
|
|
125
|
+
if (opt !== void 0) {
|
|
126
|
+
if (opt === null || typeof opt !== "object" || Array.isArray(opt)) {
|
|
127
|
+
const got = opt === null ? "null" : Array.isArray(opt) ? "an array" : `a ${typeof opt}`;
|
|
129
128
|
throw new ValidationError(`budget must be { maxRequests } or false; got ${got}`);
|
|
130
129
|
}
|
|
131
130
|
const unknown = Object.keys(opt).filter(
|
|
@@ -137,12 +136,12 @@ function resolveBudget(opt, fallback) {
|
|
|
137
136
|
);
|
|
138
137
|
}
|
|
139
138
|
}
|
|
140
|
-
const maxRequests = opt?.maxRequests
|
|
139
|
+
const maxRequests = opt?.maxRequests === void 0 ? fallback.maxRequests : opt.maxRequests;
|
|
141
140
|
requirePositiveInteger(maxRequests, "budget.maxRequests");
|
|
142
141
|
return { maxRequests };
|
|
143
142
|
}
|
|
144
143
|
function resolvePerOpBudget(opt, storeBudget) {
|
|
145
|
-
if (opt === void 0
|
|
144
|
+
if (opt === void 0) return storeBudget;
|
|
146
145
|
return resolveBudget(opt, storeBudget ?? DEFAULT_BUDGET);
|
|
147
146
|
}
|
|
148
147
|
function checkBudget(budget, projectedRequests, op) {
|
|
@@ -175,8 +174,9 @@ async function refuseAbsent(op, candidates, limit, exists, nameOf) {
|
|
|
175
174
|
const absent = candidates.filter((_, i) => !found2[i]);
|
|
176
175
|
if (absent.length === 0) return;
|
|
177
176
|
const named = absent.map((c) => `"${nameOf(c)}"`).join(", ");
|
|
177
|
+
const [operand, names, does, was, it] = absent.length > 1 ? ["operands", "name segments", "do", "were", "they"] : ["operand", "names a segment", "does", "was", "it"];
|
|
178
178
|
throw new ValidationError(
|
|
179
|
-
`${op}: operand ${named} names
|
|
179
|
+
`${op}: ${operand} ${named} ${names} that ${does} not exist or ${was} dropped, so ${it} would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
180
180
|
);
|
|
181
181
|
}
|
|
182
182
|
|
|
@@ -389,17 +389,17 @@ var AEAD_TAG_BYTES = 16;
|
|
|
389
389
|
var DEFAULT_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BITMAP_BYTES + AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
|
|
390
390
|
|
|
391
391
|
// src/core/id-window.ts
|
|
392
|
-
function assertIdBound(name, bound) {
|
|
392
|
+
function assertIdBound(name, bound, op) {
|
|
393
393
|
if (bound !== void 0 && (!Number.isInteger(bound) || bound < 0 || bound > U32_MAX)) {
|
|
394
|
-
throw new ValidationError(`${name} must be an integer from 0 to ${U32_MAX}`);
|
|
394
|
+
throw new ValidationError(`${op}: ${name} must be an integer from 0 to ${U32_MAX}`);
|
|
395
395
|
}
|
|
396
396
|
}
|
|
397
|
-
function windowOf2(range) {
|
|
397
|
+
function windowOf2(range, op) {
|
|
398
398
|
const after = range?.after;
|
|
399
399
|
const through = range?.through;
|
|
400
400
|
if (after === void 0 && through === void 0) return null;
|
|
401
|
-
assertIdBound("after", after);
|
|
402
|
-
assertIdBound("through", through);
|
|
401
|
+
assertIdBound("after", after, op);
|
|
402
|
+
assertIdBound("through", through, op);
|
|
403
403
|
const lo = after === void 0 ? 0 : after + 1;
|
|
404
404
|
const hi = through ?? U32_MAX;
|
|
405
405
|
if (lo > hi) return "empty";
|
|
@@ -525,62 +525,6 @@ var InvalidationScope = class extends InvalidationRecord {
|
|
|
525
525
|
}
|
|
526
526
|
};
|
|
527
527
|
|
|
528
|
-
// src/core/keys.ts
|
|
529
|
-
var FIELD = " ";
|
|
530
|
-
var NO_NAMESPACE = "/";
|
|
531
|
-
function segmentKey(ref) {
|
|
532
|
-
const ns = ref.namespace === void 0 ? NO_NAMESPACE : encodeNameForKey(ref.namespace);
|
|
533
|
-
return `${ns}${FIELD}${encodeNameForKey(ref.segment)}`;
|
|
534
|
-
}
|
|
535
|
-
function segmentPrefix(ref) {
|
|
536
|
-
return `${segmentKey(ref)}${FIELD}`;
|
|
537
|
-
}
|
|
538
|
-
var KEPT_SEGMENT_KEYS = 1024;
|
|
539
|
-
var KeptSegmentKeys = class {
|
|
540
|
-
constructor(encode, max = KEPT_SEGMENT_KEYS) {
|
|
541
|
-
this.encode = encode;
|
|
542
|
-
this.max = max;
|
|
543
|
-
}
|
|
544
|
-
encode;
|
|
545
|
-
max;
|
|
546
|
-
byNamespace = /* @__PURE__ */ new Map();
|
|
547
|
-
kept = 0;
|
|
548
|
-
/** The segment's encoded key. */
|
|
549
|
-
of(ref) {
|
|
550
|
-
let names = this.byNamespace.get(ref.namespace);
|
|
551
|
-
const found2 = names?.get(ref.segment);
|
|
552
|
-
if (found2 !== void 0) return found2;
|
|
553
|
-
const encoded = this.encode(ref);
|
|
554
|
-
if (this.kept >= this.max) {
|
|
555
|
-
this.byNamespace.clear();
|
|
556
|
-
this.kept = 0;
|
|
557
|
-
names = void 0;
|
|
558
|
-
}
|
|
559
|
-
if (names === void 0) {
|
|
560
|
-
names = /* @__PURE__ */ new Map();
|
|
561
|
-
this.byNamespace.set(ref.namespace, names);
|
|
562
|
-
}
|
|
563
|
-
names.set(ref.segment, encoded);
|
|
564
|
-
this.kept += 1;
|
|
565
|
-
return encoded;
|
|
566
|
-
}
|
|
567
|
-
/** How many encoded keys are kept. */
|
|
568
|
-
get size() {
|
|
569
|
-
return this.kept;
|
|
570
|
-
}
|
|
571
|
-
};
|
|
572
|
-
function chunkKeyUnder(prefix, chunkKey, version) {
|
|
573
|
-
return version === void 0 ? `${prefix}${chunkKey}` : `${prefix}${chunkKey}${FIELD}${version}`;
|
|
574
|
-
}
|
|
575
|
-
function shardOf(key, totalShards) {
|
|
576
|
-
let h = 2166136261;
|
|
577
|
-
for (let i = 0; i < key.length; i++) {
|
|
578
|
-
h ^= key.charCodeAt(i);
|
|
579
|
-
h = Math.imul(h, 16777619);
|
|
580
|
-
}
|
|
581
|
-
return (h >>> 0) % totalShards;
|
|
582
|
-
}
|
|
583
|
-
|
|
584
528
|
// src/core/audit.ts
|
|
585
529
|
var NOOP_AUDIT = {
|
|
586
530
|
onEvent() {
|
|
@@ -613,6 +557,7 @@ function ignoreRejection(returned) {
|
|
|
613
557
|
}
|
|
614
558
|
var RecordingAuditSink = class {
|
|
615
559
|
recorded = [];
|
|
560
|
+
/** Records the event, in emission order. */
|
|
616
561
|
onEvent(event) {
|
|
617
562
|
this.recorded.push(event);
|
|
618
563
|
}
|
|
@@ -622,6 +567,7 @@ var RecordingAuditSink = class {
|
|
|
622
567
|
(e) => e.kind === "segment.collect" ? { ...e, collected: [...e.collected] } : { ...e }
|
|
623
568
|
);
|
|
624
569
|
}
|
|
570
|
+
/** Forgets every recorded event. */
|
|
625
571
|
reset() {
|
|
626
572
|
this.recorded.length = 0;
|
|
627
573
|
}
|
|
@@ -667,6 +613,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
667
613
|
for (const name of OP_NAMES) ops[name] = { count: 0, totalMs: 0 };
|
|
668
614
|
return ops;
|
|
669
615
|
}
|
|
616
|
+
/** Adds the event to the totals. An `advisory` event is a note, not a quantity, and adds nothing. */
|
|
670
617
|
onEvent(event) {
|
|
671
618
|
switch (event.kind) {
|
|
672
619
|
case "storage.get":
|
|
@@ -701,6 +648,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
701
648
|
}
|
|
702
649
|
}
|
|
703
650
|
}
|
|
651
|
+
/** A copy of the totals so far, which later events do not change. */
|
|
704
652
|
snapshot() {
|
|
705
653
|
const ops = /* @__PURE__ */ Object.create(null);
|
|
706
654
|
for (const name of OP_NAMES) ops[name] = { ...this.ops[name] };
|
|
@@ -712,6 +660,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
712
660
|
ops
|
|
713
661
|
};
|
|
714
662
|
}
|
|
663
|
+
/** Sets every total back to zero. */
|
|
715
664
|
reset() {
|
|
716
665
|
this.storage = { gets: 0, bytes: 0, totalMs: 0 };
|
|
717
666
|
this.cache = { hits: 0, misses: 0 };
|
|
@@ -810,6 +759,23 @@ function chunkIds(chunk, chunkKey, w) {
|
|
|
810
759
|
for (let i = 0; i < rem.length; i++) rem[i] = base + (rem[i] & MAX_REMAINDER);
|
|
811
760
|
return rem;
|
|
812
761
|
}
|
|
762
|
+
function smallestFirst(chunks) {
|
|
763
|
+
const out = chunks.slice();
|
|
764
|
+
const sizes = out.map((c) => c.size);
|
|
765
|
+
for (let i = 1; i < out.length; i++) {
|
|
766
|
+
const c = out[i];
|
|
767
|
+
const size = sizes[i];
|
|
768
|
+
let j = i - 1;
|
|
769
|
+
while (j >= 0 && sizes[j] > size) {
|
|
770
|
+
out[j + 1] = out[j];
|
|
771
|
+
sizes[j + 1] = sizes[j];
|
|
772
|
+
j--;
|
|
773
|
+
}
|
|
774
|
+
out[j + 1] = c;
|
|
775
|
+
sizes[j + 1] = size;
|
|
776
|
+
}
|
|
777
|
+
return out;
|
|
778
|
+
}
|
|
813
779
|
function alignKeys(operands, mode) {
|
|
814
780
|
const candidates = [];
|
|
815
781
|
if (mode === "all") {
|
|
@@ -1003,7 +969,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1003
969
|
}
|
|
1004
970
|
}
|
|
1005
971
|
async *iterateRange(seg, range) {
|
|
1006
|
-
const w = windowOf2(range) ?? WHOLE_ID_SPACE;
|
|
972
|
+
const w = windowOf2(range, "iterate") ?? WHOLE_ID_SPACE;
|
|
1007
973
|
if (w === "empty") return;
|
|
1008
974
|
const run = await this.planIterate(seg, w);
|
|
1009
975
|
try {
|
|
@@ -1027,7 +993,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1027
993
|
* 65,536 ids (256 KiB) and is the caller's to keep.
|
|
1028
994
|
*/
|
|
1029
995
|
async *iterateBatches(seg, range) {
|
|
1030
|
-
const w = windowOf2(range);
|
|
996
|
+
const w = windowOf2(range, "iterate");
|
|
1031
997
|
if (w === "empty") return;
|
|
1032
998
|
const run = await this.planIterate(seg, w);
|
|
1033
999
|
try {
|
|
@@ -1167,9 +1133,9 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1167
1133
|
* `after` turn out to be fewer than `n`, and one that cuts its last chunk may read it and yield nothing.
|
|
1168
1134
|
*/
|
|
1169
1135
|
async *everyNth(seg, n, range) {
|
|
1170
|
-
requirePositiveInteger(n, "n");
|
|
1136
|
+
requirePositiveInteger(n, "everyNth: n");
|
|
1171
1137
|
const epoch = this.scope.epoch;
|
|
1172
|
-
const w = windowOf2(range);
|
|
1138
|
+
const w = windowOf2(range, "everyNth");
|
|
1173
1139
|
if (w === "empty") return;
|
|
1174
1140
|
const span = w ?? WHOLE_ID_SPACE;
|
|
1175
1141
|
const { loKey, loRem, hiKey, hiRem } = span;
|
|
@@ -1207,10 +1173,10 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1207
1173
|
const hits = [];
|
|
1208
1174
|
let before = 0;
|
|
1209
1175
|
for (const key of keys) {
|
|
1210
|
-
const
|
|
1211
|
-
const size = counts.get(key) -
|
|
1176
|
+
const skipped2 = key === loKey && cutsFirst ? below : 0;
|
|
1177
|
+
const size = counts.get(key) - skipped2;
|
|
1212
1178
|
const next = (Math.floor(before / n) + 1) * n;
|
|
1213
|
-
if (next <= before + size) hits.push({ key, index:
|
|
1179
|
+
if (next <= before + size) hits.push({ key, index: skipped2 + next - before - 1 });
|
|
1214
1180
|
before += size;
|
|
1215
1181
|
}
|
|
1216
1182
|
if (hits.length === 0) return;
|
|
@@ -1324,33 +1290,30 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1324
1290
|
* for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
|
|
1325
1291
|
* errors are the id reads' own.
|
|
1326
1292
|
*/
|
|
1327
|
-
intersectChunks(segs, options) {
|
|
1328
|
-
return this.combineChunks(segs, options?.exclude ?? [], "all", "intersect", options);
|
|
1293
|
+
intersectChunks(segs, options, label) {
|
|
1294
|
+
return this.combineChunks(segs, options?.exclude ?? [], "all", "intersect", options, label);
|
|
1329
1295
|
}
|
|
1330
1296
|
/** {@link union}, as chunks: see {@link intersectChunks}. */
|
|
1331
|
-
unionChunks(segs, options) {
|
|
1332
|
-
return this.combineChunks(segs, options?.exclude ?? [], "any", "union", options);
|
|
1297
|
+
unionChunks(segs, options, label) {
|
|
1298
|
+
return this.combineChunks(segs, options?.exclude ?? [], "any", "union", options, label);
|
|
1333
1299
|
}
|
|
1334
1300
|
/** {@link andNot}, as chunks: see {@link intersectChunks}. */
|
|
1335
|
-
andNotChunks(seg, excludes, options) {
|
|
1336
|
-
return this.combineChunks([seg], excludes, "all", "andNot", options);
|
|
1301
|
+
andNotChunks(seg, excludes, options, label) {
|
|
1302
|
+
return this.combineChunks([seg], excludes, "all", "andNot", options, label);
|
|
1337
1303
|
}
|
|
1338
1304
|
/**
|
|
1339
1305
|
* Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
|
|
1340
1306
|
* absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
|
|
1341
1307
|
* read and the batch read share it, so the two cannot disagree about what is read or charged.
|
|
1342
1308
|
*/
|
|
1343
|
-
async combinePlan(segs, excludeSegs, mode, op, options) {
|
|
1309
|
+
async combinePlan(segs, excludeSegs, mode, op, options, label = op) {
|
|
1344
1310
|
if (segs.length === 0) {
|
|
1345
1311
|
throw new ValidationError(`${op} requires at least one segment`);
|
|
1346
1312
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
}
|
|
1350
|
-
const limit = options?.concurrency ?? DEFAULT_INTERSECT_CONCURRENCY;
|
|
1351
|
-
requirePositiveInteger(limit, "concurrency");
|
|
1313
|
+
const limit = options?.concurrency === void 0 ? DEFAULT_INTERSECT_CONCURRENCY : options.concurrency;
|
|
1314
|
+
requirePositiveInteger(limit, `${label}: concurrency`);
|
|
1352
1315
|
const budget = resolvePerOpBudget(options?.budget, this.budget);
|
|
1353
|
-
const w = windowOf2(options);
|
|
1316
|
+
const w = windowOf2(options, label);
|
|
1354
1317
|
if (w === "empty") return null;
|
|
1355
1318
|
const epoch = this.scope.epoch;
|
|
1356
1319
|
const [operands, excludes] = await this.combineOperands(segs, excludeSegs, w, -1);
|
|
@@ -1412,7 +1375,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1412
1375
|
const streams = [];
|
|
1413
1376
|
const batches = [];
|
|
1414
1377
|
const streamExcludes = mode === "any" || operands.length === 1;
|
|
1415
|
-
const
|
|
1378
|
+
const open = (o, wanted) => {
|
|
1416
1379
|
o.streamed = this.openStreamed(
|
|
1417
1380
|
o.seg,
|
|
1418
1381
|
wanted === void 0 ? keys : keys.filter(wanted),
|
|
@@ -1424,8 +1387,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1424
1387
|
);
|
|
1425
1388
|
if (o.streamed) streams.push(o.streamed);
|
|
1426
1389
|
};
|
|
1427
|
-
for (const o of operands)
|
|
1428
|
-
if (streamExcludes) for (const e of excludes)
|
|
1390
|
+
for (const o of operands) open(o, mode === "all" ? void 0 : (k) => o.keys.has(k));
|
|
1391
|
+
if (streamExcludes) for (const e of excludes) open(e, (k) => e.keys.has(k));
|
|
1429
1392
|
else {
|
|
1430
1393
|
const shared = /* @__PURE__ */ new Map();
|
|
1431
1394
|
for (const e of excludes) this.openBatch(e, moves, limit, batches, shared);
|
|
@@ -1541,8 +1504,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1541
1504
|
}
|
|
1542
1505
|
}
|
|
1543
1506
|
/** The chunk read: the plan every combine shares, then each key's result, cut to the range at its edges. */
|
|
1544
|
-
async *combineChunks(segs, excludeSegs, mode, op, options) {
|
|
1545
|
-
const run = await this.combinePlan(segs, excludeSegs, mode, op, options);
|
|
1507
|
+
async *combineChunks(segs, excludeSegs, mode, op, options, label) {
|
|
1508
|
+
const run = await this.combinePlan(segs, excludeSegs, mode, op, options, label);
|
|
1546
1509
|
if (run === null) return;
|
|
1547
1510
|
const { w } = run;
|
|
1548
1511
|
try {
|
|
@@ -1559,7 +1522,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1559
1522
|
* The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
|
|
1560
1523
|
* Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
|
|
1561
1524
|
*
|
|
1562
|
-
* The accumulator is a **clone** of
|
|
1525
|
+
* The accumulator is a **clone** of one operand's chunk (the first, or for a codec that asks the one with the fewest values), and that clone is load-bearing: every other
|
|
1563
1526
|
* bitmap here is the cached, shared Storage instance and must never be mutated, or the cache is poisoned for
|
|
1564
1527
|
* every later reader. The remaining operands and the excludes are only *read* by the in-place ops, so they are
|
|
1565
1528
|
* used as-is — one clone per key, not one per operand.
|
|
@@ -1573,10 +1536,11 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1573
1536
|
const fetched = await Promise.all(reads.map((op) => this.operandChunk(op, chunkKey, moves)));
|
|
1574
1537
|
const chunks = together ? fetched.slice(0, present.length) : fetched;
|
|
1575
1538
|
if (mode === "all" && chunks.some((c) => c === null)) return null;
|
|
1576
|
-
const
|
|
1539
|
+
const ordered = mode === "all" && chunks.length > 2 && this.codec.intersectSmallestFirst === true ? smallestFirst(chunks) : chunks;
|
|
1540
|
+
const first = ordered[0];
|
|
1577
1541
|
const acc = first ? first.clone() : this.codec.empty();
|
|
1578
|
-
for (let i = 1; i <
|
|
1579
|
-
const other =
|
|
1542
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
1543
|
+
const other = ordered[i];
|
|
1580
1544
|
if (other === null || other === void 0) continue;
|
|
1581
1545
|
if (mode === "all") {
|
|
1582
1546
|
acc.andInPlace(other);
|
|
@@ -1651,8 +1615,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1651
1615
|
askBatch(batch, key) {
|
|
1652
1616
|
if (batch.closed)
|
|
1653
1617
|
return Promise.reject(new Error("the combine this read belongs to has stopped"));
|
|
1654
|
-
return new Promise((
|
|
1655
|
-
batch.pending.push({ key, resolve
|
|
1618
|
+
return new Promise((resolve, reject) => {
|
|
1619
|
+
batch.pending.push({ key, resolve, reject });
|
|
1656
1620
|
batch.held.add(key);
|
|
1657
1621
|
batch.oldest = Math.min(batch.oldest, batch.window?.slotOf(key) ?? 0);
|
|
1658
1622
|
if (batch.held.size >= batch.half) this.flushBatch(batch);
|
|
@@ -1725,8 +1689,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1725
1689
|
const streams = /* @__PURE__ */ new Set();
|
|
1726
1690
|
const streamKeys = [];
|
|
1727
1691
|
for (const ask of asks) {
|
|
1728
|
-
const
|
|
1729
|
-
if (!
|
|
1692
|
+
const open = this.scope.hasOpenRead(this.chunkCacheKey({ ...seg, chunkKey: ask.key }, gen));
|
|
1693
|
+
if (!open && streamKeys[streamKeys.length - 1] !== ask.key) {
|
|
1730
1694
|
streams.add(ask);
|
|
1731
1695
|
streamKeys.push(ask.key);
|
|
1732
1696
|
}
|
|
@@ -2000,7 +1964,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
2000
1964
|
* the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
|
|
2001
1965
|
*
|
|
2002
1966
|
* The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
|
|
2003
|
-
* not a *destruction*: a segment that was erased from, dropped,
|
|
1967
|
+
* not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
|
|
2004
1968
|
* whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
|
|
2005
1969
|
* process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
|
|
2006
1970
|
* no storage read to intercept.
|
|
@@ -2065,8 +2029,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
2065
2029
|
}
|
|
2066
2030
|
if (report && this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
2067
2031
|
}
|
|
2068
|
-
const
|
|
2069
|
-
if (
|
|
2032
|
+
const open = this.scope.openRead(cacheKey);
|
|
2033
|
+
if (open) return open;
|
|
2070
2034
|
const token = {};
|
|
2071
2035
|
const entry = { token, read: this.fetchChunk(ref, gen, cacheKey, token) };
|
|
2072
2036
|
this.scope.beginRead(cacheKey, entry);
|
|
@@ -2356,6 +2320,22 @@ function requireCodec(codec, api) {
|
|
|
2356
2320
|
// src/export/index.ts
|
|
2357
2321
|
var UTF8 = new TextEncoder();
|
|
2358
2322
|
var DEFAULT_NDJSON_BATCH_BYTES = 64 * 1024;
|
|
2323
|
+
var EXPORT_BATCH = 1 << 16;
|
|
2324
|
+
async function addAll(bitmap, ids) {
|
|
2325
|
+
const batch = new Uint32Array(EXPORT_BATCH);
|
|
2326
|
+
let held = 0;
|
|
2327
|
+
let count = 0;
|
|
2328
|
+
for await (const id of ids) {
|
|
2329
|
+
batch[held++] = id;
|
|
2330
|
+
count += 1;
|
|
2331
|
+
if (held === EXPORT_BATCH) {
|
|
2332
|
+
bitmap.addMany(batch);
|
|
2333
|
+
held = 0;
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
if (held > 0) bitmap.addMany(batch.subarray(0, held));
|
|
2337
|
+
return count;
|
|
2338
|
+
}
|
|
2359
2339
|
var errMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
2360
2340
|
async function runExport(reader, registry, sink, options = {}) {
|
|
2361
2341
|
if (sink === null || typeof sink !== "object" || typeof sink.open !== "function") {
|
|
@@ -2374,8 +2354,8 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2374
2354
|
const ext = format === "roaring" ? ".roaring" : ".ndjson";
|
|
2375
2355
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
2376
2356
|
const segments = [];
|
|
2377
|
-
const
|
|
2378
|
-
const
|
|
2357
|
+
const failed2 = [];
|
|
2358
|
+
const skipped2 = [];
|
|
2379
2359
|
let totalIds = 0;
|
|
2380
2360
|
if (typeof reader?.segment !== "function") {
|
|
2381
2361
|
throw new ValidationError(
|
|
@@ -2393,7 +2373,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2393
2373
|
await writer.abort?.();
|
|
2394
2374
|
} catch {
|
|
2395
2375
|
}
|
|
2396
|
-
|
|
2376
|
+
skipped2.push({ segment: ref.segment, namespace: ref.namespace, reason: "destroyed" });
|
|
2397
2377
|
};
|
|
2398
2378
|
const exportRef = async (ref) => {
|
|
2399
2379
|
let writer;
|
|
@@ -2415,10 +2395,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2415
2395
|
writer = await sink.open(ref, ext);
|
|
2416
2396
|
if (format === "roaring") {
|
|
2417
2397
|
const bm = requireCodec(options.codec, "runExport({ format: 'roaring' })").empty();
|
|
2418
|
-
|
|
2419
|
-
bm.add(id);
|
|
2420
|
-
count += 1;
|
|
2421
|
-
}
|
|
2398
|
+
count = await addAll(bm, ids);
|
|
2422
2399
|
if (count === 0 && await rowIsGone(ref)) return await skipGone(ref, writer);
|
|
2423
2400
|
const out = bm.serialize();
|
|
2424
2401
|
await writer.write(out);
|
|
@@ -2451,13 +2428,13 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2451
2428
|
} catch {
|
|
2452
2429
|
}
|
|
2453
2430
|
}
|
|
2454
|
-
|
|
2431
|
+
failed2.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2455
2432
|
return;
|
|
2456
2433
|
}
|
|
2457
2434
|
try {
|
|
2458
2435
|
await writer.close();
|
|
2459
2436
|
} catch (err) {
|
|
2460
|
-
|
|
2437
|
+
failed2.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2461
2438
|
return;
|
|
2462
2439
|
}
|
|
2463
2440
|
segments.push({ segment: ref.segment, namespace: ref.namespace, count, bytes });
|
|
@@ -2466,7 +2443,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2466
2443
|
for await (const rec of registry.list(options.namespace)) {
|
|
2467
2444
|
if (options.namespace === void 0 && isReservedRow(rec)) continue;
|
|
2468
2445
|
if (rec.status === "destroyed") {
|
|
2469
|
-
|
|
2446
|
+
skipped2.push({ segment: rec.segment, namespace: rec.namespace, reason: "destroyed" });
|
|
2470
2447
|
continue;
|
|
2471
2448
|
}
|
|
2472
2449
|
await exportRef({ segment: rec.segment, namespace: rec.namespace });
|
|
@@ -2477,14 +2454,14 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2477
2454
|
totalSegments: segments.length,
|
|
2478
2455
|
totalIds,
|
|
2479
2456
|
segments,
|
|
2480
|
-
failed,
|
|
2481
|
-
skipped
|
|
2457
|
+
failed: failed2,
|
|
2458
|
+
skipped: skipped2
|
|
2482
2459
|
};
|
|
2483
2460
|
}
|
|
2484
2461
|
|
|
2485
2462
|
// src/core/discover.ts
|
|
2486
2463
|
async function segmentExists(ref, registry) {
|
|
2487
|
-
validateUserRef(ref);
|
|
2464
|
+
validateUserRef(ref, "exists");
|
|
2488
2465
|
const record = await registry.get(ref);
|
|
2489
2466
|
return record !== null && record.status !== "destroyed" && record.currentGen !== null;
|
|
2490
2467
|
}
|
|
@@ -2502,1086 +2479,191 @@ async function* listSegments(registry, options = {}) {
|
|
|
2502
2479
|
}
|
|
2503
2480
|
}
|
|
2504
2481
|
|
|
2505
|
-
// src/
|
|
2506
|
-
|
|
2482
|
+
// src/core/reader-defaults.ts
|
|
2483
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
2484
|
+
var REFRESH_RETRY_MS = 500;
|
|
2485
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
2486
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
2487
|
+
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
2488
|
+
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
2507
2489
|
|
|
2508
|
-
// src/
|
|
2509
|
-
|
|
2490
|
+
// src/core/crbm/integrity.ts
|
|
2491
|
+
var crbmIntegrity = (message) => new IntegrityError(`${message}; re-load from source or roll back`);
|
|
2510
2492
|
|
|
2511
|
-
// src/core/
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
length = 0;
|
|
2520
|
-
write(bytes) {
|
|
2521
|
-
this.chunks.push(new Uint8Array(bytes));
|
|
2522
|
-
this.length += bytes.length;
|
|
2523
|
-
return Promise.resolve();
|
|
2524
|
-
}
|
|
2525
|
-
bytes() {
|
|
2526
|
-
const out = new Uint8Array(this.length);
|
|
2527
|
-
let offset = 0;
|
|
2528
|
-
for (const chunk of this.chunks) {
|
|
2529
|
-
out.set(chunk, offset);
|
|
2530
|
-
offset += chunk.length;
|
|
2531
|
-
}
|
|
2532
|
-
return out;
|
|
2533
|
-
}
|
|
2534
|
-
};
|
|
2535
|
-
var BufferReader = class {
|
|
2536
|
-
constructor(buffer) {
|
|
2537
|
-
this.buffer = buffer;
|
|
2538
|
-
}
|
|
2539
|
-
buffer;
|
|
2540
|
-
getRange(offset, length) {
|
|
2541
|
-
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0 || offset + length > this.buffer.length) {
|
|
2542
|
-
throw new ValidationError(
|
|
2543
|
-
`range [${offset}, ${offset + length}) out of bounds for ${this.buffer.length}-byte blob`
|
|
2544
|
-
);
|
|
2545
|
-
}
|
|
2546
|
-
return Promise.resolve(this.buffer.subarray(offset, offset + length));
|
|
2493
|
+
// src/core/crbm/crc32c.ts
|
|
2494
|
+
var TABLES = (() => {
|
|
2495
|
+
const POLY = 2197175160;
|
|
2496
|
+
const t0 = new Uint32Array(256);
|
|
2497
|
+
for (let n = 0; n < 256; n++) {
|
|
2498
|
+
let c = n;
|
|
2499
|
+
for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
|
|
2500
|
+
t0[n] = c >>> 0;
|
|
2547
2501
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2502
|
+
const tables = [t0];
|
|
2503
|
+
for (let k = 1; k < 8; k++) {
|
|
2504
|
+
const prev = tables[k - 1];
|
|
2505
|
+
const next = new Uint32Array(256);
|
|
2506
|
+
for (let n = 0; n < 256; n++) {
|
|
2507
|
+
const v = prev[n];
|
|
2508
|
+
next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
|
|
2553
2509
|
}
|
|
2554
|
-
|
|
2555
|
-
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
2556
|
-
return Promise.resolve({ bytes: this.buffer.subarray(size - take), size });
|
|
2557
|
-
}
|
|
2558
|
-
};
|
|
2559
|
-
|
|
2560
|
-
// src/drivers/memory.ts
|
|
2561
|
-
var MemoryRegistryDriver = class {
|
|
2562
|
-
rows = /* @__PURE__ */ new Map();
|
|
2563
|
-
now;
|
|
2564
|
-
entropy;
|
|
2565
|
-
seq = 0;
|
|
2566
|
-
constructor(options = {}) {
|
|
2567
|
-
this.now = options.now ?? (() => Date.now());
|
|
2568
|
-
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
2569
|
-
}
|
|
2570
|
-
nextToken(incarnation) {
|
|
2571
|
-
this.seq += 1;
|
|
2572
|
-
return `${incarnation}.${this.seq}.${drawWrite(this.entropy)}`;
|
|
2573
|
-
}
|
|
2574
|
-
/** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
|
|
2575
|
-
capabilities() {
|
|
2576
|
-
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
2577
|
-
}
|
|
2578
|
-
/** Every write reports its token to `onToken` before it is applied. */
|
|
2579
|
-
get reportsWriteTokens() {
|
|
2580
|
-
return true;
|
|
2581
|
-
}
|
|
2582
|
-
async get(ref) {
|
|
2583
|
-
validateSegmentRef(ref);
|
|
2584
|
-
const row = this.rows.get(segmentKey(ref));
|
|
2585
|
-
return row ? structuredClone(row) : null;
|
|
2510
|
+
tables.push(next);
|
|
2586
2511
|
}
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2512
|
+
return tables;
|
|
2513
|
+
})();
|
|
2514
|
+
var T0 = TABLES[0];
|
|
2515
|
+
var T1 = TABLES[1];
|
|
2516
|
+
var T2 = TABLES[2];
|
|
2517
|
+
var T3 = TABLES[3];
|
|
2518
|
+
var T4 = TABLES[4];
|
|
2519
|
+
var T5 = TABLES[5];
|
|
2520
|
+
var T6 = TABLES[6];
|
|
2521
|
+
var T7 = TABLES[7];
|
|
2522
|
+
function crc32c(bytes, seed = 0) {
|
|
2523
|
+
let crc = (seed ^ 4294967295) >>> 0;
|
|
2524
|
+
let i = 0;
|
|
2525
|
+
const n8 = bytes.length - bytes.length % 8;
|
|
2526
|
+
for (; i < n8; i += 8) {
|
|
2527
|
+
crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
|
|
2528
|
+
crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
|
|
2598
2529
|
}
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
* as the write, in memory, so a stale one fails exactly as a lost race does.
|
|
2602
|
-
*/
|
|
2603
|
-
async compareAndSwap(ref, expected, patch, options) {
|
|
2604
|
-
validateSegmentRef(ref);
|
|
2605
|
-
const checked = validateRegistryPatch(patch);
|
|
2606
|
-
const key = segmentKey(ref);
|
|
2607
|
-
const existing = this.rows.get(key);
|
|
2608
|
-
if (!existing || existing.token !== expected) {
|
|
2609
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2610
|
-
}
|
|
2611
|
-
const token = this.nextToken(incarnationOf(existing.token));
|
|
2612
|
-
options?.onToken?.(token);
|
|
2613
|
-
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
2614
|
-
return { token };
|
|
2530
|
+
for (; i < bytes.length; i++) {
|
|
2531
|
+
crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
|
|
2615
2532
|
}
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2533
|
+
return (crc ^ 4294967295) >>> 0;
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
// src/core/crbm/plan-reads.ts
|
|
2537
|
+
var MAX_COALESCE_GAP_BYTES = 256 * 1024;
|
|
2538
|
+
var MAX_COALESCED_READ_BYTES = DEFAULT_MAX_BITMAP_BYTES;
|
|
2539
|
+
var MAX_RANGES_IN_FLIGHT = 32;
|
|
2540
|
+
function* planChunkReads(extents, region) {
|
|
2541
|
+
let chunks = [];
|
|
2542
|
+
let start = 0;
|
|
2543
|
+
let end = 0;
|
|
2544
|
+
let previousEnd = region.start;
|
|
2545
|
+
for (const extent of extents) {
|
|
2546
|
+
const extentEnd = extent.offset + extent.length;
|
|
2547
|
+
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
2548
|
+
throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
|
|
2620
2549
|
}
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2550
|
+
previousEnd = extentEnd;
|
|
2551
|
+
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
2552
|
+
chunks.push(extent);
|
|
2553
|
+
end = extentEnd;
|
|
2554
|
+
continue;
|
|
2627
2555
|
}
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
objects = /* @__PURE__ */ new Map();
|
|
2633
|
-
stored = 0;
|
|
2634
|
-
capabilities() {
|
|
2635
|
-
return {
|
|
2636
|
-
rangeRead: true,
|
|
2637
|
-
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2638
|
-
conditionalPut: true,
|
|
2639
|
-
conditionalDelete: true
|
|
2640
|
-
};
|
|
2641
|
-
}
|
|
2642
|
-
async putImmutable(key, write) {
|
|
2643
|
-
const k = genObjectKey(key);
|
|
2644
|
-
const sink = new BufferSink();
|
|
2645
|
-
await write(sink);
|
|
2646
|
-
const body = sink.bytes();
|
|
2647
|
-
if (this.objects.has(k)) {
|
|
2648
|
-
throw new WriteConflictError(
|
|
2649
|
-
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
2650
|
-
);
|
|
2556
|
+
if (chunks.length > 0) {
|
|
2557
|
+
const done = chunks;
|
|
2558
|
+
chunks = [];
|
|
2559
|
+
yield { offset: start, length: end - start, chunks: done };
|
|
2651
2560
|
}
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2561
|
+
chunks.push(extent);
|
|
2562
|
+
start = extent.offset;
|
|
2563
|
+
end = extentEnd;
|
|
2655
2564
|
}
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
}
|
|
2665
|
-
return body.slice(offset, offset + length);
|
|
2565
|
+
if (chunks.length > 0) yield { offset: start, length: end - start, chunks };
|
|
2566
|
+
}
|
|
2567
|
+
|
|
2568
|
+
// src/core/crbm/varint.ts
|
|
2569
|
+
var U32_CEIL = U32_MAX + 1;
|
|
2570
|
+
function varintLength(value) {
|
|
2571
|
+
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
2572
|
+
throw new RangeError(`varint value must be a u32; got ${value}`);
|
|
2666
2573
|
}
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2574
|
+
if (value < 128) return 1;
|
|
2575
|
+
if (value < 16384) return 2;
|
|
2576
|
+
if (value < 2097152) return 3;
|
|
2577
|
+
if (value < 268435456) return 4;
|
|
2578
|
+
return 5;
|
|
2579
|
+
}
|
|
2580
|
+
function writeVarintAt(out, pos, value) {
|
|
2581
|
+
varintLength(value);
|
|
2582
|
+
let v = value;
|
|
2583
|
+
let at = pos;
|
|
2584
|
+
while (v > 127) {
|
|
2585
|
+
out[at++] = v & 127 | 128;
|
|
2586
|
+
v >>>= 7;
|
|
2672
2587
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2588
|
+
out[at++] = v;
|
|
2589
|
+
return at;
|
|
2590
|
+
}
|
|
2591
|
+
function readVarint(bytes, offset) {
|
|
2592
|
+
let result = 0;
|
|
2593
|
+
let shift = 0;
|
|
2594
|
+
let pos = offset;
|
|
2595
|
+
for (let i = 0; i < 5; i++) {
|
|
2596
|
+
if (pos >= bytes.length) {
|
|
2597
|
+
throw crbmIntegrity("varint truncated past end of buffer");
|
|
2681
2598
|
}
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
const prefix = `${segmentKey(ref)} `;
|
|
2687
|
-
for (const k of this.objects.keys()) {
|
|
2688
|
-
if (k.startsWith(prefix)) {
|
|
2689
|
-
yield {
|
|
2690
|
-
namespace: ref.namespace,
|
|
2691
|
-
segment: ref.segment,
|
|
2692
|
-
generation: Number(k.slice(prefix.length))
|
|
2693
|
-
};
|
|
2694
|
-
}
|
|
2599
|
+
const byte = bytes[pos];
|
|
2600
|
+
pos += 1;
|
|
2601
|
+
if (i === 4 && byte > 15) {
|
|
2602
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
2695
2603
|
}
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
if (stored === void 0) {
|
|
2700
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
2604
|
+
result = (result | (byte & 127) << shift) >>> 0;
|
|
2605
|
+
if ((byte & 128) === 0) {
|
|
2606
|
+
return { value: result, next: pos };
|
|
2701
2607
|
}
|
|
2702
|
-
|
|
2608
|
+
shift += 7;
|
|
2703
2609
|
}
|
|
2704
|
-
|
|
2705
|
-
function genObjectKey(key) {
|
|
2706
|
-
validateSegmentRef(key);
|
|
2707
|
-
requireInteger(key.generation, { min: 0, what: "generation" });
|
|
2708
|
-
return `${segmentKey(key)} ${key.generation}`;
|
|
2610
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
2709
2611
|
}
|
|
2710
2612
|
|
|
2711
|
-
// src/
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
// src/drivers/localfs/paths.ts
|
|
2718
|
-
import { join } from "node:path";
|
|
2719
|
-
var MAX_COMPONENT_BYTES = 255;
|
|
2720
|
-
var TEMP_SUFFIX_BYTES = 1 + 36 + 4;
|
|
2721
|
-
function assertFits(component, extraBytes) {
|
|
2722
|
-
if (component.length + extraBytes > MAX_COMPONENT_BYTES) {
|
|
2723
|
-
throw new ValidationError(
|
|
2724
|
-
`a name is too long for a file name on this backend: the namespace, or the segment with its generation and the temp file's suffix, must fit in ${MAX_COMPONENT_BYTES} bytes once encoded`
|
|
2725
|
-
);
|
|
2726
|
-
}
|
|
2613
|
+
// src/core/crbm/reader.ts
|
|
2614
|
+
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
2615
|
+
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
2616
|
+
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
2617
|
+
function indexCapacity(indexLength) {
|
|
2618
|
+
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
2727
2619
|
}
|
|
2728
|
-
function
|
|
2729
|
-
|
|
2730
|
-
assertFits(namespacePathPart(key.namespace), 0);
|
|
2731
|
-
assertFits(storageObjectFilename(key.segment, key.generation), TEMP_SUFFIX_BYTES);
|
|
2732
|
-
}
|
|
2733
|
-
function assertRegistryNamesFit(ref) {
|
|
2734
|
-
validateSegmentRef(ref);
|
|
2735
|
-
assertFits(namespacePathPart(ref.namespace), 0);
|
|
2736
|
-
assertFits(`${encodeNameForPath(ref.segment)}${REGISTRY_SUFFIX}`, TEMP_SUFFIX_BYTES);
|
|
2737
|
-
}
|
|
2738
|
-
function segmentsDir(root, ref) {
|
|
2739
|
-
validateSegmentRef(ref);
|
|
2740
|
-
return join(root, namespacePathPart(ref.namespace), "segments");
|
|
2741
|
-
}
|
|
2742
|
-
function storageObjectPath(root, key) {
|
|
2743
|
-
validateSegmentRef(key);
|
|
2744
|
-
requireInteger(key.generation, { min: 0, what: "generation" });
|
|
2745
|
-
return join(segmentsDir(root, key), storageObjectFilename(key.segment, key.generation));
|
|
2746
|
-
}
|
|
2747
|
-
function storageObjectFilename(segment, generation) {
|
|
2748
|
-
return `${encodeNameForPath(segment)}.${generation}.crbm`;
|
|
2749
|
-
}
|
|
2750
|
-
function parseGeneration(segment, filename) {
|
|
2751
|
-
const prefix = `${encodeNameForPath(segment)}.`;
|
|
2752
|
-
const suffix = ".crbm";
|
|
2753
|
-
if (!filename.startsWith(prefix) || !filename.endsWith(suffix)) return null;
|
|
2754
|
-
const middle = filename.slice(prefix.length, filename.length - suffix.length);
|
|
2755
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
2756
|
-
const gen = Number(middle);
|
|
2757
|
-
return Number.isSafeInteger(gen) ? gen : null;
|
|
2758
|
-
}
|
|
2759
|
-
var REGISTRY_SUFFIX = ".reg";
|
|
2760
|
-
function registryDir(root, namespace) {
|
|
2761
|
-
if (namespace !== void 0) validateSegmentRef({ segment: "x", namespace });
|
|
2762
|
-
return join(root, namespacePathPart(namespace), "registry");
|
|
2763
|
-
}
|
|
2764
|
-
function registryRowPath(root, ref) {
|
|
2765
|
-
validateSegmentRef(ref);
|
|
2766
|
-
return join(
|
|
2767
|
-
registryDir(root, ref.namespace),
|
|
2768
|
-
`${encodeNameForPath(ref.segment)}${REGISTRY_SUFFIX}`
|
|
2769
|
-
);
|
|
2620
|
+
function magicMatches(bytes, offset) {
|
|
2621
|
+
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
2770
2622
|
}
|
|
2771
|
-
function
|
|
2772
|
-
if (!
|
|
2773
|
-
|
|
2774
|
-
const segment = decodeNameFromPath(stem);
|
|
2775
|
-
if (encodeNameForPath(segment) !== stem) return null;
|
|
2776
|
-
try {
|
|
2777
|
-
validateSegmentRef({ segment });
|
|
2778
|
-
} catch {
|
|
2779
|
-
return null;
|
|
2623
|
+
function checkSize(tail, size) {
|
|
2624
|
+
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
2625
|
+
throw crbmIntegrity(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
2780
2626
|
}
|
|
2781
|
-
return segment;
|
|
2782
2627
|
}
|
|
2783
|
-
function
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
try {
|
|
2788
|
-
validateSegmentRef({ segment: "x", namespace });
|
|
2789
|
-
} catch {
|
|
2790
|
-
return null;
|
|
2628
|
+
function checkedFooter(tail, size) {
|
|
2629
|
+
checkSize(tail, size);
|
|
2630
|
+
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
2631
|
+
throw crbmIntegrity(`.crbm too small: ${size}B`);
|
|
2791
2632
|
}
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
// src/drivers/localfs/exact-case.ts
|
|
2796
|
-
import { readdir, realpath, stat } from "node:fs/promises";
|
|
2797
|
-
import { dirname as dirname2, join as join3, parse, relative, resolve, sep } from "node:path";
|
|
2798
|
-
|
|
2799
|
-
// src/drivers/localfs/fs-util.ts
|
|
2800
|
-
import { constants as FS } from "node:fs";
|
|
2801
|
-
import { randomUUID } from "node:crypto";
|
|
2802
|
-
import { lstat, mkdir, opendir, open, unlink } from "node:fs/promises";
|
|
2803
|
-
import { dirname, join as join2 } from "node:path";
|
|
2804
|
-
var isCode = (err, code) => err instanceof Error && err.code === code;
|
|
2805
|
-
var isAbsent = (err) => isCode(err, "ENOENT") || isCode(err, "ENOTDIR") || isCode(err, "ENAMETOOLONG");
|
|
2806
|
-
var TRANSIENT_FS_CODES = /* @__PURE__ */ new Set(["EBUSY", "EAGAIN", "EMFILE", "ENFILE", "ETIMEDOUT"]);
|
|
2807
|
-
var isFsTransient = (err) => err instanceof Error && TRANSIENT_FS_CODES.has(err.code ?? "");
|
|
2808
|
-
var mapFsError = (err) => isFsTransient(err) ? new TransientError(`transient filesystem fault: ${err.code}`, { cause: err }) : err;
|
|
2809
|
-
var O_NOFOLLOW = FS.O_NOFOLLOW ?? 0;
|
|
2810
|
-
var FILE_MODE = 384;
|
|
2811
|
-
var DIR_MODE = 448;
|
|
2812
|
-
async function fsyncDir(dir) {
|
|
2813
|
-
try {
|
|
2814
|
-
const handle = await open(dir, "r");
|
|
2815
|
-
try {
|
|
2816
|
-
await handle.sync();
|
|
2817
|
-
} finally {
|
|
2818
|
-
await handle.close();
|
|
2819
|
-
}
|
|
2820
|
-
} catch {
|
|
2633
|
+
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
2634
|
+
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
2635
|
+
throw crbmIntegrity(".crbm end magic mismatch");
|
|
2821
2636
|
}
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset);
|
|
2827
|
-
if (bytesWritten <= 0) {
|
|
2828
|
-
throw Object.assign(new Error("a write made no progress: the device is full"), {
|
|
2829
|
-
code: "ENOSPC"
|
|
2830
|
-
});
|
|
2831
|
-
}
|
|
2832
|
-
offset += bytesWritten;
|
|
2637
|
+
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
2638
|
+
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
2639
|
+
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
2640
|
+
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
2833
2641
|
}
|
|
2642
|
+
return { footer, fview, storedFooterCrc };
|
|
2834
2643
|
}
|
|
2835
|
-
async function
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2644
|
+
async function readFooter(blob) {
|
|
2645
|
+
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2646
|
+
const { fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
2647
|
+
return {
|
|
2648
|
+
encrypted: (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0,
|
|
2649
|
+
footerCrc: storedFooterCrc
|
|
2650
|
+
};
|
|
2841
2651
|
}
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
const last = lastSwept.get(dir);
|
|
2850
|
-
if (last !== void 0 && now - last < SWEEP_INTERVAL_MS) return;
|
|
2851
|
-
lastSwept.set(dir, now);
|
|
2852
|
-
try {
|
|
2853
|
-
const entries = await opendir(dir);
|
|
2854
|
-
for await (const entry of entries) {
|
|
2855
|
-
if (!TEMP_NAME.test(entry.name)) continue;
|
|
2856
|
-
const path = join2(dir, entry.name);
|
|
2857
|
-
const info = await lstat(path).catch(() => void 0);
|
|
2858
|
-
if (info?.isFile() === true && info.mtimeMs < now - ORPHAN_TEMP_AGE_MS) {
|
|
2859
|
-
await unlink(path).catch(() => {
|
|
2860
|
-
});
|
|
2861
|
-
}
|
|
2862
|
-
}
|
|
2863
|
-
} catch {
|
|
2652
|
+
async function footerSaysEncrypted(blob) {
|
|
2653
|
+
return (await readFooter(blob)).encrypted;
|
|
2654
|
+
}
|
|
2655
|
+
function readU64(view, offset, field) {
|
|
2656
|
+
const big = view.getBigUint64(offset, true);
|
|
2657
|
+
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
2658
|
+
throw crbmIntegrity(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
2864
2659
|
}
|
|
2660
|
+
return Number(big);
|
|
2865
2661
|
}
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
const want = expected[i];
|
|
2872
|
-
const have = actual[i];
|
|
2873
|
-
if (want !== have) return want.toLowerCase() === have.toLowerCase() ? i : -1;
|
|
2874
|
-
}
|
|
2875
|
-
return -1;
|
|
2876
|
-
}
|
|
2877
|
-
var swapCase = (s) => [...s].map((c) => c === c.toLowerCase() ? c.toUpperCase() : c.toLowerCase()).join("");
|
|
2878
|
-
var components = (path) => path.split(sep).filter((c) => c !== "");
|
|
2879
|
-
function caseVariantError() {
|
|
2880
|
-
return new ValidationError(
|
|
2881
|
-
"names differ only by case on a case-insensitive filesystem: a name that differs only by case from one already stored is the same file here, so it is refused rather than written over"
|
|
2882
|
-
);
|
|
2883
|
-
}
|
|
2884
|
-
var ExactCase = class {
|
|
2885
|
-
constructor(root) {
|
|
2886
|
-
this.root = root;
|
|
2887
|
-
}
|
|
2888
|
-
root;
|
|
2889
|
-
/** `undefined` until the root holds an entry to try; then fixed, as a filesystem does not change its mind. */
|
|
2890
|
-
insensitive;
|
|
2891
|
-
/**
|
|
2892
|
-
* Whether the root's filesystem folds case, learned from an entry already in the root: the entry's name with its
|
|
2893
|
-
* case swapped is the same file or it is not. An empty or absent root holds nothing to alias, so it is not
|
|
2894
|
-
* insensitive yet, and is looked at again on the next call.
|
|
2895
|
-
*/
|
|
2896
|
-
async foldsCase() {
|
|
2897
|
-
if (this.insensitive !== void 0) return this.insensitive;
|
|
2898
|
-
let names;
|
|
2899
|
-
try {
|
|
2900
|
-
names = await readdir(this.root);
|
|
2901
|
-
} catch (err) {
|
|
2902
|
-
if (isAbsent(err)) return this.foldsAtAncestor();
|
|
2903
|
-
throw err;
|
|
2904
|
-
}
|
|
2905
|
-
for (const name of names) {
|
|
2906
|
-
const swapped = swapCase(name);
|
|
2907
|
-
if (swapped === name) continue;
|
|
2908
|
-
const own = await stat(join3(this.root, name), { bigint: true }).catch(() => void 0);
|
|
2909
|
-
if (own === void 0) continue;
|
|
2910
|
-
const other = await stat(join3(this.root, swapped), { bigint: true }).catch(() => void 0);
|
|
2911
|
-
this.insensitive = other !== void 0 && other.ino === own.ino && other.dev === own.dev;
|
|
2912
|
-
return this.insensitive;
|
|
2913
|
-
}
|
|
2914
|
-
return this.foldsAtAncestor();
|
|
2915
|
-
}
|
|
2916
|
-
/**
|
|
2917
|
-
* The same question asked of the root's own name, or the nearest existing directory above it, when the root holds
|
|
2918
|
-
* nothing to try: a root with no entries has none to alias yet, but two names created at once still would. Not
|
|
2919
|
-
* remembered, since it is the volume above the root that answers.
|
|
2920
|
-
*/
|
|
2921
|
-
async foldsAtAncestor() {
|
|
2922
|
-
let dir = resolve(this.root);
|
|
2923
|
-
for (; ; ) {
|
|
2924
|
-
const top = parse(dir).root;
|
|
2925
|
-
const parts = components(relative(top, dir));
|
|
2926
|
-
let at = parts.length - 1;
|
|
2927
|
-
while (at >= 0 && swapCase(parts[at]) === parts[at]) at--;
|
|
2928
|
-
if (at === -1) return false;
|
|
2929
|
-
const swapped = [...parts];
|
|
2930
|
-
swapped[at] = swapCase(parts[at]);
|
|
2931
|
-
let own;
|
|
2932
|
-
try {
|
|
2933
|
-
own = await stat(dir, { bigint: true });
|
|
2934
|
-
} catch (err) {
|
|
2935
|
-
if (!isAbsent(err)) return false;
|
|
2936
|
-
const parent = dirname2(dir);
|
|
2937
|
-
if (parent === dir) return false;
|
|
2938
|
-
dir = parent;
|
|
2939
|
-
continue;
|
|
2940
|
-
}
|
|
2941
|
-
const other = await stat(join3(top, ...swapped), { bigint: true }).catch(() => void 0);
|
|
2942
|
-
return other !== void 0 && other.ino === own.ino && other.dev === own.dev;
|
|
2943
|
-
}
|
|
2944
|
-
}
|
|
2945
|
-
/** Whether the root's filesystem folds case: names that differ only by case are one file there. */
|
|
2946
|
-
folds() {
|
|
2947
|
-
return this.foldsCase();
|
|
2948
|
-
}
|
|
2949
|
-
/**
|
|
2950
|
-
* `true` when `path` (inside the root) is, on disk, a path whose name differs only by case from the one given:
|
|
2951
|
-
* the deepest part of it that exists is resolved to its real name and compared exactly. A path that does not exist
|
|
2952
|
-
* yet, or that nothing differs from, is `false`.
|
|
2953
|
-
*/
|
|
2954
|
-
async differs(path) {
|
|
2955
|
-
if (!await this.foldsCase()) return false;
|
|
2956
|
-
const rootAbs = resolve(this.root);
|
|
2957
|
-
let probe = resolve(path);
|
|
2958
|
-
let real;
|
|
2959
|
-
for (; ; ) {
|
|
2960
|
-
if (probe === rootAbs || !relative(rootAbs, probe) || relative(rootAbs, probe).startsWith("..")) {
|
|
2961
|
-
return false;
|
|
2962
|
-
}
|
|
2963
|
-
try {
|
|
2964
|
-
real = await realpath(probe);
|
|
2965
|
-
break;
|
|
2966
|
-
} catch (err) {
|
|
2967
|
-
if (!isAbsent(err)) throw err;
|
|
2968
|
-
probe = dirname2(probe);
|
|
2969
|
-
}
|
|
2970
|
-
}
|
|
2971
|
-
const realRoot = await realpath(rootAbs).catch(() => void 0);
|
|
2972
|
-
if (realRoot === void 0) return false;
|
|
2973
|
-
const actual = relative(realRoot, real);
|
|
2974
|
-
if (actual.startsWith("..")) return false;
|
|
2975
|
-
return firstCaseDifference(components(relative(rootAbs, probe)), components(actual)) !== -1;
|
|
2976
|
-
}
|
|
2977
|
-
/** Refuse a write to `path` when its real name differs from it only by case. */
|
|
2978
|
-
async refuseVariant(path) {
|
|
2979
|
-
if (await this.differs(path)) throw caseVariantError();
|
|
2980
|
-
}
|
|
2981
|
-
};
|
|
2982
|
-
|
|
2983
|
-
// src/drivers/localfs/storage.ts
|
|
2984
|
-
async function readWhole(handle, key, length, offset) {
|
|
2985
|
-
const buf = new Uint8Array(length);
|
|
2986
|
-
let filled = 0;
|
|
2987
|
-
while (filled < length) {
|
|
2988
|
-
const { bytesRead } = await handle.read(buf, filled, length - filled, offset + filled);
|
|
2989
|
-
if (bytesRead === 0) {
|
|
2990
|
-
throw new IntegrityError(
|
|
2991
|
-
`short read of ${key.segment}.${key.generation}: wanted ${length}B at ${offset}, got ${filled}B`
|
|
2992
|
-
);
|
|
2993
|
-
}
|
|
2994
|
-
filled += bytesRead;
|
|
2995
|
-
}
|
|
2996
|
-
return buf;
|
|
2997
|
-
}
|
|
2998
|
-
var LocalFsStorageDriver = class {
|
|
2999
|
-
/**
|
|
3000
|
-
* `options.privateRoot` is the directory this one lives under, from which down directories are created private; it
|
|
3001
|
-
* defaults to `root`.
|
|
3002
|
-
*/
|
|
3003
|
-
constructor(root, options = {}) {
|
|
3004
|
-
this.root = root;
|
|
3005
|
-
this.options = options;
|
|
3006
|
-
this.exactCase = new ExactCase(root);
|
|
3007
|
-
}
|
|
3008
|
-
root;
|
|
3009
|
-
options;
|
|
3010
|
-
exactCase;
|
|
3011
|
-
capabilities() {
|
|
3012
|
-
return {
|
|
3013
|
-
rangeRead: true,
|
|
3014
|
-
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
3015
|
-
conditionalPut: false,
|
|
3016
|
-
conditionalDelete: false
|
|
3017
|
-
};
|
|
3018
|
-
}
|
|
3019
|
-
async putImmutable(key, write) {
|
|
3020
|
-
const finalPath = storageObjectPath(this.root, key);
|
|
3021
|
-
assertStorageNamesFit(key);
|
|
3022
|
-
await this.exactCase.refuseVariant(finalPath);
|
|
3023
|
-
await makeDirs(dirname3(finalPath), this.options.privateRoot ?? this.root);
|
|
3024
|
-
void sweepOrphanTemps(dirname3(finalPath));
|
|
3025
|
-
const tmpPath = tempPathFor(finalPath);
|
|
3026
|
-
const handle = await open2(
|
|
3027
|
-
tmpPath,
|
|
3028
|
-
FS2.O_CREAT | FS2.O_EXCL | FS2.O_WRONLY | O_NOFOLLOW,
|
|
3029
|
-
FILE_MODE
|
|
3030
|
-
);
|
|
3031
|
-
const hash = createHash2("sha256");
|
|
3032
|
-
let size = 0;
|
|
3033
|
-
const sink = {
|
|
3034
|
-
async write(bytes) {
|
|
3035
|
-
await writeAll(handle, bytes);
|
|
3036
|
-
hash.update(bytes);
|
|
3037
|
-
size += bytes.length;
|
|
3038
|
-
}
|
|
3039
|
-
};
|
|
3040
|
-
try {
|
|
3041
|
-
await write(sink);
|
|
3042
|
-
await handle.sync();
|
|
3043
|
-
} catch (err) {
|
|
3044
|
-
await handle.close().catch(() => {
|
|
3045
|
-
});
|
|
3046
|
-
await unlink2(tmpPath).catch(() => {
|
|
3047
|
-
});
|
|
3048
|
-
throw mapFsError(err);
|
|
3049
|
-
}
|
|
3050
|
-
await handle.close();
|
|
3051
|
-
try {
|
|
3052
|
-
await link(tmpPath, finalPath);
|
|
3053
|
-
} catch (err) {
|
|
3054
|
-
await unlink2(tmpPath).catch(() => {
|
|
3055
|
-
});
|
|
3056
|
-
if (isCode(err, "ENOENT")) {
|
|
3057
|
-
throw new TransientError(
|
|
3058
|
-
`the temporary file of ${key.segment}.${key.generation} was removed mid-write`,
|
|
3059
|
-
{
|
|
3060
|
-
cause: err
|
|
3061
|
-
}
|
|
3062
|
-
);
|
|
3063
|
-
}
|
|
3064
|
-
if (isCode(err, "EEXIST")) {
|
|
3065
|
-
throw new WriteConflictError(
|
|
3066
|
-
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
3067
|
-
);
|
|
3068
|
-
}
|
|
3069
|
-
throw mapFsError(err);
|
|
3070
|
-
}
|
|
3071
|
-
await unlink2(tmpPath).catch(() => {
|
|
3072
|
-
});
|
|
3073
|
-
await fsyncDir(dirname3(finalPath));
|
|
3074
|
-
return { size, sha256: hash.digest("hex") };
|
|
3075
|
-
}
|
|
3076
|
-
async getRange(key, offset, length) {
|
|
3077
|
-
requireInteger(offset, { min: 0, what: "range offset" });
|
|
3078
|
-
requireInteger(length, { min: 0, what: "range length" });
|
|
3079
|
-
const handle = await this.openRead(key);
|
|
3080
|
-
try {
|
|
3081
|
-
const { size } = await handle.stat();
|
|
3082
|
-
if (offset + length > size) {
|
|
3083
|
-
throw new ValidationError(
|
|
3084
|
-
`range [${offset}, ${offset + length}) out of bounds for ${size}B`
|
|
3085
|
-
);
|
|
3086
|
-
}
|
|
3087
|
-
return await readWhole(handle, key, length, offset);
|
|
3088
|
-
} finally {
|
|
3089
|
-
await handle.close();
|
|
3090
|
-
}
|
|
3091
|
-
}
|
|
3092
|
-
async getTail(key, maxBytes) {
|
|
3093
|
-
checkTailLength(maxBytes);
|
|
3094
|
-
const handle = await this.openRead(key);
|
|
3095
|
-
try {
|
|
3096
|
-
const { size } = await handle.stat();
|
|
3097
|
-
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
3098
|
-
return { bytes: await readWhole(handle, key, take, size - take), size };
|
|
3099
|
-
} finally {
|
|
3100
|
-
await handle.close();
|
|
3101
|
-
}
|
|
3102
|
-
}
|
|
3103
|
-
/**
|
|
3104
|
-
* Idempotent. Ignores `options.ifVersion`, as `conditionalDelete: false` says: see the note on the class for why it
|
|
3105
|
-
* cannot apply one. The parameter is declared so that a wrapper of this driver can hand the port's options on.
|
|
3106
|
-
*/
|
|
3107
|
-
async delete(key, options) {
|
|
3108
|
-
void options;
|
|
3109
|
-
const path = storageObjectPath(this.root, key);
|
|
3110
|
-
if (await this.exactCase.differs(path)) return;
|
|
3111
|
-
await unlink2(path).catch((err) => {
|
|
3112
|
-
if (!isAbsent(err)) throw mapFsError(err);
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
async *list(ref) {
|
|
3116
|
-
const dir = segmentsDir(this.root, ref);
|
|
3117
|
-
if (await this.exactCase.differs(dir)) return;
|
|
3118
|
-
let names;
|
|
3119
|
-
try {
|
|
3120
|
-
names = await readdir2(dir);
|
|
3121
|
-
} catch (err) {
|
|
3122
|
-
if (isAbsent(err)) return;
|
|
3123
|
-
throw mapFsError(err);
|
|
3124
|
-
}
|
|
3125
|
-
for (const name of names) {
|
|
3126
|
-
const generation = parseGeneration(ref.segment, name);
|
|
3127
|
-
if (generation !== null) {
|
|
3128
|
-
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3129
|
-
}
|
|
3130
|
-
}
|
|
3131
|
-
}
|
|
3132
|
-
async openRead(key) {
|
|
3133
|
-
const path = storageObjectPath(this.root, key);
|
|
3134
|
-
if (await this.exactCase.differs(path)) {
|
|
3135
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3136
|
-
}
|
|
3137
|
-
try {
|
|
3138
|
-
return await open2(path, FS2.O_RDONLY | O_NOFOLLOW);
|
|
3139
|
-
} catch (err) {
|
|
3140
|
-
if (isAbsent(err)) {
|
|
3141
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3142
|
-
}
|
|
3143
|
-
if (isCode(err, "ELOOP")) {
|
|
3144
|
-
throw new NotFoundError(
|
|
3145
|
-
`generation path is a symlink, refusing: ${key.segment}.${key.generation}`
|
|
3146
|
-
);
|
|
3147
|
-
}
|
|
3148
|
-
throw mapFsError(err);
|
|
3149
|
-
}
|
|
3150
|
-
}
|
|
3151
|
-
};
|
|
3152
|
-
|
|
3153
|
-
// src/drivers/localfs/registry.ts
|
|
3154
|
-
import { constants as FS3 } from "node:fs";
|
|
3155
|
-
import { open as open3, opendir as opendir2, readdir as readdir3, realpath as realpath2, rename, unlink as unlink3 } from "node:fs/promises";
|
|
3156
|
-
import { basename, dirname as dirname4, join as join4, relative as relative2, resolve as resolve2 } from "node:path";
|
|
3157
|
-
var DEFAULT_MAX_ROW_BYTES = 1 * 1024 * 1024;
|
|
3158
|
-
var rowChains = /* @__PURE__ */ new Map();
|
|
3159
|
-
async function rowLockKey(path) {
|
|
3160
|
-
const abs = resolve2(path);
|
|
3161
|
-
const tail = [basename(abs)];
|
|
3162
|
-
let dir = dirname4(abs);
|
|
3163
|
-
for (; ; ) {
|
|
3164
|
-
try {
|
|
3165
|
-
return join4(await realpath2(dir), ...tail);
|
|
3166
|
-
} catch (err) {
|
|
3167
|
-
const parent = dirname4(dir);
|
|
3168
|
-
if (!isAbsent(err) || parent === dir) throw mapFsError(err);
|
|
3169
|
-
tail.unshift(basename(dir));
|
|
3170
|
-
dir = parent;
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
}
|
|
3174
|
-
var LocalFsRegistryDriver = class {
|
|
3175
|
-
constructor(root, options = {}) {
|
|
3176
|
-
this.root = root;
|
|
3177
|
-
this.options = options;
|
|
3178
|
-
this.exactCase = new ExactCase(root);
|
|
3179
|
-
this.now = options.now ?? (() => Date.now());
|
|
3180
|
-
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
3181
|
-
}
|
|
3182
|
-
root;
|
|
3183
|
-
options;
|
|
3184
|
-
now;
|
|
3185
|
-
entropy;
|
|
3186
|
-
exactCase;
|
|
3187
|
-
/**
|
|
3188
|
-
* `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
|
|
3189
|
-
* the root in this process takes, so no write can land between the check and the unlink.
|
|
3190
|
-
*/
|
|
3191
|
-
capabilities() {
|
|
3192
|
-
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
3193
|
-
}
|
|
3194
|
-
/** Every write reports its token to `onToken` before the row's file is written. */
|
|
3195
|
-
get reportsWriteTokens() {
|
|
3196
|
-
return true;
|
|
3197
|
-
}
|
|
3198
|
-
async get(ref) {
|
|
3199
|
-
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
3200
|
-
return env && !env.deleted ? env.record : null;
|
|
3201
|
-
}
|
|
3202
|
-
async create(ref, record, options) {
|
|
3203
|
-
const checked = validateNewRegistryRecord(record);
|
|
3204
|
-
const path = registryRowPath(this.root, ref);
|
|
3205
|
-
assertRegistryNamesFit(ref);
|
|
3206
|
-
return this.withRowLock(path, async () => {
|
|
3207
|
-
const current = await this.readRow(path, ref);
|
|
3208
|
-
if (current !== null && !current.deleted) {
|
|
3209
|
-
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
3210
|
-
}
|
|
3211
|
-
const token = newIncarnationToken(this.entropy, current?.record);
|
|
3212
|
-
options?.onToken?.(token);
|
|
3213
|
-
await this.writeRow(path, recordFromNew(ref, checked, this.now(), token));
|
|
3214
|
-
return { token };
|
|
3215
|
-
});
|
|
3216
|
-
}
|
|
3217
|
-
/**
|
|
3218
|
-
* A caller's `held` row (see `RegistryWriteOptions`) has no read to spare here: the row's file is read under its lock and its token compared
|
|
3219
|
-
* there, a local read and no request, so a stale one fails exactly as a lost race does.
|
|
3220
|
-
*/
|
|
3221
|
-
async compareAndSwap(ref, expected, patch, options) {
|
|
3222
|
-
const checked = validateRegistryPatch(patch);
|
|
3223
|
-
const path = registryRowPath(this.root, ref);
|
|
3224
|
-
assertRegistryNamesFit(ref);
|
|
3225
|
-
return this.withRowLock(path, async () => {
|
|
3226
|
-
const current = await this.readRow(path, ref);
|
|
3227
|
-
if (current === null || current.deleted || current.record.token !== expected) {
|
|
3228
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3229
|
-
}
|
|
3230
|
-
const token = nextRegistryToken(current.record, this.entropy);
|
|
3231
|
-
options?.onToken?.(token);
|
|
3232
|
-
await this.writeRow(path, applyRegistryPatch(current.record, checked, this.now(), token));
|
|
3233
|
-
return { token };
|
|
3234
|
-
});
|
|
3235
|
-
}
|
|
3236
|
-
async *list(namespace) {
|
|
3237
|
-
for (const ns of await this.namespaceDirs(namespace)) {
|
|
3238
|
-
const dir = registryDir(this.root, ns);
|
|
3239
|
-
if (await this.exactCase.differs(dir)) continue;
|
|
3240
|
-
let entries;
|
|
3241
|
-
try {
|
|
3242
|
-
entries = await opendir2(dir);
|
|
3243
|
-
} catch (err) {
|
|
3244
|
-
if (isAbsent(err)) continue;
|
|
3245
|
-
throw mapFsError(err);
|
|
3246
|
-
}
|
|
3247
|
-
for await (const entry of entries) {
|
|
3248
|
-
const segment = parseRegistryRow(entry.name);
|
|
3249
|
-
if (segment === null) continue;
|
|
3250
|
-
const ref = { namespace: ns, segment };
|
|
3251
|
-
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
3252
|
-
if (env && !env.deleted) yield env.record;
|
|
3253
|
-
}
|
|
3254
|
-
}
|
|
3255
|
-
}
|
|
3256
|
-
async delete(ref, expected) {
|
|
3257
|
-
const path = registryRowPath(this.root, ref);
|
|
3258
|
-
return this.withRowLock(path, async () => {
|
|
3259
|
-
const current = await this.readRow(path, ref);
|
|
3260
|
-
if (expected !== void 0) {
|
|
3261
|
-
if (current === null || current.deleted || current.record.token !== expected) {
|
|
3262
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3263
|
-
}
|
|
3264
|
-
} else if (current === null || current.deleted) {
|
|
3265
|
-
return;
|
|
3266
|
-
}
|
|
3267
|
-
await this.unlinkRow(path);
|
|
3268
|
-
});
|
|
3269
|
-
}
|
|
3270
|
-
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
3271
|
-
async unlinkRow(path) {
|
|
3272
|
-
try {
|
|
3273
|
-
await unlink3(path);
|
|
3274
|
-
} catch (err) {
|
|
3275
|
-
if (!isCode(err, "ENOENT")) throw mapFsError(err);
|
|
3276
|
-
}
|
|
3277
|
-
await fsyncDir(dirname4(path));
|
|
3278
|
-
}
|
|
3279
|
-
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
3280
|
-
async namespaceDirs(namespace) {
|
|
3281
|
-
if (namespace !== void 0) return [namespace];
|
|
3282
|
-
let entries;
|
|
3283
|
-
try {
|
|
3284
|
-
entries = await readdir3(this.root);
|
|
3285
|
-
} catch (err) {
|
|
3286
|
-
if (isCode(err, "ENOENT")) return [];
|
|
3287
|
-
throw mapFsError(err);
|
|
3288
|
-
}
|
|
3289
|
-
const namespaces = [];
|
|
3290
|
-
for (const entry of entries) {
|
|
3291
|
-
const parsed = parseNamespaceDir(entry);
|
|
3292
|
-
if (parsed !== null) namespaces.push(parsed.namespace);
|
|
3293
|
-
}
|
|
3294
|
-
return namespaces;
|
|
3295
|
-
}
|
|
3296
|
-
/** Read the row at `path`, which must be the row of `ref`: one that names another segment is refused. */
|
|
3297
|
-
async readRow(path, ref) {
|
|
3298
|
-
let handle;
|
|
3299
|
-
try {
|
|
3300
|
-
if (await this.exactCase.differs(path)) return null;
|
|
3301
|
-
handle = await open3(path, FS3.O_RDONLY | O_NOFOLLOW);
|
|
3302
|
-
} catch (err) {
|
|
3303
|
-
if (isAbsent(err) || isCode(err, "ELOOP")) return null;
|
|
3304
|
-
throw mapFsError(err);
|
|
3305
|
-
}
|
|
3306
|
-
try {
|
|
3307
|
-
const { size } = await handle.stat();
|
|
3308
|
-
if (size > DEFAULT_MAX_ROW_BYTES) {
|
|
3309
|
-
throw new IntegrityError(`registry row ${size}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`);
|
|
3310
|
-
}
|
|
3311
|
-
const text = (await handle.readFile()).toString("utf8");
|
|
3312
|
-
return parseRegistryEnvelope(text, relative2(this.root, path), ref);
|
|
3313
|
-
} finally {
|
|
3314
|
-
await handle.close();
|
|
3315
|
-
}
|
|
3316
|
-
}
|
|
3317
|
-
async writeRow(path, record) {
|
|
3318
|
-
await this.exactCase.refuseVariant(path);
|
|
3319
|
-
await makeDirs(dirname4(path), this.options.privateRoot ?? this.root);
|
|
3320
|
-
void sweepOrphanTemps(dirname4(path));
|
|
3321
|
-
const out = Buffer.from(serializeRegistryEnvelope({ deleted: false, record }), "utf8");
|
|
3322
|
-
if (out.length > DEFAULT_MAX_ROW_BYTES) {
|
|
3323
|
-
throw new ValidationError(
|
|
3324
|
-
`registry row ${out.length}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`
|
|
3325
|
-
);
|
|
3326
|
-
}
|
|
3327
|
-
const tmp = tempPathFor(path);
|
|
3328
|
-
let handle;
|
|
3329
|
-
try {
|
|
3330
|
-
handle = await open3(tmp, FS3.O_CREAT | FS3.O_EXCL | FS3.O_WRONLY | O_NOFOLLOW, FILE_MODE);
|
|
3331
|
-
} catch (err) {
|
|
3332
|
-
throw mapFsError(err);
|
|
3333
|
-
}
|
|
3334
|
-
try {
|
|
3335
|
-
await writeAll(handle, out);
|
|
3336
|
-
await handle.sync();
|
|
3337
|
-
} catch (err) {
|
|
3338
|
-
await handle.close().catch(() => {
|
|
3339
|
-
});
|
|
3340
|
-
await unlink3(tmp).catch(() => {
|
|
3341
|
-
});
|
|
3342
|
-
throw mapFsError(err);
|
|
3343
|
-
}
|
|
3344
|
-
await handle.close();
|
|
3345
|
-
await rename(tmp, path).catch(async (err) => {
|
|
3346
|
-
await unlink3(tmp).catch(() => {
|
|
3347
|
-
});
|
|
3348
|
-
if (isCode(err, "ENOENT")) {
|
|
3349
|
-
throw new TransientError("the temporary file of a registry row was removed mid-write", {
|
|
3350
|
-
cause: err
|
|
3351
|
-
});
|
|
3352
|
-
}
|
|
3353
|
-
throw mapFsError(err);
|
|
3354
|
-
});
|
|
3355
|
-
await fsyncDir(dirname4(path));
|
|
3356
|
-
}
|
|
3357
|
-
/** Serialize callbacks for a row so read-modify-write is atomic across every instance in the process. */
|
|
3358
|
-
async withRowLock(path, fn) {
|
|
3359
|
-
const resolved = await rowLockKey(path);
|
|
3360
|
-
const key = await this.exactCase.folds() ? resolved.toLowerCase() : resolved;
|
|
3361
|
-
const prev = rowChains.get(key) ?? Promise.resolve();
|
|
3362
|
-
const result = prev.then(fn, fn);
|
|
3363
|
-
const tail = result.then(
|
|
3364
|
-
() => void 0,
|
|
3365
|
-
() => void 0
|
|
3366
|
-
);
|
|
3367
|
-
rowChains.set(key, tail);
|
|
3368
|
-
void tail.then(() => {
|
|
3369
|
-
if (rowChains.get(key) === tail) rowChains.delete(key);
|
|
3370
|
-
});
|
|
3371
|
-
return result;
|
|
3372
|
-
}
|
|
3373
|
-
};
|
|
3374
|
-
|
|
3375
|
-
// src/drivers/backends.ts
|
|
3376
|
-
var CLOCK_OPTION_KEYS = ["now"];
|
|
3377
|
-
var MemoryStorage = class {
|
|
3378
|
-
storage;
|
|
3379
|
-
registry;
|
|
3380
|
-
constructor(options = {}) {
|
|
3381
|
-
refuseUnknownOptions("MemoryStorage", options, CLOCK_OPTION_KEYS);
|
|
3382
|
-
checkNow("MemoryStorage", options.now);
|
|
3383
|
-
this.storage = new MemoryStorageDriver();
|
|
3384
|
-
this.registry = new MemoryRegistryDriver(options.now === void 0 ? {} : { now: options.now });
|
|
3385
|
-
brandAsBackend(this);
|
|
3386
|
-
}
|
|
3387
|
-
};
|
|
3388
|
-
var LocalFsStorage = class {
|
|
3389
|
-
constructor(root, options = {}) {
|
|
3390
|
-
this.root = root;
|
|
3391
|
-
if (typeof root !== "string" || root.length === 0) {
|
|
3392
|
-
throw new ValidationError(
|
|
3393
|
-
`LocalFsStorage needs a root directory: a non-empty string, got ${root === "" ? "an empty string" : typeof root}`
|
|
3394
|
-
);
|
|
3395
|
-
}
|
|
3396
|
-
refuseUnknownOptions("LocalFsStorage", options, CLOCK_OPTION_KEYS);
|
|
3397
|
-
checkNow("LocalFsStorage", options.now);
|
|
3398
|
-
this.storage = new LocalFsStorageDriver(join5(root, "storage"), { privateRoot: root });
|
|
3399
|
-
this.registry = new LocalFsRegistryDriver(
|
|
3400
|
-
join5(root, "registry"),
|
|
3401
|
-
options.now === void 0 ? { privateRoot: root } : { now: options.now, privateRoot: root }
|
|
3402
|
-
);
|
|
3403
|
-
brandAsBackend(this);
|
|
3404
|
-
}
|
|
3405
|
-
root;
|
|
3406
|
-
storage;
|
|
3407
|
-
registry;
|
|
3408
|
-
};
|
|
3409
|
-
|
|
3410
|
-
// src/core/reader-defaults.ts
|
|
3411
|
-
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
3412
|
-
var REFRESH_RETRY_MS = 500;
|
|
3413
|
-
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
3414
|
-
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
3415
|
-
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
3416
|
-
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
3417
|
-
|
|
3418
|
-
// src/core/crbm/integrity.ts
|
|
3419
|
-
var crbmIntegrity = (message) => new IntegrityError(`${message}; re-load from source or roll back`);
|
|
3420
|
-
|
|
3421
|
-
// src/core/crbm/crc32c.ts
|
|
3422
|
-
var TABLES = (() => {
|
|
3423
|
-
const POLY = 2197175160;
|
|
3424
|
-
const t0 = new Uint32Array(256);
|
|
3425
|
-
for (let n = 0; n < 256; n++) {
|
|
3426
|
-
let c = n;
|
|
3427
|
-
for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
|
|
3428
|
-
t0[n] = c >>> 0;
|
|
3429
|
-
}
|
|
3430
|
-
const tables = [t0];
|
|
3431
|
-
for (let k = 1; k < 8; k++) {
|
|
3432
|
-
const prev = tables[k - 1];
|
|
3433
|
-
const next = new Uint32Array(256);
|
|
3434
|
-
for (let n = 0; n < 256; n++) {
|
|
3435
|
-
const v = prev[n];
|
|
3436
|
-
next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
|
|
3437
|
-
}
|
|
3438
|
-
tables.push(next);
|
|
3439
|
-
}
|
|
3440
|
-
return tables;
|
|
3441
|
-
})();
|
|
3442
|
-
var T0 = TABLES[0];
|
|
3443
|
-
var T1 = TABLES[1];
|
|
3444
|
-
var T2 = TABLES[2];
|
|
3445
|
-
var T3 = TABLES[3];
|
|
3446
|
-
var T4 = TABLES[4];
|
|
3447
|
-
var T5 = TABLES[5];
|
|
3448
|
-
var T6 = TABLES[6];
|
|
3449
|
-
var T7 = TABLES[7];
|
|
3450
|
-
function crc32c(bytes, seed = 0) {
|
|
3451
|
-
let crc = (seed ^ 4294967295) >>> 0;
|
|
3452
|
-
let i = 0;
|
|
3453
|
-
const n8 = bytes.length - bytes.length % 8;
|
|
3454
|
-
for (; i < n8; i += 8) {
|
|
3455
|
-
crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
|
|
3456
|
-
crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
|
|
3457
|
-
}
|
|
3458
|
-
for (; i < bytes.length; i++) {
|
|
3459
|
-
crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
|
|
3460
|
-
}
|
|
3461
|
-
return (crc ^ 4294967295) >>> 0;
|
|
3462
|
-
}
|
|
3463
|
-
|
|
3464
|
-
// src/core/crbm/plan-reads.ts
|
|
3465
|
-
var MAX_COALESCE_GAP_BYTES = 256 * 1024;
|
|
3466
|
-
var MAX_COALESCED_READ_BYTES = DEFAULT_MAX_BITMAP_BYTES;
|
|
3467
|
-
var MAX_RANGES_IN_FLIGHT = 32;
|
|
3468
|
-
function* planChunkReads(extents, region) {
|
|
3469
|
-
let chunks = [];
|
|
3470
|
-
let start = 0;
|
|
3471
|
-
let end = 0;
|
|
3472
|
-
let previousEnd = region.start;
|
|
3473
|
-
for (const extent of extents) {
|
|
3474
|
-
const extentEnd = extent.offset + extent.length;
|
|
3475
|
-
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
3476
|
-
throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
|
|
3477
|
-
}
|
|
3478
|
-
previousEnd = extentEnd;
|
|
3479
|
-
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
3480
|
-
chunks.push(extent);
|
|
3481
|
-
end = extentEnd;
|
|
3482
|
-
continue;
|
|
3483
|
-
}
|
|
3484
|
-
if (chunks.length > 0) {
|
|
3485
|
-
const done = chunks;
|
|
3486
|
-
chunks = [];
|
|
3487
|
-
yield { offset: start, length: end - start, chunks: done };
|
|
3488
|
-
}
|
|
3489
|
-
chunks.push(extent);
|
|
3490
|
-
start = extent.offset;
|
|
3491
|
-
end = extentEnd;
|
|
3492
|
-
}
|
|
3493
|
-
if (chunks.length > 0) yield { offset: start, length: end - start, chunks };
|
|
3494
|
-
}
|
|
3495
|
-
|
|
3496
|
-
// src/core/crbm/varint.ts
|
|
3497
|
-
var U32_CEIL = U32_MAX + 1;
|
|
3498
|
-
function writeVarint(out, value) {
|
|
3499
|
-
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
3500
|
-
throw new RangeError(`varint value must be a u32; got ${value}`);
|
|
3501
|
-
}
|
|
3502
|
-
let v = value;
|
|
3503
|
-
while (v > 127) {
|
|
3504
|
-
out.push(v & 127 | 128);
|
|
3505
|
-
v >>>= 7;
|
|
3506
|
-
}
|
|
3507
|
-
out.push(v);
|
|
3508
|
-
}
|
|
3509
|
-
function readVarint(bytes, offset) {
|
|
3510
|
-
let result = 0;
|
|
3511
|
-
let shift = 0;
|
|
3512
|
-
let pos = offset;
|
|
3513
|
-
for (let i = 0; i < 5; i++) {
|
|
3514
|
-
if (pos >= bytes.length) {
|
|
3515
|
-
throw crbmIntegrity("varint truncated past end of buffer");
|
|
3516
|
-
}
|
|
3517
|
-
const byte = bytes[pos];
|
|
3518
|
-
pos += 1;
|
|
3519
|
-
if (i === 4 && byte > 15) {
|
|
3520
|
-
throw crbmIntegrity("varint exceeds u32 range");
|
|
3521
|
-
}
|
|
3522
|
-
result = (result | (byte & 127) << shift) >>> 0;
|
|
3523
|
-
if ((byte & 128) === 0) {
|
|
3524
|
-
return { value: result, next: pos };
|
|
3525
|
-
}
|
|
3526
|
-
shift += 7;
|
|
3527
|
-
}
|
|
3528
|
-
throw crbmIntegrity("varint exceeds u32 range");
|
|
3529
|
-
}
|
|
3530
|
-
|
|
3531
|
-
// src/core/crbm/reader.ts
|
|
3532
|
-
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
3533
|
-
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
3534
|
-
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
3535
|
-
function indexCapacity(indexLength) {
|
|
3536
|
-
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
3537
|
-
}
|
|
3538
|
-
function magicMatches(bytes, offset) {
|
|
3539
|
-
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
3540
|
-
}
|
|
3541
|
-
function checkSize(tail, size) {
|
|
3542
|
-
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3543
|
-
throw crbmIntegrity(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
3544
|
-
}
|
|
3545
|
-
}
|
|
3546
|
-
function checkedFooter(tail, size) {
|
|
3547
|
-
checkSize(tail, size);
|
|
3548
|
-
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
3549
|
-
throw crbmIntegrity(`.crbm too small: ${size}B`);
|
|
3550
|
-
}
|
|
3551
|
-
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
3552
|
-
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
3553
|
-
throw crbmIntegrity(".crbm end magic mismatch");
|
|
3554
|
-
}
|
|
3555
|
-
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
3556
|
-
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
3557
|
-
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
3558
|
-
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
3559
|
-
}
|
|
3560
|
-
return { footer, fview, storedFooterCrc };
|
|
3561
|
-
}
|
|
3562
|
-
async function readFooter(blob) {
|
|
3563
|
-
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
3564
|
-
const { fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
3565
|
-
return {
|
|
3566
|
-
encrypted: (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0,
|
|
3567
|
-
footerCrc: storedFooterCrc
|
|
3568
|
-
};
|
|
3569
|
-
}
|
|
3570
|
-
async function footerSaysEncrypted(blob) {
|
|
3571
|
-
return (await readFooter(blob)).encrypted;
|
|
3572
|
-
}
|
|
3573
|
-
function readU64(view, offset, field) {
|
|
3574
|
-
const big = view.getBigUint64(offset, true);
|
|
3575
|
-
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
3576
|
-
throw crbmIntegrity(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
3577
|
-
}
|
|
3578
|
-
return Number(big);
|
|
3579
|
-
}
|
|
3580
|
-
var keepLimits = /* @__PURE__ */ new WeakMap();
|
|
3581
|
-
function openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo) {
|
|
3582
|
-
const asked = { ...options };
|
|
3583
|
-
if (keepChunkBytesUpTo !== void 0) keepLimits.set(asked, keepChunkBytesUpTo);
|
|
3584
|
-
return CrbmReader.open(blob, asked);
|
|
2662
|
+
var keepLimits = /* @__PURE__ */ new WeakMap();
|
|
2663
|
+
function openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo) {
|
|
2664
|
+
const asked = { ...options };
|
|
2665
|
+
if (keepChunkBytesUpTo !== void 0) keepLimits.set(asked, keepChunkBytesUpTo);
|
|
2666
|
+
return CrbmReader.open(blob, asked);
|
|
3585
2667
|
}
|
|
3586
2668
|
function openTailBytes(options) {
|
|
3587
2669
|
return Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
@@ -3924,7 +3006,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3924
3006
|
}
|
|
3925
3007
|
/**
|
|
3926
3008
|
* Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
|
|
3927
|
-
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the
|
|
3009
|
+
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the codec's
|
|
3928
3010
|
* deserializer). The returned bytes may be shared (a small generation's chunks are kept by the reader and handed out
|
|
3929
3011
|
* again), so callers must not change them.
|
|
3930
3012
|
*/
|
|
@@ -4028,13 +3110,13 @@ var CrbmReader = class _CrbmReader {
|
|
|
4028
3110
|
* error.
|
|
4029
3111
|
*/
|
|
4030
3112
|
readChunks(chunkKeys, options = {}) {
|
|
4031
|
-
const
|
|
4032
|
-
const stream = this.streamChunks(chunkKeys, options,
|
|
3113
|
+
const open = { window: void 0 };
|
|
3114
|
+
const stream = this.streamChunks(chunkKeys, options, open);
|
|
4033
3115
|
return Object.assign(stream, {
|
|
4034
|
-
settled: () =>
|
|
3116
|
+
settled: () => open.window?.settle() ?? Promise.resolve()
|
|
4035
3117
|
});
|
|
4036
3118
|
}
|
|
4037
|
-
async *streamChunks(chunkKeys, options,
|
|
3119
|
+
async *streamChunks(chunkKeys, options, open) {
|
|
4038
3120
|
const width = options.concurrency ?? MAX_RANGES_IN_FLIGHT;
|
|
4039
3121
|
requirePositiveInteger(width, "concurrency");
|
|
4040
3122
|
validateChunkKeyOrder(chunkKeys);
|
|
@@ -4082,7 +3164,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
4082
3164
|
ramp !== false,
|
|
4083
3165
|
typeof ramp === "number" ? ramp : 1
|
|
4084
3166
|
);
|
|
4085
|
-
|
|
3167
|
+
open.window = window;
|
|
4086
3168
|
const cur = { range: void 0, at: 0 };
|
|
4087
3169
|
let last;
|
|
4088
3170
|
try {
|
|
@@ -4307,8 +3389,9 @@ async function provenObject(storage, key, fingerprint) {
|
|
|
4307
3389
|
version ??= tail.version;
|
|
4308
3390
|
});
|
|
4309
3391
|
return await CrbmReader.sameObject(blob, fingerprint) ? { version } : null;
|
|
4310
|
-
} catch {
|
|
4311
|
-
return null;
|
|
3392
|
+
} catch (err) {
|
|
3393
|
+
if (isNotFoundError(err)) return null;
|
|
3394
|
+
throw err;
|
|
4312
3395
|
}
|
|
4313
3396
|
}
|
|
4314
3397
|
function sameObjectUnder(storage, key, fingerprint) {
|
|
@@ -4359,12 +3442,7 @@ async function footerCrcUnder(storage, key) {
|
|
|
4359
3442
|
return (await readFooter(storageBlobReader(storage, key))).footerCrc;
|
|
4360
3443
|
}
|
|
4361
3444
|
async function provesOwnObject(storage, key, fingerprint) {
|
|
4362
|
-
|
|
4363
|
-
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
4364
|
-
} catch (err) {
|
|
4365
|
-
if (isNotFoundError(err)) return false;
|
|
4366
|
-
throw err;
|
|
4367
|
-
}
|
|
3445
|
+
return await provenObject(storage, key, fingerprint) !== null;
|
|
4368
3446
|
}
|
|
4369
3447
|
var openedVersions = /* @__PURE__ */ new WeakMap();
|
|
4370
3448
|
function openGenerationReader(storage, key, crypto, options = {}, named) {
|
|
@@ -4587,16 +3665,16 @@ function viaRetry(retry, request) {
|
|
|
4587
3665
|
return retry === void 0 ? request() : retry(request);
|
|
4588
3666
|
}
|
|
4589
3667
|
function validateRead(ref, keys) {
|
|
4590
|
-
validateUserRef(ref);
|
|
3668
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
4591
3669
|
if (keys.length > 0) validateSegmentRef({ ...ref });
|
|
4592
3670
|
for (const chunkKey of keys) validateChunkKey(chunkKey);
|
|
4593
3671
|
validateChunkKeyOrder(keys);
|
|
4594
3672
|
}
|
|
4595
3673
|
var Snapshot = class _Snapshot {
|
|
4596
|
-
constructor(resolution, target,
|
|
3674
|
+
constructor(resolution, target, open, prior) {
|
|
4597
3675
|
this.resolution = resolution;
|
|
4598
3676
|
this.target = target;
|
|
4599
|
-
this.open =
|
|
3677
|
+
this.open = open;
|
|
4600
3678
|
this.prior = prior;
|
|
4601
3679
|
}
|
|
4602
3680
|
resolution;
|
|
@@ -4607,8 +3685,8 @@ var Snapshot = class _Snapshot {
|
|
|
4607
3685
|
onReader;
|
|
4608
3686
|
opened;
|
|
4609
3687
|
outcome;
|
|
4610
|
-
static live(resolution, target,
|
|
4611
|
-
return new _Snapshot(resolution, target,
|
|
3688
|
+
static live(resolution, target, open, prior) {
|
|
3689
|
+
return new _Snapshot(resolution, target, open, prior);
|
|
4612
3690
|
}
|
|
4613
3691
|
static eager(reader) {
|
|
4614
3692
|
const snap = new _Snapshot(void 0, void 0, void 0, void 0);
|
|
@@ -4698,6 +3776,11 @@ var CrbmStorageChunkSource = class {
|
|
|
4698
3776
|
maxOpenIndexBytes,
|
|
4699
3777
|
...readerOptions
|
|
4700
3778
|
} = options;
|
|
3779
|
+
checkRequireEncryption(requireEncryption);
|
|
3780
|
+
if (maxOpenSegments !== void 0) requirePositiveInteger(maxOpenSegments, "maxOpenSegments");
|
|
3781
|
+
if (maxOpenIndexBytes !== void 0) {
|
|
3782
|
+
requirePositiveInteger(maxOpenIndexBytes, "maxOpenIndexBytes");
|
|
3783
|
+
}
|
|
4701
3784
|
if (keystore !== void 0 && registry === void 0) {
|
|
4702
3785
|
throw new CapabilityError(
|
|
4703
3786
|
"a keystore needs a storage backend \u2014 that is where the wrapped DEKs live. A bare IStorageDriver has no registry to hold them"
|
|
@@ -4710,7 +3793,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4710
3793
|
this.keystore = keystore;
|
|
4711
3794
|
this.requireEncryption = requireEncryption ?? false;
|
|
4712
3795
|
this.readerOptions = readerOptions;
|
|
4713
|
-
if (currentGenTtlMs !== void 0
|
|
3796
|
+
if (currentGenTtlMs !== void 0) {
|
|
4714
3797
|
requireFiniteNumber(currentGenTtlMs, {
|
|
4715
3798
|
min: 0,
|
|
4716
3799
|
what: "cache.genTtlMs",
|
|
@@ -5040,6 +4123,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5040
4123
|
* storage side can close the window; with no registry, no clock or `cache.genTtlMs: 0`, nothing bounds it at all.
|
|
5041
4124
|
*/
|
|
5042
4125
|
invalidate(ref) {
|
|
4126
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5043
4127
|
const key = segmentKey(ref);
|
|
5044
4128
|
this.invalidated.invalidate(key);
|
|
5045
4129
|
this.snapshots.delete(key);
|
|
@@ -5109,7 +4193,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5109
4193
|
* once the swept one is gone is exactly what a pin taken now should hold. One retry, then propagate.
|
|
5110
4194
|
*/
|
|
5111
4195
|
async pinGeneration(ref, lease) {
|
|
5112
|
-
validateUserRef(ref);
|
|
4196
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5113
4197
|
for (let moved = 0; ; moved++) {
|
|
5114
4198
|
try {
|
|
5115
4199
|
try {
|
|
@@ -5141,7 +4225,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5141
4225
|
* of the row's summary against it, since that check is made where the row's own generation is read.
|
|
5142
4226
|
*/
|
|
5143
4227
|
async pinGenerationAt(ref, at, lease) {
|
|
5144
|
-
validateUserRef(ref);
|
|
4228
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5145
4229
|
if (typeof at?.fingerprint !== "string")
|
|
5146
4230
|
throw new ValidationError(
|
|
5147
4231
|
`a pin of segment "${ref.segment}" needs the fingerprint of the object it names`
|
|
@@ -5291,7 +4375,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5291
4375
|
* the one read path with no resilience.
|
|
5292
4376
|
*
|
|
5293
4377
|
* **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
|
|
5294
|
-
* crypto-shred must not keep unwrapping a DEK the shred
|
|
4378
|
+
* crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
|
|
4379
|
+
* generation
|
|
5295
4380
|
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
5296
4381
|
* the key it unwrapped until it is evicted or invalidated.
|
|
5297
4382
|
*
|
|
@@ -5358,7 +4443,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5358
4443
|
* object, and the read fails if the generation is now another object.
|
|
5359
4444
|
*/
|
|
5360
4445
|
async getChunkAt(ref, generation, held) {
|
|
5361
|
-
validateUserRef(ref);
|
|
4446
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5362
4447
|
const reader = await this.readerAt(ref, generation, held);
|
|
5363
4448
|
if (reader === null) return null;
|
|
5364
4449
|
if (held !== void 0 && this.replacedPins.get(this.heldKey(ref, held.version, held.fingerprint)) !== void 0)
|
|
@@ -5416,7 +4501,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5416
4501
|
}
|
|
5417
4502
|
/** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
|
|
5418
4503
|
async listChunkKeysAt(ref, generation, held) {
|
|
5419
|
-
validateUserRef(ref);
|
|
4504
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5420
4505
|
const reader = await this.readerAt(ref, generation, held);
|
|
5421
4506
|
return reader === null ? [] : reader.chunkKeys();
|
|
5422
4507
|
}
|
|
@@ -5425,27 +4510,28 @@ var CrbmStorageChunkSource = class {
|
|
|
5425
4510
|
* for {@link getChunkAt}.
|
|
5426
4511
|
*/
|
|
5427
4512
|
async cardinalitiesAt(ref, generation, held) {
|
|
5428
|
-
validateUserRef(ref);
|
|
4513
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5429
4514
|
const reader = await this.readerAt(ref, generation, held);
|
|
5430
4515
|
return reader === null ? null : reader.cardinalities();
|
|
5431
4516
|
}
|
|
5432
4517
|
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
5433
4518
|
async sizeOfAt(ref, generation, held) {
|
|
5434
|
-
validateUserRef(ref);
|
|
4519
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5435
4520
|
const reader = await this.readerAt(ref, generation, held);
|
|
5436
4521
|
return reader === null ? null : { sizeBytes: reader.sizeBytes };
|
|
5437
4522
|
}
|
|
5438
4523
|
/**
|
|
5439
4524
|
* Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
|
|
5440
4525
|
* including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
|
|
5441
|
-
* created.
|
|
4526
|
+
* created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
|
|
4527
|
+
* registration to
|
|
5442
4528
|
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
5443
4529
|
*
|
|
5444
4530
|
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
5445
4531
|
* tell apart.
|
|
5446
4532
|
*/
|
|
5447
4533
|
async exists(ref) {
|
|
5448
|
-
validateUserRef(ref);
|
|
4534
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5449
4535
|
if (this.registry !== void 0) {
|
|
5450
4536
|
const row = await this.registry.get(ref);
|
|
5451
4537
|
return row !== null && row.status !== "destroyed";
|
|
@@ -5739,10 +4825,10 @@ var CrbmStorageChunkSource = class {
|
|
|
5739
4825
|
async *getChunksAt(ref, generation, keys, held, options) {
|
|
5740
4826
|
validateRead(ref, keys);
|
|
5741
4827
|
const retry = options?.retry;
|
|
5742
|
-
const
|
|
4828
|
+
const open = () => this.readerAt(ref, generation, held);
|
|
5743
4829
|
const segment = segmentKey(ref);
|
|
5744
4830
|
let since = this.invalidated.epoch;
|
|
5745
|
-
const reader = await viaRetry(retry,
|
|
4831
|
+
const reader = await viaRetry(retry, open);
|
|
5746
4832
|
if (reader === null) {
|
|
5747
4833
|
for (const key of keys) yield { key, bytes: null };
|
|
5748
4834
|
return;
|
|
@@ -5755,7 +4841,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5755
4841
|
while (await pull.advance()) {
|
|
5756
4842
|
if (this.invalidated.invalidatedSince(segment, since)) {
|
|
5757
4843
|
since = this.invalidated.epoch;
|
|
5758
|
-
if (await viaRetry(retry,
|
|
4844
|
+
if (await viaRetry(retry, open) === null) throw goneUnder(ref, generation);
|
|
5759
4845
|
}
|
|
5760
4846
|
yield pull.take((chunk) => chunk);
|
|
5761
4847
|
}
|
|
@@ -5771,11 +4857,11 @@ var CrbmStorageChunkSource = class {
|
|
|
5771
4857
|
}
|
|
5772
4858
|
}
|
|
5773
4859
|
async listChunkKeys(ref) {
|
|
5774
|
-
validateUserRef(ref);
|
|
4860
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5775
4861
|
return this.withFreshSnapshot(ref, (reader) => reader.chunkKeys(), []);
|
|
5776
4862
|
}
|
|
5777
4863
|
async sizeOf(ref) {
|
|
5778
|
-
validateUserRef(ref);
|
|
4864
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5779
4865
|
return this.withFreshSnapshot(
|
|
5780
4866
|
ref,
|
|
5781
4867
|
(reader) => ({ sizeBytes: reader.sizeBytes }),
|
|
@@ -5783,7 +4869,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5783
4869
|
);
|
|
5784
4870
|
}
|
|
5785
4871
|
async cardinalities(ref) {
|
|
5786
|
-
validateUserRef(ref);
|
|
4872
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5787
4873
|
return this.withFreshSnapshot(
|
|
5788
4874
|
ref,
|
|
5789
4875
|
(reader) => reader.cardinalities(),
|
|
@@ -5804,7 +4890,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5804
4890
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
5805
4891
|
*/
|
|
5806
4892
|
async summary(ref) {
|
|
5807
|
-
validateUserRef(ref);
|
|
4893
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5808
4894
|
return this.described(
|
|
5809
4895
|
ref,
|
|
5810
4896
|
(generation, row) => summaryOfGeneration(generation, row),
|
|
@@ -5837,7 +4923,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5837
4923
|
}
|
|
5838
4924
|
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5839
4925
|
async summaryAt(ref, generation, held) {
|
|
5840
|
-
validateUserRef(ref);
|
|
4926
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5841
4927
|
const reader = await this.readerAt(ref, generation, held);
|
|
5842
4928
|
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
5843
4929
|
}
|
|
@@ -5851,7 +4937,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5851
4937
|
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
5852
4938
|
*/
|
|
5853
4939
|
async stat(ref) {
|
|
5854
|
-
validateUserRef(ref);
|
|
4940
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5855
4941
|
return this.described(
|
|
5856
4942
|
ref,
|
|
5857
4943
|
(generation, row) => ({
|
|
@@ -5863,7 +4949,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5863
4949
|
}
|
|
5864
4950
|
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5865
4951
|
async statAt(ref, generation, held) {
|
|
5866
|
-
validateUserRef(ref);
|
|
4952
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5867
4953
|
const reader = await this.readerAt(ref, generation, held);
|
|
5868
4954
|
return reader === null ? null : sizedSummary(reader);
|
|
5869
4955
|
}
|
|
@@ -5885,8 +4971,9 @@ var CrbmStorageChunkSource = class {
|
|
|
5885
4971
|
*
|
|
5886
4972
|
* A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
|
|
5887
4973
|
* absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
|
|
5888
|
-
* generation to serve: storage is empty, or the row
|
|
5889
|
-
*
|
|
4974
|
+
* generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
|
|
4975
|
+
* dropped), in which case
|
|
4976
|
+
* reading empty is the documented outcome rather than a failure.
|
|
5890
4977
|
*
|
|
5891
4978
|
* **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
|
|
5892
4979
|
* re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
|
|
@@ -5984,9 +5071,10 @@ var PinnedStorageChunkSource = class {
|
|
|
5984
5071
|
this.inner = inner;
|
|
5985
5072
|
this.pins = pins;
|
|
5986
5073
|
for (const [key, pin] of pins) {
|
|
5987
|
-
|
|
5074
|
+
const held = pin;
|
|
5075
|
+
if (held.generation !== null && (typeof held.version !== "string" || typeof held.fingerprint !== "string"))
|
|
5988
5076
|
throw new ValidationError(
|
|
5989
|
-
`pin for "${key}" names generation ${
|
|
5077
|
+
`pin for "${key}" names generation ${held.generation} without its version and object fingerprint`
|
|
5990
5078
|
);
|
|
5991
5079
|
}
|
|
5992
5080
|
}
|
|
@@ -6093,6 +5181,7 @@ var PinnedStorageChunkSource = class {
|
|
|
6093
5181
|
|
|
6094
5182
|
// src/core/cooperative.ts
|
|
6095
5183
|
var YIELD_EVERY = 1024;
|
|
5184
|
+
var YIELD_EVERY_IDS = 1 << 14;
|
|
6096
5185
|
function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
6097
5186
|
if (clock === void 0) return () => null;
|
|
6098
5187
|
const sleep = clock.sleep;
|
|
@@ -6120,6 +5209,51 @@ var ChunkLoadInput = class {
|
|
|
6120
5209
|
}
|
|
6121
5210
|
chunks;
|
|
6122
5211
|
};
|
|
5212
|
+
var sourceErrors = /* @__PURE__ */ new WeakSet();
|
|
5213
|
+
function raisedBySource(err) {
|
|
5214
|
+
if (typeof err === "object" && err !== null) sourceErrors.add(err);
|
|
5215
|
+
return err;
|
|
5216
|
+
}
|
|
5217
|
+
function isSourceError(err) {
|
|
5218
|
+
return typeof err === "object" && err !== null && sourceErrors.has(err);
|
|
5219
|
+
}
|
|
5220
|
+
var SortedLoadInput = class {
|
|
5221
|
+
constructor(ids) {
|
|
5222
|
+
this.ids = ids;
|
|
5223
|
+
}
|
|
5224
|
+
ids;
|
|
5225
|
+
failure;
|
|
5226
|
+
/** Keep `error` as what the ids raised, unless an earlier one is kept. */
|
|
5227
|
+
fail(error) {
|
|
5228
|
+
this.failure ??= { error };
|
|
5229
|
+
}
|
|
5230
|
+
/** What the load throws for a write that failed with `err`: the error the ids raised, when they raised one. */
|
|
5231
|
+
thrown(err) {
|
|
5232
|
+
return this.failure === void 0 ? err : raisedBySource(this.failure.error);
|
|
5233
|
+
}
|
|
5234
|
+
};
|
|
5235
|
+
function kindOfOption(value) {
|
|
5236
|
+
if (value === null) return "null";
|
|
5237
|
+
if (Array.isArray(value)) return "an array";
|
|
5238
|
+
const kind = typeof value;
|
|
5239
|
+
return kind === "object" || kind === "undefined" ? `an ${kind}` : `a ${kind}`;
|
|
5240
|
+
}
|
|
5241
|
+
function wantsSorted(sorted) {
|
|
5242
|
+
if (sorted === void 0 || sorted === false) return false;
|
|
5243
|
+
if (sorted !== true) {
|
|
5244
|
+
throw new ValidationError(`sorted must be a boolean; got ${kindOfOption(sorted)}`);
|
|
5245
|
+
}
|
|
5246
|
+
return true;
|
|
5247
|
+
}
|
|
5248
|
+
function sortedLoadInput(prepared, sorted) {
|
|
5249
|
+
if (!wantsSorted(sorted)) return prepared;
|
|
5250
|
+
if (prepared instanceof DecodedLoadInput) {
|
|
5251
|
+
throw new ValidationError(
|
|
5252
|
+
"sorted: true applies to ids, and this load was given a bitmap ({ serialized }, { bitmap } or a bitmap itself), which is already held whole and is written from its own chunks: leave sorted off"
|
|
5253
|
+
);
|
|
5254
|
+
}
|
|
5255
|
+
return new SortedLoadInput(prepared);
|
|
5256
|
+
}
|
|
6123
5257
|
function tagOf(value) {
|
|
6124
5258
|
return Object.prototype.toString.call(value).slice(8, -1);
|
|
6125
5259
|
}
|
|
@@ -6204,6 +5338,98 @@ function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
|
6204
5338
|
}
|
|
6205
5339
|
}
|
|
6206
5340
|
|
|
5341
|
+
// src/core/sorted-chunks.ts
|
|
5342
|
+
var NOT_ASCENDING = "lower than the id before it; { sorted: true } takes ids in ascending order (repeats are fine): sort at the source, or leave sorted off";
|
|
5343
|
+
var SourceFailed = class extends Error {
|
|
5344
|
+
constructor() {
|
|
5345
|
+
super("the ids of a sorted load raised an error");
|
|
5346
|
+
}
|
|
5347
|
+
};
|
|
5348
|
+
function sortedChunks(input, codec, clock) {
|
|
5349
|
+
return chunksOf(input, codec, clock);
|
|
5350
|
+
}
|
|
5351
|
+
function chunkCutter(codec) {
|
|
5352
|
+
const remainders = new Uint32Array(1 << 16);
|
|
5353
|
+
let position = 0;
|
|
5354
|
+
let last = -1;
|
|
5355
|
+
let chunk = -1;
|
|
5356
|
+
let held = 0;
|
|
5357
|
+
const cut = () => ({
|
|
5358
|
+
chunkKey: chunk,
|
|
5359
|
+
bitmap: codec.fromValues(remainders.subarray(0, held))
|
|
5360
|
+
});
|
|
5361
|
+
return {
|
|
5362
|
+
accept(id) {
|
|
5363
|
+
if (!Number.isInteger(id) || id < 0 || id > U32_MAX) {
|
|
5364
|
+
const cause = refusalOfId(id);
|
|
5365
|
+
const positioned = new ValidationError(`ids[${position}]: ${cause.message}`);
|
|
5366
|
+
positioned.cause = cause;
|
|
5367
|
+
throw positioned;
|
|
5368
|
+
}
|
|
5369
|
+
if (id <= last) {
|
|
5370
|
+
if (id < last) throw new ValidationError(`ids[${position}]: ${NOT_ASCENDING}`);
|
|
5371
|
+
position++;
|
|
5372
|
+
return void 0;
|
|
5373
|
+
}
|
|
5374
|
+
last = id;
|
|
5375
|
+
position++;
|
|
5376
|
+
const key = id >>> 16;
|
|
5377
|
+
let closed;
|
|
5378
|
+
if (key !== chunk) {
|
|
5379
|
+
if (held > 0) closed = cut();
|
|
5380
|
+
chunk = key;
|
|
5381
|
+
held = 0;
|
|
5382
|
+
}
|
|
5383
|
+
remainders[held++] = id & 65535;
|
|
5384
|
+
return closed;
|
|
5385
|
+
},
|
|
5386
|
+
rest() {
|
|
5387
|
+
return held > 0 ? cut() : void 0;
|
|
5388
|
+
}
|
|
5389
|
+
};
|
|
5390
|
+
}
|
|
5391
|
+
async function* chunksOf(input, codec, clock) {
|
|
5392
|
+
const cutter = chunkCutter(codec);
|
|
5393
|
+
const tick = yieldEvery(clock, YIELD_EVERY_IDS);
|
|
5394
|
+
let handing = false;
|
|
5395
|
+
const { ids } = input;
|
|
5396
|
+
try {
|
|
5397
|
+
if (Symbol.asyncIterator in ids) {
|
|
5398
|
+
for await (const id of ids) {
|
|
5399
|
+
const closed = cutter.accept(id);
|
|
5400
|
+
if (closed !== void 0) {
|
|
5401
|
+
handing = true;
|
|
5402
|
+
yield closed;
|
|
5403
|
+
handing = false;
|
|
5404
|
+
}
|
|
5405
|
+
const pause = tick();
|
|
5406
|
+
if (pause !== null) await pause;
|
|
5407
|
+
}
|
|
5408
|
+
} else {
|
|
5409
|
+
for (const id of ids) {
|
|
5410
|
+
const closed = cutter.accept(id);
|
|
5411
|
+
if (closed !== void 0) {
|
|
5412
|
+
handing = true;
|
|
5413
|
+
yield closed;
|
|
5414
|
+
handing = false;
|
|
5415
|
+
}
|
|
5416
|
+
const pause = tick();
|
|
5417
|
+
if (pause !== null) await pause;
|
|
5418
|
+
}
|
|
5419
|
+
}
|
|
5420
|
+
const last = cutter.rest();
|
|
5421
|
+
if (last !== void 0) {
|
|
5422
|
+
handing = true;
|
|
5423
|
+
yield last;
|
|
5424
|
+
handing = false;
|
|
5425
|
+
}
|
|
5426
|
+
} catch (err) {
|
|
5427
|
+
if (handing) throw err;
|
|
5428
|
+
input.fail(err);
|
|
5429
|
+
throw new SourceFailed();
|
|
5430
|
+
}
|
|
5431
|
+
}
|
|
5432
|
+
|
|
6207
5433
|
// src/core/crbm/writer.ts
|
|
6208
5434
|
var CrbmWriter = class {
|
|
6209
5435
|
constructor(sink, options) {
|
|
@@ -6351,25 +5577,42 @@ var CrbmWriter = class {
|
|
|
6351
5577
|
block.set(EXT_MAGIC, sectionsLength + 8);
|
|
6352
5578
|
return block;
|
|
6353
5579
|
}
|
|
5580
|
+
/**
|
|
5581
|
+
* The index region: per entry, four varints (key delta, offset gap, length, cardinality) and a fixed 4-byte
|
|
5582
|
+
* little-endian crc32c. A first pass sizes the region exactly, so the bytes are written once into one buffer and no
|
|
5583
|
+
* intermediate array is built.
|
|
5584
|
+
*/
|
|
6354
5585
|
encodeIndex() {
|
|
6355
|
-
|
|
5586
|
+
let size = 0;
|
|
6356
5587
|
let prevKey = 0;
|
|
6357
5588
|
let prevEnd = PAYLOAD_START;
|
|
6358
5589
|
for (const e of this.entries) {
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
);
|
|
5590
|
+
size += varintLength(e.chunkKey - prevKey) + varintLength(e.offset - prevEnd) + varintLength(e.length) + varintLength(e.cardinality) + 4;
|
|
5591
|
+
prevKey = e.chunkKey;
|
|
5592
|
+
prevEnd = e.offset + e.length;
|
|
5593
|
+
}
|
|
5594
|
+
const out = new Uint8Array(size);
|
|
5595
|
+
let pos = 0;
|
|
5596
|
+
prevKey = 0;
|
|
5597
|
+
prevEnd = PAYLOAD_START;
|
|
5598
|
+
for (const e of this.entries) {
|
|
5599
|
+
pos = writeVarintAt(out, pos, e.chunkKey - prevKey);
|
|
5600
|
+
pos = writeVarintAt(out, pos, e.offset - prevEnd);
|
|
5601
|
+
pos = writeVarintAt(out, pos, e.length);
|
|
5602
|
+
pos = writeVarintAt(out, pos, e.cardinality);
|
|
5603
|
+
out[pos++] = e.crc32c & 255;
|
|
5604
|
+
out[pos++] = e.crc32c >>> 8 & 255;
|
|
5605
|
+
out[pos++] = e.crc32c >>> 16 & 255;
|
|
5606
|
+
out[pos++] = e.crc32c >>> 24 & 255;
|
|
6369
5607
|
prevKey = e.chunkKey;
|
|
6370
5608
|
prevEnd = e.offset + e.length;
|
|
6371
5609
|
}
|
|
6372
|
-
|
|
5610
|
+
if (pos !== out.length) {
|
|
5611
|
+
throw new IntegrityError(
|
|
5612
|
+
`the chunk index was sized at ${out.length} bytes and encoded as ${pos}; the index was not written`
|
|
5613
|
+
);
|
|
5614
|
+
}
|
|
5615
|
+
return out;
|
|
6373
5616
|
}
|
|
6374
5617
|
buildFooter(indexOffset, indexRegion, indexNonce, indexTag) {
|
|
6375
5618
|
const encrypted = this.options.crypto !== void 0;
|
|
@@ -6451,13 +5694,15 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
6451
5694
|
let cardinality = 0;
|
|
6452
5695
|
let identity;
|
|
6453
5696
|
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
5697
|
+
chunkCount = 0;
|
|
5698
|
+
cardinality = 0;
|
|
6454
5699
|
const writer = new CrbmWriter(sink, {
|
|
6455
5700
|
generation: key.generation,
|
|
6456
5701
|
crypto: options.crypto,
|
|
6457
5702
|
metadata: options.metadata,
|
|
6458
5703
|
entropy: options.entropy
|
|
6459
5704
|
});
|
|
6460
|
-
for (const chunk of chunks) {
|
|
5705
|
+
for (const chunk of chunks()) {
|
|
6461
5706
|
await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
|
|
6462
5707
|
chunkCount++;
|
|
6463
5708
|
cardinality += chunk.cardinality;
|
|
@@ -6486,24 +5731,42 @@ async function writeCrbmGenerationStream(driver, key, chunks, options) {
|
|
|
6486
5731
|
let cardinality = 0;
|
|
6487
5732
|
const tick = yieldEvery(options.clock);
|
|
6488
5733
|
let identity;
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
if (
|
|
6498
|
-
|
|
6499
|
-
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
6500
|
-
chunkKeys.push(chunkKey);
|
|
6501
|
-
cardinality += bitmap.size;
|
|
6502
|
-
const pause = tick();
|
|
6503
|
-
if (pause !== null) await pause;
|
|
5734
|
+
let runs = 0;
|
|
5735
|
+
let rerun;
|
|
5736
|
+
let firstRun;
|
|
5737
|
+
const put = driver.putImmutable(key, async (sink) => {
|
|
5738
|
+
if (++runs > 1) {
|
|
5739
|
+
rerun = new IntegrityError(
|
|
5740
|
+
`the storage driver ran the write of ${key.segment}.${key.generation} again: its chunks are streamed and can be read once, so nothing was written`
|
|
5741
|
+
);
|
|
5742
|
+
if (firstRun !== void 0) rerun.cause = firstRun.error;
|
|
5743
|
+
throw rerun;
|
|
6504
5744
|
}
|
|
6505
|
-
|
|
6506
|
-
|
|
5745
|
+
try {
|
|
5746
|
+
const writer = new CrbmWriter(sink, {
|
|
5747
|
+
generation: key.generation,
|
|
5748
|
+
crypto: options.crypto,
|
|
5749
|
+
metadata: options.metadata,
|
|
5750
|
+
entropy: options.entropy
|
|
5751
|
+
});
|
|
5752
|
+
for await (const { chunkKey, bitmap } of chunks) {
|
|
5753
|
+
if (bitmap.isEmpty) continue;
|
|
5754
|
+
bitmap.optimize?.();
|
|
5755
|
+
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
5756
|
+
chunkKeys.push(chunkKey);
|
|
5757
|
+
cardinality += bitmap.size;
|
|
5758
|
+
const pause = tick();
|
|
5759
|
+
if (pause !== null) await pause;
|
|
5760
|
+
}
|
|
5761
|
+
await writer.finish();
|
|
5762
|
+
identity = writer.identity;
|
|
5763
|
+
} catch (err) {
|
|
5764
|
+
firstRun = { error: err };
|
|
5765
|
+
throw err;
|
|
5766
|
+
}
|
|
5767
|
+
});
|
|
5768
|
+
const { size, sha256 } = await put.catch((err) => {
|
|
5769
|
+
throw rerun ?? err;
|
|
6507
5770
|
});
|
|
6508
5771
|
if (identity === void 0) {
|
|
6509
5772
|
throw new IntegrityError(
|
|
@@ -6567,17 +5830,18 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6567
5830
|
let throughChurn = false;
|
|
6568
5831
|
let waited;
|
|
6569
5832
|
let sawUnanswered = false;
|
|
5833
|
+
let readFault;
|
|
5834
|
+
const readAgain = readOf(registry, key, (e) => readFault = e);
|
|
5835
|
+
let tokenReported;
|
|
6570
5836
|
let lastRecord;
|
|
6571
5837
|
const sent = registry.reportsWriteTokens === true ? /* @__PURE__ */ new Set() : void 0;
|
|
6572
|
-
const report = sent === void 0 ? void 0 : (token) => void sent.add(token);
|
|
5838
|
+
const report = sent === void 0 ? void 0 : (token) => void (sent.add(token), tokenReported = true);
|
|
6573
5839
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
6574
5840
|
const reused = attempt === 0 && options.row !== void 0;
|
|
6575
|
-
const holdsRow = fresh === void 0;
|
|
5841
|
+
const holdsRow = fresh === void 0 || fresh === waited;
|
|
6576
5842
|
const record = fresh !== void 0 ? fresh : reused ? options.row ?? null : await registry.get(key);
|
|
6577
|
-
const write = holdsRow
|
|
6578
|
-
|
|
6579
|
-
...report !== void 0 ? { onToken: report } : {}
|
|
6580
|
-
} : void 0;
|
|
5843
|
+
const write = writeOptions(holdsRow ? { held: record } : {}, report);
|
|
5844
|
+
tokenReported = false;
|
|
6581
5845
|
fresh = void 0;
|
|
6582
5846
|
unanswered = void 0;
|
|
6583
5847
|
lastRecord = record;
|
|
@@ -6614,7 +5878,7 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6614
5878
|
if (record !== null && held != null && record === waited && onlyLeasesDiffer(held, record, options.expectRowIgnores)) {
|
|
6615
5879
|
throughChurn = true;
|
|
6616
5880
|
} else {
|
|
6617
|
-
const settled2 = record === waited ? void 0 : await churn.settle(held, record,
|
|
5881
|
+
const settled2 = record === waited ? void 0 : await churn.settle(held, record, readAgain, options.expectRowIgnores);
|
|
6618
5882
|
if (settled2 !== void 0) {
|
|
6619
5883
|
waited = settled2.row;
|
|
6620
5884
|
fresh = settled2.row;
|
|
@@ -6642,7 +5906,7 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6642
5906
|
)).token;
|
|
6643
5907
|
} else if (record.status === "destroyed") {
|
|
6644
5908
|
throw new ValidationError(
|
|
6645
|
-
`segment "${key.segment}" was destroyed
|
|
5909
|
+
`segment "${key.segment}" was destroyed while generation ${key.generation} was being written \u2014 refusing to publish it: no generation resolves through a destroyed row. Use a new segment.`
|
|
6646
5910
|
);
|
|
6647
5911
|
} else if (record.currentGen === null) {
|
|
6648
5912
|
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
@@ -6690,13 +5954,14 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6690
5954
|
};
|
|
6691
5955
|
} catch (err) {
|
|
6692
5956
|
if (isWriteConflictError(err)) {
|
|
5957
|
+
if (isUnsettledConflict(err)) sawUnanswered = noteUnsettled(options);
|
|
6693
5958
|
failedOn = record;
|
|
6694
5959
|
if (throughChurn) attempt -= 1;
|
|
6695
5960
|
continue;
|
|
6696
5961
|
}
|
|
6697
|
-
if (!isTransientError(err)) throw err;
|
|
6698
|
-
|
|
6699
|
-
sawUnanswered =
|
|
5962
|
+
if (!isTransientError(err) || err === readFault) throw err;
|
|
5963
|
+
if (sent !== void 0 && !tokenReported) throw err;
|
|
5964
|
+
sawUnanswered = noteUnsettled(options);
|
|
6700
5965
|
let now;
|
|
6701
5966
|
try {
|
|
6702
5967
|
now = await registry.get(key);
|
|
@@ -6740,6 +6005,23 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6740
6005
|
`publishGeneration: contention setting currentGen for ${key.segment}${CONTENTION_NEXT_STEP}`
|
|
6741
6006
|
);
|
|
6742
6007
|
}
|
|
6008
|
+
function writeOptions(held, onToken) {
|
|
6009
|
+
return held.held === void 0 && onToken === void 0 ? void 0 : { ...held, ...onToken === void 0 ? {} : { onToken } };
|
|
6010
|
+
}
|
|
6011
|
+
function readOf(registry, key, onFault) {
|
|
6012
|
+
return async () => {
|
|
6013
|
+
try {
|
|
6014
|
+
return await registry.get(key);
|
|
6015
|
+
} catch (err) {
|
|
6016
|
+
onFault(err);
|
|
6017
|
+
throw err;
|
|
6018
|
+
}
|
|
6019
|
+
};
|
|
6020
|
+
}
|
|
6021
|
+
function noteUnsettled(options) {
|
|
6022
|
+
options.onUnanswered?.();
|
|
6023
|
+
return true;
|
|
6024
|
+
}
|
|
6743
6025
|
var REFUSED = { published: false };
|
|
6744
6026
|
async function publishGeneration(registry, key, options) {
|
|
6745
6027
|
return (await publishGenerationKept(registry, key, options)).published;
|
|
@@ -6747,7 +6029,7 @@ async function publishGeneration(registry, key, options) {
|
|
|
6747
6029
|
async function landedHere(now, before, key, holdsOwnObject, sent) {
|
|
6748
6030
|
if (now === null || now.status === "destroyed" || now.currentGen !== key.generation) return false;
|
|
6749
6031
|
if (before !== null && !sameIncarnation(before, now)) return false;
|
|
6750
|
-
if (sent !== void 0) return
|
|
6032
|
+
if (sent !== void 0) return pointerSetBy(now, before, key.generation, sent);
|
|
6751
6033
|
if (holdsOwnObject === void 0) {
|
|
6752
6034
|
throw new TransientError(
|
|
6753
6035
|
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled, since the caller gave no way to tell its own object from another's. Its object is kept; re-run the write, which numbers past it.`
|
|
@@ -6768,18 +6050,21 @@ function outcomeUnknown(key, cause) {
|
|
|
6768
6050
|
|
|
6769
6051
|
// src/core/bulk-load.ts
|
|
6770
6052
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
6771
|
-
var YIELD_EVERY_IDS = 1 << 14;
|
|
6772
6053
|
async function encodedChunksOfInput(ids, codec, clock) {
|
|
6773
|
-
|
|
6054
|
+
const { bitmap } = ids instanceof DecodedLoadInput ? ids : { bitmap: void 0 };
|
|
6055
|
+
if (bitmap?.encodeChunks !== void 0) {
|
|
6774
6056
|
const pause = yieldEvery(clock, 1);
|
|
6775
6057
|
await pause();
|
|
6776
|
-
|
|
6058
|
+
let first = bitmap.encodeChunks();
|
|
6777
6059
|
await pause();
|
|
6778
|
-
return
|
|
6060
|
+
return () => {
|
|
6061
|
+
const chunks = first ?? bitmap.encodeChunks();
|
|
6062
|
+
first = void 0;
|
|
6063
|
+
return chunks;
|
|
6064
|
+
};
|
|
6779
6065
|
}
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
);
|
|
6066
|
+
const bucketed = await bucketIds(bitmap ?? ids, codec, clock);
|
|
6067
|
+
return () => encodeEach(bucketed);
|
|
6783
6068
|
}
|
|
6784
6069
|
async function bucketIds(ids, codec, clock) {
|
|
6785
6070
|
const byChunk = /* @__PURE__ */ new Map();
|
|
@@ -6852,47 +6137,22 @@ function refuseCleartextWhenEncryptionRequired(options, ref, currentGen) {
|
|
|
6852
6137
|
}
|
|
6853
6138
|
}
|
|
6854
6139
|
async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
6140
|
+
checkRequireEncryption(options.requireEncryption);
|
|
6855
6141
|
requireKeystoreWhenEncryptionRequired(options);
|
|
6856
6142
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
6857
6143
|
const entropy = requireEntropy(options.entropy, "a load");
|
|
6858
|
-
let
|
|
6859
|
-
if (ids instanceof
|
|
6860
|
-
|
|
6144
|
+
let write;
|
|
6145
|
+
if (ids instanceof SortedLoadInput) {
|
|
6146
|
+
write = (key2, writeOptions2) => writeSorted(driver, key2, ids, codec, writeOptions2);
|
|
6861
6147
|
} else {
|
|
6862
|
-
chunks = await
|
|
6863
|
-
|
|
6864
|
-
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6865
|
-
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
6866
|
-
const passed = options.row;
|
|
6867
|
-
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(ref) : null;
|
|
6868
|
-
if (existing?.status === "destroyed") {
|
|
6869
|
-
throw new ValidationError(
|
|
6870
|
-
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
6871
|
-
);
|
|
6872
|
-
}
|
|
6873
|
-
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
6874
|
-
throw new KeyUnavailableError(
|
|
6875
|
-
`segment "${ref.segment}" is encrypted but this load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
6876
|
-
);
|
|
6148
|
+
const chunks = await encodedBeforeTheRow(ids, codec, options.clock);
|
|
6149
|
+
write = (key2, writeOptions2) => writeEncodedChunks(driver, key2, chunks, writeOptions2);
|
|
6877
6150
|
}
|
|
6151
|
+
const existing = await rowForWrite(ref, options);
|
|
6878
6152
|
const generation = await ahead.generation;
|
|
6879
6153
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
if (options.keystore !== void 0) {
|
|
6883
|
-
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0) {
|
|
6884
|
-
const early = ahead.unwrapped;
|
|
6885
|
-
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
6886
|
-
crypto = { aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
6887
|
-
} else if (existing !== null && existing.currentGen !== null) {
|
|
6888
|
-
refuseCleartextWhenEncryptionRequired(options, ref, existing.currentGen);
|
|
6889
|
-
} else {
|
|
6890
|
-
const minted = await options.keystore.createDek();
|
|
6891
|
-
newWrapped = minted.wrapped;
|
|
6892
|
-
crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
6893
|
-
}
|
|
6894
|
-
}
|
|
6895
|
-
const { size, sha256, chunkCount, cardinality, fingerprint, tailBytes } = await writeEncodedChunks(driver, key, chunks, {
|
|
6154
|
+
const { crypto, newWrapped } = await keyForWrite(existing, key, ahead, options);
|
|
6155
|
+
const { size, sha256, chunkCount, cardinality, fingerprint, tailBytes } = await write(key, {
|
|
6896
6156
|
crypto,
|
|
6897
6157
|
clock: options.clock,
|
|
6898
6158
|
metadata: options.metadata,
|
|
@@ -6904,38 +6164,7 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6904
6164
|
{ cardinality, metadata: options.metadata, fingerprint, tailBytes },
|
|
6905
6165
|
crypto?.aead
|
|
6906
6166
|
);
|
|
6907
|
-
|
|
6908
|
-
const published = await publishGenerationKept(options.registry, key, {
|
|
6909
|
-
wrappedDeks: newWrapped,
|
|
6910
|
-
summary,
|
|
6911
|
-
footerCrc: footerCrcOf(fingerprint),
|
|
6912
|
-
cleartext: crypto === void 0,
|
|
6913
|
-
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
6914
|
-
clock: options.clock
|
|
6915
|
-
});
|
|
6916
|
-
const becameCurrent = published.published;
|
|
6917
|
-
if (becameCurrent) {
|
|
6918
|
-
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
6919
|
-
kind: "segment.publish",
|
|
6920
|
-
namespace: key.namespace,
|
|
6921
|
-
segment: key.segment,
|
|
6922
|
-
...incarnationField(published.token),
|
|
6923
|
-
generation: key.generation
|
|
6924
|
-
});
|
|
6925
|
-
}
|
|
6926
|
-
return {
|
|
6927
|
-
size,
|
|
6928
|
-
sha256,
|
|
6929
|
-
chunkCount,
|
|
6930
|
-
cardinality,
|
|
6931
|
-
summary,
|
|
6932
|
-
becameCurrent,
|
|
6933
|
-
wrappedDeks: newWrapped,
|
|
6934
|
-
encrypted: crypto !== void 0,
|
|
6935
|
-
fingerprint
|
|
6936
|
-
};
|
|
6937
|
-
}
|
|
6938
|
-
return {
|
|
6167
|
+
const result = {
|
|
6939
6168
|
size,
|
|
6940
6169
|
sha256,
|
|
6941
6170
|
chunkCount,
|
|
@@ -6945,6 +6174,84 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6945
6174
|
encrypted: crypto !== void 0,
|
|
6946
6175
|
fingerprint
|
|
6947
6176
|
};
|
|
6177
|
+
if (options.registry === void 0 || options.publish === false) return result;
|
|
6178
|
+
const published = await publishGenerationKept(options.registry, key, {
|
|
6179
|
+
wrappedDeks: newWrapped,
|
|
6180
|
+
summary,
|
|
6181
|
+
footerCrc: footerCrcOf(fingerprint),
|
|
6182
|
+
cleartext: crypto === void 0,
|
|
6183
|
+
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
6184
|
+
clock: options.clock
|
|
6185
|
+
});
|
|
6186
|
+
const becameCurrent = published.published;
|
|
6187
|
+
if (becameCurrent) {
|
|
6188
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
6189
|
+
kind: "segment.publish",
|
|
6190
|
+
namespace: key.namespace,
|
|
6191
|
+
segment: key.segment,
|
|
6192
|
+
...incarnationField(published.token),
|
|
6193
|
+
generation: key.generation
|
|
6194
|
+
});
|
|
6195
|
+
}
|
|
6196
|
+
return { ...result, becameCurrent };
|
|
6197
|
+
}
|
|
6198
|
+
async function encodedBeforeTheRow(ids, codec, clock) {
|
|
6199
|
+
try {
|
|
6200
|
+
if (ids instanceof ChunkLoadInput) {
|
|
6201
|
+
const chunks = await collectChunks(ids, codec, clock);
|
|
6202
|
+
return () => encodeEach(rangeCheckedAtWrite(chunks));
|
|
6203
|
+
}
|
|
6204
|
+
return await encodedChunksOfInput(ids, codec, clock);
|
|
6205
|
+
} catch (err) {
|
|
6206
|
+
throw raisedBySource(err);
|
|
6207
|
+
}
|
|
6208
|
+
}
|
|
6209
|
+
async function rowForWrite(ref, options) {
|
|
6210
|
+
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6211
|
+
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
6212
|
+
const passed = options.row;
|
|
6213
|
+
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(ref) : null;
|
|
6214
|
+
if (existing?.status === "destroyed") {
|
|
6215
|
+
throw new ValidationError(
|
|
6216
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to write: no generation resolves through a destroyed row; use a new segment`
|
|
6217
|
+
);
|
|
6218
|
+
}
|
|
6219
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
6220
|
+
throw new KeyUnavailableError(
|
|
6221
|
+
`segment "${ref.segment}" is encrypted but this load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
6222
|
+
);
|
|
6223
|
+
}
|
|
6224
|
+
return existing;
|
|
6225
|
+
}
|
|
6226
|
+
async function keyForWrite(existing, key, ahead, options) {
|
|
6227
|
+
if (options.keystore === void 0) return {};
|
|
6228
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0) {
|
|
6229
|
+
const early = ahead.unwrapped;
|
|
6230
|
+
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
6231
|
+
return { crypto: { aead, aadFor: (scope) => aadFor(key, key.generation, scope) } };
|
|
6232
|
+
}
|
|
6233
|
+
if (existing !== null && existing.currentGen !== null) {
|
|
6234
|
+
refuseCleartextWhenEncryptionRequired(options, key, existing.currentGen);
|
|
6235
|
+
return {};
|
|
6236
|
+
}
|
|
6237
|
+
const minted = await options.keystore.createDek();
|
|
6238
|
+
return {
|
|
6239
|
+
crypto: { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) },
|
|
6240
|
+
newWrapped: minted.wrapped
|
|
6241
|
+
};
|
|
6242
|
+
}
|
|
6243
|
+
async function writeSorted(driver, key, input, codec, options) {
|
|
6244
|
+
try {
|
|
6245
|
+
const { chunkKeys, ...written } = await writeCrbmGenerationStream(
|
|
6246
|
+
driver,
|
|
6247
|
+
key,
|
|
6248
|
+
sortedChunks(input, codec, options.clock),
|
|
6249
|
+
options
|
|
6250
|
+
);
|
|
6251
|
+
return { ...written, chunkCount: chunkKeys.length };
|
|
6252
|
+
} catch (err) {
|
|
6253
|
+
throw input.thrown(err);
|
|
6254
|
+
}
|
|
6948
6255
|
}
|
|
6949
6256
|
function* rangeCheckedAtWrite(chunks) {
|
|
6950
6257
|
for (const chunk of chunks) {
|
|
@@ -7039,7 +6346,84 @@ async function readSeed(storage, ref, row, generation, keep, listed2, leased, re
|
|
|
7039
6346
|
return read.filter((e) => e !== void 0);
|
|
7040
6347
|
}
|
|
7041
6348
|
|
|
6349
|
+
// src/core/settle.ts
|
|
6350
|
+
function writeTokens(registry, onSent) {
|
|
6351
|
+
const sent = registry.reportsWriteTokens === true ? /* @__PURE__ */ new Set() : void 0;
|
|
6352
|
+
return {
|
|
6353
|
+
sent,
|
|
6354
|
+
options: sent === void 0 ? void 0 : {
|
|
6355
|
+
onToken: (token) => {
|
|
6356
|
+
sent.add(token);
|
|
6357
|
+
onSent?.(token);
|
|
6358
|
+
}
|
|
6359
|
+
},
|
|
6360
|
+
wrote: (row) => row !== null && sent !== void 0 && (sent.has(row.token) || sent.has(row.pointerId))
|
|
6361
|
+
};
|
|
6362
|
+
}
|
|
6363
|
+
var READ_BACK_TRIES = 3;
|
|
6364
|
+
async function readBack(read, unanswered, waits = {}) {
|
|
6365
|
+
let last;
|
|
6366
|
+
for (let attempt = 0; attempt < READ_BACK_TRIES; attempt++) {
|
|
6367
|
+
try {
|
|
6368
|
+
return await read();
|
|
6369
|
+
} catch (err) {
|
|
6370
|
+
if (!isTransientError(err)) throw err;
|
|
6371
|
+
last = err;
|
|
6372
|
+
}
|
|
6373
|
+
if (attempt < READ_BACK_TRIES - 1) {
|
|
6374
|
+
await jitteredWait(waits.clock, waits.rng, UNANSWERED_BASE_MS * 2 ** attempt);
|
|
6375
|
+
}
|
|
6376
|
+
}
|
|
6377
|
+
if (unanswered.cause === void 0) unanswered.cause = last;
|
|
6378
|
+
throw unanswered;
|
|
6379
|
+
}
|
|
6380
|
+
async function deleteSettled(storage, key, options, read = (op) => op(), waits = {}) {
|
|
6381
|
+
try {
|
|
6382
|
+
await storage.delete(key, options);
|
|
6383
|
+
return;
|
|
6384
|
+
} catch (err) {
|
|
6385
|
+
if (!isTransientError(err)) throw err;
|
|
6386
|
+
const tail = await readBack(
|
|
6387
|
+
() => read(() => storage.getTail(key, 0)).then(
|
|
6388
|
+
(t) => t,
|
|
6389
|
+
(readErr) => {
|
|
6390
|
+
if (isNotFoundError(readErr)) return null;
|
|
6391
|
+
throw readErr;
|
|
6392
|
+
}
|
|
6393
|
+
),
|
|
6394
|
+
err,
|
|
6395
|
+
waits
|
|
6396
|
+
);
|
|
6397
|
+
if (tail === null) return;
|
|
6398
|
+
const version = tail.version;
|
|
6399
|
+
if (options?.ifVersion !== void 0 && version !== void 0 && version !== options.ifVersion) {
|
|
6400
|
+
throw new WriteConflictError(
|
|
6401
|
+
`generation s.${key.generation} is another object than the version given; not deleted`
|
|
6402
|
+
);
|
|
6403
|
+
}
|
|
6404
|
+
throw err;
|
|
6405
|
+
}
|
|
6406
|
+
}
|
|
6407
|
+
async function rowDeleteSettled(registry, ref, expected, waits = {}) {
|
|
6408
|
+
try {
|
|
6409
|
+
await registry.delete(ref, expected);
|
|
6410
|
+
return;
|
|
6411
|
+
} catch (err) {
|
|
6412
|
+
if (!isTransientError(err) && !isWriteConflictError(err)) throw err;
|
|
6413
|
+
const row = await readBack(() => registry.get(ref), err, waits);
|
|
6414
|
+
if (row === null) return;
|
|
6415
|
+
throw err;
|
|
6416
|
+
}
|
|
6417
|
+
}
|
|
6418
|
+
|
|
7042
6419
|
// src/core/generation-gc.ts
|
|
6420
|
+
function deleteGeneration(deps, ref, generation) {
|
|
6421
|
+
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
6422
|
+
return deleteSettled(deps.storage, key, void 0, void 0, {
|
|
6423
|
+
clock: deps.clock,
|
|
6424
|
+
rng: deps.rng
|
|
6425
|
+
});
|
|
6426
|
+
}
|
|
7043
6427
|
var DELETE_SETTLE_WAITS_MS = Object.freeze([250, 500, 1e3]);
|
|
7044
6428
|
function watchDeletes(deps, ref) {
|
|
7045
6429
|
const gone = /* @__PURE__ */ new Set();
|
|
@@ -7127,19 +6511,44 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
|
7127
6511
|
async function collectBelowPointer(ref, deps, options = {}) {
|
|
7128
6512
|
const { deleted, missing, knownSpared } = await listingPass(ref, deps, {
|
|
7129
6513
|
keep: options.keep,
|
|
7130
|
-
skipKnown: options.skipKnown
|
|
6514
|
+
skipKnown: options.skipKnown,
|
|
6515
|
+
incarnation: options.incarnation
|
|
7131
6516
|
});
|
|
7132
6517
|
return missing === void 0 ? { deleted, knownSpared } : { deleted, missing, knownSpared };
|
|
7133
6518
|
}
|
|
6519
|
+
function isAskedRow(asked, row) {
|
|
6520
|
+
return asked === void 0 || sameIncarnation(asked, row) && asked.status === "destroyed" === (row.status === "destroyed");
|
|
6521
|
+
}
|
|
6522
|
+
async function readHeldRow(deps, ref, asked) {
|
|
6523
|
+
const row = await deps.registry.get(ref);
|
|
6524
|
+
if (row === null || isAskedRow(asked, row)) return row;
|
|
6525
|
+
throw refusedOtherRow(ref);
|
|
6526
|
+
}
|
|
6527
|
+
function quietOf(options) {
|
|
6528
|
+
if (options.quiet === true && options.incarnation !== void 0) {
|
|
6529
|
+
throw new ValidationError("a quiet collection pass cannot be held to a row incarnation");
|
|
6530
|
+
}
|
|
6531
|
+
return options.quiet === true;
|
|
6532
|
+
}
|
|
6533
|
+
function changedWhileCollecting(ref) {
|
|
6534
|
+
return new WriteConflictError(
|
|
6535
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
6536
|
+
);
|
|
6537
|
+
}
|
|
6538
|
+
function refusedOtherRow(ref) {
|
|
6539
|
+
return new WriteConflictError(
|
|
6540
|
+
`segment ${ref.segment} is not the row this collection works from: the row was purged and created again, or is no longer of the same kind (a tombstone or an active row), since it was read`
|
|
6541
|
+
);
|
|
6542
|
+
}
|
|
7134
6543
|
async function listingPass(ref, deps, options) {
|
|
7135
6544
|
const keep = options.keep ?? 1;
|
|
7136
|
-
const quiet = options
|
|
6545
|
+
const quiet = quietOf(options);
|
|
7137
6546
|
requireInteger(keep, { min: 0, what: "keep" });
|
|
7138
|
-
const record = await deps.
|
|
6547
|
+
const record = await readHeldRow(deps, ref, options.incarnation);
|
|
7139
6548
|
if (record === null) return { deleted: [] };
|
|
7140
6549
|
const current = record.currentGen;
|
|
7141
6550
|
const gens = await generationsOf(deps.storage, ref);
|
|
7142
|
-
const after = await deps.
|
|
6551
|
+
const after = await readHeldRow(deps, ref, options.incarnation);
|
|
7143
6552
|
if (after === null) {
|
|
7144
6553
|
if (quiet) return { deleted: [] };
|
|
7145
6554
|
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
@@ -7167,11 +6576,10 @@ async function listingPass(ref, deps, options) {
|
|
|
7167
6576
|
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
7168
6577
|
const now = protect === void 0 ? void 0 : windowOf(still);
|
|
7169
6578
|
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
7170
|
-
|
|
6579
|
+
const asked = isAskedRow(options.incarnation, still);
|
|
6580
|
+
if (!ok || !unnamed || !asked) {
|
|
7171
6581
|
if (quiet) return "stop";
|
|
7172
|
-
throw
|
|
7173
|
-
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
7174
|
-
);
|
|
6582
|
+
throw asked ? changedWhileCollecting(ref) : refusedOtherRow(ref);
|
|
7175
6583
|
}
|
|
7176
6584
|
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
7177
6585
|
if (options.skipKnown === true && record.status !== "destroyed" && usableKnownGens(still)?.some((e) => e.generation === generation) === true) {
|
|
@@ -7209,7 +6617,7 @@ async function listingPass(ref, deps, options) {
|
|
|
7209
6617
|
leasedSpared.add(generation);
|
|
7210
6618
|
continue;
|
|
7211
6619
|
}
|
|
7212
|
-
await deps
|
|
6620
|
+
await deleteGeneration(deps, ref, generation);
|
|
7213
6621
|
done.push(generation);
|
|
7214
6622
|
watch.deleted(generation);
|
|
7215
6623
|
}
|
|
@@ -7292,7 +6700,7 @@ async function deleteEvicted(ref, deps, options) {
|
|
|
7292
6700
|
continue;
|
|
7293
6701
|
}
|
|
7294
6702
|
held ??= still;
|
|
7295
|
-
await deps
|
|
6703
|
+
await deleteGeneration(deps, ref, generation);
|
|
7296
6704
|
deleted.push(generation);
|
|
7297
6705
|
watch.deleted(generation);
|
|
7298
6706
|
}
|
|
@@ -7351,17 +6759,18 @@ function guardRefusal(cardinality, before, guard, allowEmpty) {
|
|
|
7351
6759
|
return void 0;
|
|
7352
6760
|
}
|
|
7353
6761
|
async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
7354
|
-
validateUserRef(ref);
|
|
6762
|
+
validateUserRef(ref, "judgeLoad");
|
|
7355
6763
|
requireInteger(cardinality, { min: 0, what: "judgeLoad: cardinality" });
|
|
7356
6764
|
const guard = checkedGuard(options.guard, "");
|
|
7357
6765
|
assertRegistryCanWrite(deps.registry, "load");
|
|
6766
|
+
checkRequireEncryption(deps.requireEncryption);
|
|
7358
6767
|
if (deps.requireEncryption === true && deps.keystore === void 0) {
|
|
7359
6768
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
7360
6769
|
}
|
|
7361
6770
|
const row = await deps.registry.get(ref);
|
|
7362
6771
|
if (row?.status === "destroyed") {
|
|
7363
6772
|
throw new ValidationError(
|
|
7364
|
-
`segment "${ref.segment}" is destroyed
|
|
6773
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to write: no generation resolves through a destroyed row; use a new segment`
|
|
7365
6774
|
);
|
|
7366
6775
|
}
|
|
7367
6776
|
const cleartext = row?.wrappedDeks === void 0 || row.wrappedDeks.length === 0;
|
|
@@ -7396,6 +6805,9 @@ async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
|
7396
6805
|
function sizeNeeded(guard, allowEmpty) {
|
|
7397
6806
|
return guard?.minRetained !== void 0 || guard?.maxGrowth !== void 0 || allowEmpty !== true;
|
|
7398
6807
|
}
|
|
6808
|
+
function lostNumberRace(err) {
|
|
6809
|
+
return isWriteConflictError(err) && !isSourceError(err);
|
|
6810
|
+
}
|
|
7399
6811
|
function settled(promise) {
|
|
7400
6812
|
promise.catch(() => void 0);
|
|
7401
6813
|
return promise;
|
|
@@ -7433,7 +6845,12 @@ async function currentCardinality(ref, deps, record, unwrapped) {
|
|
|
7433
6845
|
return { cardinality: reader.count(), fromSummary: false };
|
|
7434
6846
|
}
|
|
7435
6847
|
function loadSegment(ref, input, deps, options = {}) {
|
|
7436
|
-
return runLoad(
|
|
6848
|
+
return runLoad(
|
|
6849
|
+
ref,
|
|
6850
|
+
(codec) => sortedLoadInput(prepareLoadInput(input, codec), options.sorted),
|
|
6851
|
+
deps,
|
|
6852
|
+
options
|
|
6853
|
+
);
|
|
7437
6854
|
}
|
|
7438
6855
|
function startUnwrap(row, keystore) {
|
|
7439
6856
|
const wrappedKeys = row?.wrappedDeks;
|
|
@@ -7448,6 +6865,11 @@ function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
|
7448
6865
|
"loadSegmentChunks takes an async iterable of { chunkKey, bitmap }"
|
|
7449
6866
|
);
|
|
7450
6867
|
}
|
|
6868
|
+
if (wantsSorted(options.sorted)) {
|
|
6869
|
+
throw new ValidationError(
|
|
6870
|
+
"loadSegmentChunks takes no sorted: true: its chunks ascend by key already, and are written as they are"
|
|
6871
|
+
);
|
|
6872
|
+
}
|
|
7451
6873
|
return new ChunkLoadInput(chunks);
|
|
7452
6874
|
},
|
|
7453
6875
|
deps,
|
|
@@ -7455,11 +6877,11 @@ function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
|
7455
6877
|
);
|
|
7456
6878
|
}
|
|
7457
6879
|
async function runLoad(ref, prepare, deps, options) {
|
|
7458
|
-
validateUserRef(ref);
|
|
6880
|
+
validateUserRef(ref, "load");
|
|
7459
6881
|
checkedAuditSink(options.audit, "load");
|
|
7460
6882
|
const codec = requireCodec(deps.codec, "loadSegment");
|
|
7461
6883
|
const entropy = requireEntropy(deps.entropy, "a load");
|
|
7462
|
-
const keep = options.keep
|
|
6884
|
+
const keep = options.keep === void 0 ? 1 : options.keep;
|
|
7463
6885
|
requireInteger(keep, { min: 0, what: "keep" });
|
|
7464
6886
|
const guard = checkedGuard(options.guard, "");
|
|
7465
6887
|
const metadata = copiedMetadata(options.metadata, (message) => {
|
|
@@ -7496,7 +6918,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7496
6918
|
}
|
|
7497
6919
|
);
|
|
7498
6920
|
} catch (err) {
|
|
7499
|
-
if (!
|
|
6921
|
+
if (!lostNumberRace(err)) throw err;
|
|
7500
6922
|
const { generation: generation2 } = await numbering;
|
|
7501
6923
|
audit.onEvent({
|
|
7502
6924
|
kind: "segment.load-refused",
|
|
@@ -7877,7 +7299,7 @@ var KeyBits = class {
|
|
|
7877
7299
|
};
|
|
7878
7300
|
|
|
7879
7301
|
// src/core/combine-ledger.ts
|
|
7880
|
-
var RESIDENT_BITMAP_OVERHEAD =
|
|
7302
|
+
var RESIDENT_BITMAP_OVERHEAD = 832;
|
|
7881
7303
|
var MAX_CHUNK_SERIALIZED = 8208;
|
|
7882
7304
|
function serializedBound(cardinality) {
|
|
7883
7305
|
return Math.min(2 * cardinality + 16, MAX_CHUNK_SERIALIZED);
|
|
@@ -7885,7 +7307,7 @@ function serializedBound(cardinality) {
|
|
|
7885
7307
|
var MAX_RESIDENT_DATA = 8800;
|
|
7886
7308
|
function residentBytes(serialized) {
|
|
7887
7309
|
return Math.ceil(
|
|
7888
|
-
RESIDENT_BITMAP_OVERHEAD + Math.min(1.6 * serialized, serialized +
|
|
7310
|
+
RESIDENT_BITMAP_OVERHEAD + Math.min(1.6 * serialized, serialized + 2200, MAX_RESIDENT_DATA)
|
|
7889
7311
|
);
|
|
7890
7312
|
}
|
|
7891
7313
|
function residentBound(cardinality) {
|
|
@@ -8405,9 +7827,13 @@ function compileCombineMany(request) {
|
|
|
8405
7827
|
max: MAX_BATCH_PARALLELISM,
|
|
8406
7828
|
what: "publishConcurrency"
|
|
8407
7829
|
});
|
|
8408
|
-
requireInteger(request.concurrency, {
|
|
8409
|
-
|
|
8410
|
-
|
|
7830
|
+
requireInteger(request.concurrency, {
|
|
7831
|
+
min: 1,
|
|
7832
|
+
max: MAX_BATCH_PARALLELISM,
|
|
7833
|
+
what: "materializeMany: concurrency"
|
|
7834
|
+
});
|
|
7835
|
+
assertIdBound("after", request.after, "materializeMany");
|
|
7836
|
+
assertIdBound("through", request.through, "materializeMany");
|
|
8411
7837
|
const index = /* @__PURE__ */ new Map();
|
|
8412
7838
|
operands.forEach((op, i) => index.set(op.name, i));
|
|
8413
7839
|
const feed = request.feed === void 0 ? void 0 : checkFeed(request.feed, new Set(index.keys()));
|
|
@@ -8494,7 +7920,7 @@ var Gate = class {
|
|
|
8494
7920
|
return this.peak;
|
|
8495
7921
|
}
|
|
8496
7922
|
async run(task) {
|
|
8497
|
-
if (this.active >= this.limit) await new Promise((
|
|
7923
|
+
if (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
8498
7924
|
else this.active++;
|
|
8499
7925
|
if (this.active > this.peak) this.peak = this.active;
|
|
8500
7926
|
try {
|
|
@@ -8679,7 +8105,7 @@ var Run = class {
|
|
|
8679
8105
|
running = [];
|
|
8680
8106
|
bounds;
|
|
8681
8107
|
async execute() {
|
|
8682
|
-
this.window = windowOf2(this.req);
|
|
8108
|
+
this.window = windowOf2(this.req, "materializeMany");
|
|
8683
8109
|
if (this.checkedFeed !== void 0) this.feed = this.openFeed(this.checkedFeed);
|
|
8684
8110
|
await this.readIndexes();
|
|
8685
8111
|
await this.refuseAbsentOperands();
|
|
@@ -8867,9 +8293,9 @@ var Run = class {
|
|
|
8867
8293
|
}
|
|
8868
8294
|
const bounds = this.boundsList();
|
|
8869
8295
|
for (const o of this.outputs) {
|
|
8870
|
-
const
|
|
8871
|
-
if (
|
|
8872
|
-
this.fail(o,
|
|
8296
|
+
const failed2 = o.compiled.operands.map((i) => this.operands[i]).find((op) => op.error);
|
|
8297
|
+
if (failed2?.error !== void 0) {
|
|
8298
|
+
this.fail(o, failed2.error.error);
|
|
8873
8299
|
continue;
|
|
8874
8300
|
}
|
|
8875
8301
|
if (this.window === "empty") {
|
|
@@ -9403,7 +8829,7 @@ var Run = class {
|
|
|
9403
8829
|
const wake = () => {
|
|
9404
8830
|
const now = waiters;
|
|
9405
8831
|
waiters = [];
|
|
9406
|
-
for (const
|
|
8832
|
+
for (const resolve of now) resolve();
|
|
9407
8833
|
};
|
|
9408
8834
|
const worker = async () => {
|
|
9409
8835
|
for (; ; ) {
|
|
@@ -9422,7 +8848,7 @@ var Run = class {
|
|
|
9422
8848
|
const transient = o.serialized;
|
|
9423
8849
|
let admitted = this.ledger.tryCharge(transient);
|
|
9424
8850
|
while (!admitted && inFlight > 0) {
|
|
9425
|
-
await new Promise((
|
|
8851
|
+
await new Promise((resolve) => waiters.push(resolve));
|
|
9426
8852
|
admitted = this.ledger.tryCharge(transient);
|
|
9427
8853
|
}
|
|
9428
8854
|
if (!admitted) {
|
|
@@ -9692,9 +9118,9 @@ function evaluate(node, kc) {
|
|
|
9692
9118
|
|
|
9693
9119
|
// src/core/rollback.ts
|
|
9694
9120
|
async function listGenerations(ref, deps) {
|
|
9695
|
-
validateUserRef(ref);
|
|
9121
|
+
validateUserRef(ref, "generations");
|
|
9696
9122
|
const record = await deps.registry.get(ref);
|
|
9697
|
-
const current = record?.currentGen ?? null;
|
|
9123
|
+
const current = record?.status === "destroyed" ? null : record?.currentGen ?? null;
|
|
9698
9124
|
const seen = await generationsOf(deps.storage, ref);
|
|
9699
9125
|
const described = record === null ? void 0 : await currentDescription(ref, record, deps);
|
|
9700
9126
|
const known = new Set((usableKnownGens(record) ?? []).map((e) => e.generation));
|
|
@@ -9718,7 +9144,7 @@ async function currentDescription(ref, record, deps) {
|
|
|
9718
9144
|
return metadata === void 0 || Object.keys(metadata).length === 0 ? { cardinality: usable.cardinality } : { cardinality: usable.cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
9719
9145
|
}
|
|
9720
9146
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
9721
|
-
validateUserRef(ref);
|
|
9147
|
+
validateUserRef(ref, "rollback");
|
|
9722
9148
|
checkedAuditSink(options.audit, "rollback");
|
|
9723
9149
|
requireInteger(toGeneration, { min: 0, what: "rollback: generation" });
|
|
9724
9150
|
const record = await deps.registry.get(ref);
|
|
@@ -9727,7 +9153,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9727
9153
|
}
|
|
9728
9154
|
if (record.status === "destroyed") {
|
|
9729
9155
|
throw new ValidationError(
|
|
9730
|
-
`rollback: segment "${ref.segment}" is destroyed
|
|
9156
|
+
`rollback: segment "${ref.segment}" is destroyed \u2014 no generation resolves through a destroyed row`
|
|
9731
9157
|
);
|
|
9732
9158
|
}
|
|
9733
9159
|
const known = usableKnownGens(record);
|
|
@@ -9800,43 +9226,13 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9800
9226
|
footerCrc: footer.footerCrc
|
|
9801
9227
|
});
|
|
9802
9228
|
const churn = leaseChurn(deps);
|
|
9803
|
-
|
|
9804
|
-
|
|
9805
|
-
|
|
9806
|
-
|
|
9807
|
-
|
|
9808
|
-
|
|
9809
|
-
|
|
9810
|
-
...knownAfter === void 0 ? {} : { knownGens: knownAfter }
|
|
9811
|
-
});
|
|
9812
|
-
swapped = { pointerId: token, token };
|
|
9813
|
-
break;
|
|
9814
|
-
} catch (err) {
|
|
9815
|
-
if (isTransientError(err)) {
|
|
9816
|
-
const landed = await swapLanded(
|
|
9817
|
-
ref,
|
|
9818
|
-
record,
|
|
9819
|
-
{ toGeneration, summary, known: knownAfter },
|
|
9820
|
-
deps.registry,
|
|
9821
|
-
err
|
|
9822
|
-
);
|
|
9823
|
-
swapped = { pointerId: landed.pointerId, token: landed.token };
|
|
9824
|
-
break;
|
|
9825
|
-
}
|
|
9826
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9827
|
-
const seen = await deps.registry.get(ref);
|
|
9828
|
-
const settled2 = await churn.settle(record, seen, () => deps.registry.get(ref));
|
|
9829
|
-
const now = settled2?.row ?? null;
|
|
9830
|
-
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now)) {
|
|
9831
|
-
const lost = new WriteConflictError(
|
|
9832
|
-
"rollback: the segment's row changed while the rollback ran and the pointer was not moved; check store.generations(ref) and run it again"
|
|
9833
|
-
);
|
|
9834
|
-
lost.cause = err;
|
|
9835
|
-
throw lost;
|
|
9836
|
-
}
|
|
9837
|
-
against = now.token;
|
|
9838
|
-
}
|
|
9839
|
-
}
|
|
9229
|
+
const swapped = await swapPointer(
|
|
9230
|
+
ref,
|
|
9231
|
+
record,
|
|
9232
|
+
{ toGeneration, summary, known: knownAfter },
|
|
9233
|
+
deps,
|
|
9234
|
+
churn
|
|
9235
|
+
);
|
|
9840
9236
|
const emitMove = () => {
|
|
9841
9237
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9842
9238
|
kind: "segment.rollback",
|
|
@@ -9862,43 +9258,9 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9862
9258
|
}
|
|
9863
9259
|
}
|
|
9864
9260
|
if (!stillThere || replaced) {
|
|
9865
|
-
|
|
9866
|
-
let undoAgainst = swapped.token;
|
|
9867
|
-
for (; ; ) {
|
|
9868
|
-
try {
|
|
9869
|
-
const restored = describingKnown(record, toGeneration)?.filter(
|
|
9870
|
-
(e) => e.generation !== toGeneration || e.generation === record.currentGen
|
|
9871
|
-
);
|
|
9872
|
-
await deps.registry.compareAndSwap(ref, undoAgainst, {
|
|
9873
|
-
currentGen: record.currentGen,
|
|
9874
|
-
summary: describingSummary(record),
|
|
9875
|
-
...restored === void 0 ? {} : { knownGens: restored }
|
|
9876
|
-
});
|
|
9877
|
-
undone = true;
|
|
9878
|
-
break;
|
|
9879
|
-
} catch (err) {
|
|
9880
|
-
if (!isWriteConflictError(err)) break;
|
|
9881
|
-
let now;
|
|
9882
|
-
try {
|
|
9883
|
-
const seen = await deps.registry.get(ref);
|
|
9884
|
-
if (seen === null || seen.currentGen !== toGeneration) break;
|
|
9885
|
-
const settled2 = await churn.settle(
|
|
9886
|
-
record,
|
|
9887
|
-
seen,
|
|
9888
|
-
() => deps.registry.get(ref),
|
|
9889
|
-
SWAP_MOVES
|
|
9890
|
-
);
|
|
9891
|
-
now = settled2?.row ?? null;
|
|
9892
|
-
} catch {
|
|
9893
|
-
break;
|
|
9894
|
-
}
|
|
9895
|
-
if (now === null || now.currentGen !== toGeneration || now.pointerId !== swapped.pointerId || !onlyLeasesDiffer(record, now, SWAP_MOVES))
|
|
9896
|
-
break;
|
|
9897
|
-
undoAgainst = now.token;
|
|
9898
|
-
}
|
|
9899
|
-
}
|
|
9261
|
+
const undone = await moveBack(ref, record, toGeneration, swapped, deps.registry, churn, deps);
|
|
9900
9262
|
throw new NotFoundError(
|
|
9901
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (
|
|
9263
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (the row could not show whether the move back landed). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
9902
9264
|
);
|
|
9903
9265
|
}
|
|
9904
9266
|
emitMove();
|
|
@@ -9951,13 +9313,113 @@ function keyNotHeld(err) {
|
|
|
9951
9313
|
function keyServiceUnanswered(err) {
|
|
9952
9314
|
return isTransientError(err) || err instanceof Error && err.name === "TimeoutError";
|
|
9953
9315
|
}
|
|
9954
|
-
async function
|
|
9955
|
-
let
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
if (
|
|
9960
|
-
|
|
9316
|
+
async function moveBack(ref, record, toGeneration, swapped, registry, churn, waits) {
|
|
9317
|
+
let undone = false;
|
|
9318
|
+
let undoAgainst = swapped.token;
|
|
9319
|
+
const undo = writeTokens(registry);
|
|
9320
|
+
const undoLanded = (row) => {
|
|
9321
|
+
if (row === null || undo.sent === void 0) return false;
|
|
9322
|
+
if (record.currentGen !== null) return pointerSetBy(row, record, record.currentGen, undo.sent);
|
|
9323
|
+
return row.currentGen === null && row.status !== "destroyed" && sameIncarnation(record, row) && undo.sent.has(row.pointerId);
|
|
9324
|
+
};
|
|
9325
|
+
for (; ; ) {
|
|
9326
|
+
try {
|
|
9327
|
+
const restored = describingKnown(record, toGeneration)?.filter(
|
|
9328
|
+
(e) => e.generation !== toGeneration || e.generation === record.currentGen
|
|
9329
|
+
);
|
|
9330
|
+
await registry.compareAndSwap(
|
|
9331
|
+
ref,
|
|
9332
|
+
undoAgainst,
|
|
9333
|
+
{
|
|
9334
|
+
currentGen: record.currentGen,
|
|
9335
|
+
summary: describingSummary(record),
|
|
9336
|
+
...restored === void 0 ? {} : { knownGens: restored }
|
|
9337
|
+
},
|
|
9338
|
+
undo.options
|
|
9339
|
+
);
|
|
9340
|
+
undone = true;
|
|
9341
|
+
break;
|
|
9342
|
+
} catch (err) {
|
|
9343
|
+
if (isTransientError(err)) {
|
|
9344
|
+
try {
|
|
9345
|
+
const row = await readBack(() => registry.get(ref), err, waits);
|
|
9346
|
+
if (undoLanded(row)) undone = true;
|
|
9347
|
+
} catch {
|
|
9348
|
+
}
|
|
9349
|
+
break;
|
|
9350
|
+
}
|
|
9351
|
+
if (!isWriteConflictError(err)) break;
|
|
9352
|
+
let now;
|
|
9353
|
+
try {
|
|
9354
|
+
const seen = await readBack(() => registry.get(ref), err, waits);
|
|
9355
|
+
if (undoLanded(seen)) {
|
|
9356
|
+
undone = true;
|
|
9357
|
+
break;
|
|
9358
|
+
}
|
|
9359
|
+
if (seen === null || seen.currentGen !== toGeneration) break;
|
|
9360
|
+
const settled2 = await churn.settle(record, seen, () => registry.get(ref), SWAP_MOVES);
|
|
9361
|
+
now = settled2?.row ?? null;
|
|
9362
|
+
} catch {
|
|
9363
|
+
break;
|
|
9364
|
+
}
|
|
9365
|
+
if (now === null || now.currentGen !== toGeneration || now.pointerId !== swapped.pointerId || !onlyLeasesDiffer(record, now, SWAP_MOVES))
|
|
9366
|
+
break;
|
|
9367
|
+
undoAgainst = now.token;
|
|
9368
|
+
}
|
|
9369
|
+
}
|
|
9370
|
+
return undone;
|
|
9371
|
+
}
|
|
9372
|
+
function unconfirmedSwap(err, cause = err) {
|
|
9373
|
+
const lost = new WriteConflictError(
|
|
9374
|
+
"rollback: the segment's row changed while the rollback ran, and this rollback could not confirm that its swap is the one the row shows; check store.generations(ref) and run it again"
|
|
9375
|
+
);
|
|
9376
|
+
lost.cause = cause;
|
|
9377
|
+
return lost;
|
|
9378
|
+
}
|
|
9379
|
+
async function swapPointer(ref, record, swap, deps, churn) {
|
|
9380
|
+
const { registry } = deps;
|
|
9381
|
+
let against = record.token;
|
|
9382
|
+
const { sent, options: writeOptions2 } = writeTokens(registry);
|
|
9383
|
+
for (; ; ) {
|
|
9384
|
+
try {
|
|
9385
|
+
const { token } = await registry.compareAndSwap(
|
|
9386
|
+
ref,
|
|
9387
|
+
against,
|
|
9388
|
+
{
|
|
9389
|
+
currentGen: swap.toGeneration,
|
|
9390
|
+
summary: swap.summary,
|
|
9391
|
+
...swap.known === void 0 ? {} : { knownGens: swap.known }
|
|
9392
|
+
},
|
|
9393
|
+
writeOptions2
|
|
9394
|
+
);
|
|
9395
|
+
return { pointerId: token, token };
|
|
9396
|
+
} catch (err) {
|
|
9397
|
+
if (isTransientError(err)) return swapLanded(ref, record, swap, registry, err, sent, deps);
|
|
9398
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9399
|
+
let seen;
|
|
9400
|
+
try {
|
|
9401
|
+
seen = await readBack(() => registry.get(ref), err, deps);
|
|
9402
|
+
} catch (readErr) {
|
|
9403
|
+
if (readErr === err) throw unconfirmedSwap(err, err.cause);
|
|
9404
|
+
throw readErr;
|
|
9405
|
+
}
|
|
9406
|
+
if (seen !== null && sent !== void 0 && pointerSetBy(seen, record, swap.toGeneration, sent)) {
|
|
9407
|
+
return seen;
|
|
9408
|
+
}
|
|
9409
|
+
const settled2 = await churn.settle(record, seen, () => registry.get(ref));
|
|
9410
|
+
const now = settled2?.row ?? null;
|
|
9411
|
+
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now)) {
|
|
9412
|
+
throw unconfirmedSwap(err);
|
|
9413
|
+
}
|
|
9414
|
+
against = now.token;
|
|
9415
|
+
}
|
|
9416
|
+
}
|
|
9417
|
+
}
|
|
9418
|
+
async function swapLanded(ref, record, swap, registry, unanswered, sent, waits) {
|
|
9419
|
+
const now = await readBack(() => registry.get(ref), unanswered, waits);
|
|
9420
|
+
if (sent !== void 0) {
|
|
9421
|
+
if (now === null || !pointerSetBy(now, record, swap.toGeneration, sent)) throw unanswered;
|
|
9422
|
+
return now;
|
|
9961
9423
|
}
|
|
9962
9424
|
const left = {
|
|
9963
9425
|
...record,
|
|
@@ -10006,7 +9468,7 @@ function recordCollect(ref, result, pass, options) {
|
|
|
10006
9468
|
return result;
|
|
10007
9469
|
}
|
|
10008
9470
|
async function eraseOnce(ref, id, deps, options, seen) {
|
|
10009
|
-
validateUserRef(ref);
|
|
9471
|
+
validateUserRef(ref, "eraseIdFromSegment");
|
|
10010
9472
|
checkedAuditSink(options.audit, "eraseIdFromSegment");
|
|
10011
9473
|
const { chunkKey, remainder } = splitId(id);
|
|
10012
9474
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
@@ -10044,6 +9506,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
10044
9506
|
read,
|
|
10045
9507
|
sealed: /* @__PURE__ */ new Set(),
|
|
10046
9508
|
versions: /* @__PURE__ */ new Map(),
|
|
9509
|
+
fromFingerprint: void 0,
|
|
10047
9510
|
premise: record,
|
|
10048
9511
|
premiseToken: fromToken,
|
|
10049
9512
|
fenceLost: 0,
|
|
@@ -10120,6 +9583,7 @@ async function holds(c, generation) {
|
|
|
10120
9583
|
reader = await read(() => openGenerationReader(deps.storage, key, void 0));
|
|
10121
9584
|
}
|
|
10122
9585
|
versions.set(generation, objectVersionOf(reader));
|
|
9586
|
+
sealed.delete(generation);
|
|
10123
9587
|
let bytes;
|
|
10124
9588
|
try {
|
|
10125
9589
|
bytes = await read(() => reader.getChunk(chunkKey));
|
|
@@ -10163,11 +9627,32 @@ async function holdsEach(c, generations, { stopAtHolder }) {
|
|
|
10163
9627
|
}
|
|
10164
9628
|
async function holderLeft(c, clean) {
|
|
10165
9629
|
const { ref, deps } = c;
|
|
10166
|
-
const
|
|
9630
|
+
const listed2 = (await generationsOf(deps.storage, ref)).reverse();
|
|
9631
|
+
const own = (generation) => clean.has(generation) ? ownFingerprint(c, generation) : void 0;
|
|
9632
|
+
const known = listed2.some((generation) => own(generation) !== void 0) ? await ledgerAfterListing(c) : void 0;
|
|
9633
|
+
const vouched = (generation) => {
|
|
9634
|
+
const fingerprint = own(generation);
|
|
9635
|
+
if (fingerprint === void 0 || known === void 0) return false;
|
|
9636
|
+
const footerCrc = footerCrcOf(fingerprint);
|
|
9637
|
+
return known.some((e) => e.generation === generation && e.footerCrc === footerCrc);
|
|
9638
|
+
};
|
|
9639
|
+
const toRead = listed2.filter((generation) => !vouched(generation));
|
|
10167
9640
|
return (await holdsEach(c, toRead, { stopAtHolder: true })).find(
|
|
10168
9641
|
(outcome) => outcome.held === true
|
|
10169
9642
|
)?.generation;
|
|
10170
9643
|
}
|
|
9644
|
+
function ownFingerprint(c, generation) {
|
|
9645
|
+
if (generation === c.from) return c.fromFingerprint;
|
|
9646
|
+
if (generation === c.rewrote?.generation) return c.rewrote.fingerprint;
|
|
9647
|
+
return void 0;
|
|
9648
|
+
}
|
|
9649
|
+
async function ledgerAfterListing(c) {
|
|
9650
|
+
try {
|
|
9651
|
+
return usableKnownGens(await c.deps.registry.get(c.ref));
|
|
9652
|
+
} catch {
|
|
9653
|
+
return void 0;
|
|
9654
|
+
}
|
|
9655
|
+
}
|
|
10171
9656
|
function cannotRemove(c, generation) {
|
|
10172
9657
|
const { ref, sealed } = c;
|
|
10173
9658
|
return new WriteConflictError(
|
|
@@ -10224,7 +9709,8 @@ async function fence(c, plan) {
|
|
|
10224
9709
|
const ledger = usableKnownGens(row);
|
|
10225
9710
|
const doomed = doomedGenerations(c, plan, byName, ledger, pointer);
|
|
10226
9711
|
const named = ledger !== void 0 && ledger.some((e) => doomed.has(e.generation));
|
|
10227
|
-
const
|
|
9712
|
+
const sweepOwes = plan.sweeps && !named && row.pointerId === record.pointerId && sameKnownGens(row.knownGens, record.knownGens);
|
|
9713
|
+
const renew = byName.length > 0 || sweepOwes;
|
|
10228
9714
|
const renewedAfter = held && row.pointerId !== first.pointerId;
|
|
10229
9715
|
if (!named && (!renew || renewedAfter)) {
|
|
10230
9716
|
c.premise = row;
|
|
@@ -10301,7 +9787,13 @@ async function deleteFenced(c, holders, fenced) {
|
|
|
10301
9787
|
}
|
|
10302
9788
|
if (moved !== null) break;
|
|
10303
9789
|
try {
|
|
10304
|
-
await
|
|
9790
|
+
await deleteSettled(
|
|
9791
|
+
deps.storage,
|
|
9792
|
+
{ ...base, generation },
|
|
9793
|
+
{ ifVersion: versions.get(generation) },
|
|
9794
|
+
c.read,
|
|
9795
|
+
{ clock: deps.clock, rng: deps.rng }
|
|
9796
|
+
);
|
|
10305
9797
|
} catch (err) {
|
|
10306
9798
|
if (!isWriteConflictError(err)) throw err;
|
|
10307
9799
|
moved = rowVerdict(c, await deps.registry.get(ref)) ?? "superseded";
|
|
@@ -10365,14 +9857,35 @@ async function unpublished(c) {
|
|
|
10365
9857
|
}
|
|
10366
9858
|
throw cannotRemove(c, left);
|
|
10367
9859
|
}
|
|
9860
|
+
async function holderUnderTombstone(c) {
|
|
9861
|
+
const newestFirst = (await c.read(() => generationsOf(c.deps.storage, c.ref))).reverse();
|
|
9862
|
+
return (await holdsEach(c, newestFirst, { stopAtHolder: true })).find((h) => h.held === true);
|
|
9863
|
+
}
|
|
9864
|
+
async function findIdUnderTombstone(ref, id, deps) {
|
|
9865
|
+
validateUserRef(ref, "findIdUnderTombstone");
|
|
9866
|
+
const { chunkKey, remainder } = splitId(id);
|
|
9867
|
+
const codec = requireCodec(deps.codec, "findIdUnderTombstone");
|
|
9868
|
+
const c = {
|
|
9869
|
+
ref,
|
|
9870
|
+
deps,
|
|
9871
|
+
chunkKey,
|
|
9872
|
+
remainder,
|
|
9873
|
+
codec,
|
|
9874
|
+
maxBytes: deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES,
|
|
9875
|
+
base: { segment: ref.segment, namespace: ref.namespace },
|
|
9876
|
+
tombstoned: true,
|
|
9877
|
+
aead: void 0,
|
|
9878
|
+
read: (op) => retryRead(op, deps.readRetry),
|
|
9879
|
+
sealed: /* @__PURE__ */ new Set(),
|
|
9880
|
+
versions: /* @__PURE__ */ new Map()
|
|
9881
|
+
};
|
|
9882
|
+
return (await holderUnderTombstone(c))?.generation ?? null;
|
|
9883
|
+
}
|
|
10368
9884
|
async function underTombstone(c) {
|
|
10369
9885
|
const { ref, deps, base } = c;
|
|
10370
|
-
const
|
|
10371
|
-
const holder = (await holdsEach(c, newestFirst, { stopAtHolder: true })).find(
|
|
10372
|
-
(h) => h.held === true
|
|
10373
|
-
);
|
|
9886
|
+
const holder = await holderUnderTombstone(c);
|
|
10374
9887
|
if (holder === void 0) return { ...base, erased: false, reason: "destroyed", collected: [] };
|
|
10375
|
-
const collected = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
9888
|
+
const collected = await gcOrphanGenerations(ref, deps, { keep: 0, incarnation: c.record });
|
|
10376
9889
|
const left = await holderLeft(c, /* @__PURE__ */ new Set());
|
|
10377
9890
|
if (left !== void 0) throw cannotRemove(c, left);
|
|
10378
9891
|
return collectedAll(c, holder.generation, collected);
|
|
@@ -10488,6 +10001,7 @@ async function stage(c) {
|
|
|
10488
10001
|
const reader = await read(
|
|
10489
10002
|
() => openGenerationReader(deps.storage, fromKey, cryptoAt(c, from), {}, described?.fingerprint)
|
|
10490
10003
|
);
|
|
10004
|
+
c.fromFingerprint = reader.fingerprint;
|
|
10491
10005
|
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
10492
10006
|
if (bytes === null) return await notInCurrent(c);
|
|
10493
10007
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
@@ -10498,17 +10012,29 @@ async function stage(c) {
|
|
|
10498
10012
|
const generation = await nextGeneration(ref, deps);
|
|
10499
10013
|
const key = { ...base, generation };
|
|
10500
10014
|
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
10501
|
-
|
|
10502
|
-
|
|
10503
|
-
|
|
10504
|
-
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
10015
|
+
let tally;
|
|
10016
|
+
try {
|
|
10017
|
+
tally = await writeCrbmGenerationStream(
|
|
10018
|
+
deps.storage,
|
|
10019
|
+
key,
|
|
10020
|
+
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
10021
|
+
{
|
|
10022
|
+
crypto: cryptoAt(c, generation),
|
|
10023
|
+
clock: deps.clock,
|
|
10024
|
+
metadata,
|
|
10025
|
+
entropy: deps.entropy
|
|
10026
|
+
}
|
|
10027
|
+
);
|
|
10028
|
+
} catch (err) {
|
|
10029
|
+
if (!isWriteConflictError(err)) throw err;
|
|
10030
|
+
let now;
|
|
10031
|
+
try {
|
|
10032
|
+
now = await deps.registry.get(ref);
|
|
10033
|
+
} catch {
|
|
10034
|
+
throw err;
|
|
10510
10035
|
}
|
|
10511
|
-
|
|
10036
|
+
return refusedResult(c, rowVerdict(c, now) ?? "superseded");
|
|
10037
|
+
}
|
|
10512
10038
|
written = generation;
|
|
10513
10039
|
c.rewrote = { generation, fingerprint: tally.fingerprint };
|
|
10514
10040
|
const beforeVerify = await deps.registry.get(ref);
|
|
@@ -10542,6 +10068,34 @@ async function stage(c) {
|
|
|
10542
10068
|
throw err;
|
|
10543
10069
|
}
|
|
10544
10070
|
}
|
|
10071
|
+
async function provenOwn(c, key, fingerprint) {
|
|
10072
|
+
const { deps } = c;
|
|
10073
|
+
const retry = deps.readRetry;
|
|
10074
|
+
const base = retry?.isRetryable ?? isTransientError;
|
|
10075
|
+
let fault;
|
|
10076
|
+
try {
|
|
10077
|
+
return await retryRead(
|
|
10078
|
+
async () => {
|
|
10079
|
+
try {
|
|
10080
|
+
return await provenObject(deps.storage, key, fingerprint);
|
|
10081
|
+
} catch (err) {
|
|
10082
|
+
if (!isIntegrityError(err)) fault = err;
|
|
10083
|
+
throw err;
|
|
10084
|
+
}
|
|
10085
|
+
},
|
|
10086
|
+
retry === void 0 ? void 0 : {
|
|
10087
|
+
...retry,
|
|
10088
|
+
isRetryable: (err) => isIntegrityError(err) || base(err),
|
|
10089
|
+
onRetry: (info) => {
|
|
10090
|
+
if (!isIntegrityError(info.err)) retry.onRetry?.(info);
|
|
10091
|
+
}
|
|
10092
|
+
}
|
|
10093
|
+
);
|
|
10094
|
+
} catch (err) {
|
|
10095
|
+
if (isIntegrityError(err) && fault === void 0) return null;
|
|
10096
|
+
throw fault ?? err;
|
|
10097
|
+
}
|
|
10098
|
+
}
|
|
10545
10099
|
async function discardRefused(c, own) {
|
|
10546
10100
|
const { ref, deps, base, from } = c;
|
|
10547
10101
|
const key = { ...base, generation: own.generation };
|
|
@@ -10550,17 +10104,23 @@ async function discardRefused(c, own) {
|
|
|
10550
10104
|
let row = await deps.registry.get(ref);
|
|
10551
10105
|
for (let attempts = 1; ; attempts++) {
|
|
10552
10106
|
if (row !== null && row.status === "destroyed") {
|
|
10553
|
-
await deps.storage.
|
|
10107
|
+
await deleteSettled(deps.storage, key, void 0, c.read, {
|
|
10108
|
+
clock: deps.clock,
|
|
10109
|
+
rng: deps.rng
|
|
10110
|
+
});
|
|
10554
10111
|
return row;
|
|
10555
10112
|
}
|
|
10556
10113
|
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
10557
10114
|
if (row.currentGen <= from || row.currentGen >= own.generation) return row;
|
|
10558
|
-
const proven = await
|
|
10115
|
+
const proven = await provenOwn(c, key, own.fingerprint);
|
|
10559
10116
|
if (proven === null) return row;
|
|
10560
10117
|
const renewed = await renewal.renew(row);
|
|
10561
10118
|
if ("row" in renewed) {
|
|
10562
10119
|
try {
|
|
10563
|
-
await deps.storage
|
|
10120
|
+
await deleteSettled(deps.storage, key, { ifVersion: proven.version }, c.read, {
|
|
10121
|
+
clock: deps.clock,
|
|
10122
|
+
rng: deps.rng
|
|
10123
|
+
});
|
|
10564
10124
|
} catch (err) {
|
|
10565
10125
|
if (!isWriteConflictError(err)) throw err;
|
|
10566
10126
|
}
|
|
@@ -10640,17 +10200,20 @@ async function finishRewrite(c, staged) {
|
|
|
10640
10200
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
10641
10201
|
}
|
|
10642
10202
|
async function resolveDek(ref, deps, record, row) {
|
|
10203
|
+
checkRequireEncryption(deps.requireEncryption);
|
|
10643
10204
|
if (row.tombstoned) return void 0;
|
|
10644
10205
|
if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
10645
10206
|
if (deps.keystore === void 0) {
|
|
10646
10207
|
throw new KeyUnavailableError(
|
|
10647
|
-
`segment "${ref.segment}" is encrypted
|
|
10208
|
+
`segment "${ref.segment}" is encrypted, and the erasure has no keystore to open its key (encryption.keystore on the store)`
|
|
10648
10209
|
);
|
|
10649
10210
|
}
|
|
10650
10211
|
return await deps.keystore.openDek(record.wrappedDeks);
|
|
10651
10212
|
}
|
|
10652
10213
|
if (!row.pointerless && deps.requireEncryption === true) {
|
|
10653
|
-
throw new
|
|
10214
|
+
throw new KeyUnavailableError(
|
|
10215
|
+
`requireEncryption: segment "${ref.segment}" is cleartext but encryption is required`
|
|
10216
|
+
);
|
|
10654
10217
|
}
|
|
10655
10218
|
return void 0;
|
|
10656
10219
|
}
|
|
@@ -10749,116 +10312,6 @@ function assertRemaindersInRange(bitmap, chunkKey) {
|
|
|
10749
10312
|
}
|
|
10750
10313
|
}
|
|
10751
10314
|
|
|
10752
|
-
// src/drivers/crypto.ts
|
|
10753
|
-
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
10754
|
-
var UTF82 = new TextEncoder();
|
|
10755
|
-
var KEY_BYTES = 32;
|
|
10756
|
-
var NONCE_BYTES = 12;
|
|
10757
|
-
var TAG_BYTES = 16;
|
|
10758
|
-
var isBytes = (v) => Object.prototype.toString.call(v) === "[object Uint8Array]";
|
|
10759
|
-
function ownKey(key) {
|
|
10760
|
-
const own = Buffer.alloc(key.length);
|
|
10761
|
-
own.set(key);
|
|
10762
|
-
return own;
|
|
10763
|
-
}
|
|
10764
|
-
var NodeAead = class {
|
|
10765
|
-
key;
|
|
10766
|
-
constructor(key) {
|
|
10767
|
-
if (!isBytes(key))
|
|
10768
|
-
throw new ValidationError("AEAD key must be a Uint8Array of key bytes, not text");
|
|
10769
|
-
if (key.length !== KEY_BYTES)
|
|
10770
|
-
throw new ValidationError(`AEAD key must be ${KEY_BYTES} bytes, got ${key.length}`);
|
|
10771
|
-
this.key = ownKey(key);
|
|
10772
|
-
}
|
|
10773
|
-
seal(plaintext, aad) {
|
|
10774
|
-
const nonce = randomBytes(NONCE_BYTES);
|
|
10775
|
-
const cipher = createCipheriv("aes-256-gcm", this.key, nonce);
|
|
10776
|
-
cipher.setAAD(aad);
|
|
10777
|
-
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
10778
|
-
return { nonce, ciphertext, tag: cipher.getAuthTag() };
|
|
10779
|
-
}
|
|
10780
|
-
open(sealed, aad) {
|
|
10781
|
-
if (sealed.nonce.length !== NONCE_BYTES || sealed.tag.length !== TAG_BYTES)
|
|
10782
|
-
throw new IntegrityError("AEAD nonce/tag have the wrong length");
|
|
10783
|
-
const decipher = createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
|
|
10784
|
-
decipher.setAAD(aad);
|
|
10785
|
-
decipher.setAuthTag(Buffer.from(sealed.tag));
|
|
10786
|
-
try {
|
|
10787
|
-
return Buffer.concat([decipher.update(sealed.ciphertext), decipher.final()]);
|
|
10788
|
-
} catch {
|
|
10789
|
-
throw new IntegrityError(
|
|
10790
|
-
"AEAD authentication failed (wrong key, tampered data, or wrong context)"
|
|
10791
|
-
);
|
|
10792
|
-
}
|
|
10793
|
-
}
|
|
10794
|
-
};
|
|
10795
|
-
function wrapKey(dek, kek, keyId) {
|
|
10796
|
-
const sealed = new NodeAead(kek).seal(dek, UTF82.encode(keyId));
|
|
10797
|
-
return Buffer.concat([sealed.nonce, sealed.ciphertext, sealed.tag]).toString("base64");
|
|
10798
|
-
}
|
|
10799
|
-
function unwrapKey(wrapped, kek, keyId) {
|
|
10800
|
-
const blob = Buffer.from(wrapped, "base64");
|
|
10801
|
-
if (blob.length < NONCE_BYTES + TAG_BYTES)
|
|
10802
|
-
throw new IntegrityError("wrapped DEK is too short to be valid");
|
|
10803
|
-
const nonce = blob.subarray(0, NONCE_BYTES);
|
|
10804
|
-
const tag = blob.subarray(blob.length - TAG_BYTES);
|
|
10805
|
-
const ciphertext = blob.subarray(NONCE_BYTES, blob.length - TAG_BYTES);
|
|
10806
|
-
return new NodeAead(kek).open({ nonce, ciphertext, tag }, UTF82.encode(keyId));
|
|
10807
|
-
}
|
|
10808
|
-
var InProcessKeystore = class {
|
|
10809
|
-
keys;
|
|
10810
|
-
activeKeyId;
|
|
10811
|
-
recoveryKeyId;
|
|
10812
|
-
constructor(options) {
|
|
10813
|
-
const entries = Object.entries(options.keys);
|
|
10814
|
-
if (entries.length === 0) throw new ValidationError("keystore needs at least one KEK");
|
|
10815
|
-
for (const [id, k] of entries) {
|
|
10816
|
-
if (!isBytes(k))
|
|
10817
|
-
throw new ValidationError(`KEK "${id}" must be a Uint8Array of key bytes, not text`);
|
|
10818
|
-
if (k.length !== KEY_BYTES)
|
|
10819
|
-
throw new ValidationError(`KEK "${id}" must be ${KEY_BYTES} bytes, got ${k.length}`);
|
|
10820
|
-
}
|
|
10821
|
-
this.keys = new Map(entries.map(([id, k]) => [id, ownKey(k)]));
|
|
10822
|
-
if (!this.keys.has(options.activeKeyId))
|
|
10823
|
-
throw new ValidationError(`activeKeyId "${String(options.activeKeyId)}" is not in keys`);
|
|
10824
|
-
if (options.recoveryKeyId !== void 0 && !this.keys.has(options.recoveryKeyId))
|
|
10825
|
-
throw new ValidationError(`recoveryKeyId "${String(options.recoveryKeyId)}" is not in keys`);
|
|
10826
|
-
this.activeKeyId = options.activeKeyId;
|
|
10827
|
-
this.recoveryKeyId = options.recoveryKeyId === options.activeKeyId ? void 0 : options.recoveryKeyId;
|
|
10828
|
-
}
|
|
10829
|
-
// async to satisfy the IKeystore seam (a KMS adapter awaits a network call here); the in-process path is
|
|
10830
|
-
// synchronous, but staying async keeps unwrap failures as rejections, matching a remote keystore.
|
|
10831
|
-
async createDek() {
|
|
10832
|
-
const dek = randomBytes(KEY_BYTES);
|
|
10833
|
-
const ids = this.recoveryKeyId ? [this.activeKeyId, this.recoveryKeyId] : [this.activeKeyId];
|
|
10834
|
-
const wrapped = ids.map((keyId) => ({
|
|
10835
|
-
keyId,
|
|
10836
|
-
wrapped: wrapKey(dek, this.keys.get(keyId), keyId)
|
|
10837
|
-
}));
|
|
10838
|
-
return { wrapped, aead: new NodeAead(dek) };
|
|
10839
|
-
}
|
|
10840
|
-
// async for the same reason as createDek — a synchronous unwrap failure surfaces as a promise rejection.
|
|
10841
|
-
async openDek(wrapped) {
|
|
10842
|
-
let anyKekHeld = false;
|
|
10843
|
-
let lastUnwrapError;
|
|
10844
|
-
for (const w of wrapped) {
|
|
10845
|
-
const kek = this.keys.get(w.keyId);
|
|
10846
|
-
if (kek === void 0) continue;
|
|
10847
|
-
anyKekHeld = true;
|
|
10848
|
-
try {
|
|
10849
|
-
return new NodeAead(unwrapKey(w.wrapped, kek, w.keyId));
|
|
10850
|
-
} catch (err) {
|
|
10851
|
-
lastUnwrapError = err;
|
|
10852
|
-
}
|
|
10853
|
-
}
|
|
10854
|
-
if (anyKekHeld) throw lastUnwrapError;
|
|
10855
|
-
const need = wrapped.map((w) => w.keyId).join(", ") || "(none)";
|
|
10856
|
-
throw new KeyUnavailableError(
|
|
10857
|
-
`no held KEK can unwrap this DEK \u2014 the wrappings reference [${need}]. Restore the missing KEK (or its recovery KEK), or rebuild the segment from source.`
|
|
10858
|
-
);
|
|
10859
|
-
}
|
|
10860
|
-
};
|
|
10861
|
-
|
|
10862
10315
|
// src/core/cas-retry.ts
|
|
10863
10316
|
async function casRetry(deps, ref, plan, step) {
|
|
10864
10317
|
const churn = leaseChurn(deps);
|
|
@@ -10904,6 +10357,7 @@ function validateRetentionPolicy(policy) {
|
|
|
10904
10357
|
`retention policy must be an object such as { expiresAt }; got ${policy === null ? "null" : Array.isArray(policy) ? "an array" : `a ${typeof policy}`}`
|
|
10905
10358
|
);
|
|
10906
10359
|
}
|
|
10360
|
+
refuseUnknownOptions("setRetention policy", policy, ["expiresAt"]);
|
|
10907
10361
|
const { expiresAt } = policy;
|
|
10908
10362
|
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
10909
10363
|
throw new ValidationError(
|
|
@@ -10931,7 +10385,7 @@ function readRetentionPolicy(meta) {
|
|
|
10931
10385
|
return { expiresAt: raw };
|
|
10932
10386
|
}
|
|
10933
10387
|
async function getSegmentRetention(ref, deps) {
|
|
10934
|
-
validateUserRef(ref);
|
|
10388
|
+
validateUserRef(ref, "getRetention");
|
|
10935
10389
|
const record = await deps.registry.get(ref);
|
|
10936
10390
|
if (record === null || record.status === "destroyed") return null;
|
|
10937
10391
|
return readRetentionPolicy(record.retention);
|
|
@@ -10940,20 +10394,25 @@ function readExpiresAt(record) {
|
|
|
10940
10394
|
const parsed = record === null ? null : readRetentionPolicy(record.retention);
|
|
10941
10395
|
return parsed === null || parsed === "invalid" ? void 0 : parsed.expiresAt;
|
|
10942
10396
|
}
|
|
10943
|
-
async function reindex(ref, deps, expiresAt,
|
|
10397
|
+
async function reindex(ref, deps, expiresAt, previous) {
|
|
10944
10398
|
if (!canIndex(ref)) return false;
|
|
10945
10399
|
const nextBucket = expiresAt === void 0 ? void 0 : dueBucket(expiresAt);
|
|
10946
|
-
const oldBucket = previousExpiresAt === void 0 ? void 0 : dueBucket(previousExpiresAt);
|
|
10947
10400
|
let indexed = expiresAt === void 0;
|
|
10948
10401
|
if (nextBucket !== void 0) {
|
|
10402
|
+
const pointer = dueIndexRef(nextBucket, ref);
|
|
10949
10403
|
try {
|
|
10950
|
-
await deps.registry.create(
|
|
10404
|
+
await deps.registry.create(pointer, { currentGen: null });
|
|
10951
10405
|
indexed = true;
|
|
10952
10406
|
} catch (err) {
|
|
10953
|
-
if (isWriteConflictError(err))
|
|
10407
|
+
if (isWriteConflictError(err)) {
|
|
10408
|
+
indexed = true;
|
|
10409
|
+
} else {
|
|
10410
|
+
indexed = await deps.registry.get(pointer).catch(() => null) !== null;
|
|
10411
|
+
}
|
|
10954
10412
|
}
|
|
10955
10413
|
}
|
|
10956
|
-
|
|
10414
|
+
for (const oldBucket of new Set([...previous].map(dueBucket))) {
|
|
10415
|
+
if (oldBucket === nextBucket) continue;
|
|
10957
10416
|
try {
|
|
10958
10417
|
await deps.registry.delete(dueIndexRef(oldBucket, ref));
|
|
10959
10418
|
} catch {
|
|
@@ -10961,10 +10420,38 @@ async function reindex(ref, deps, expiresAt, previousExpiresAt) {
|
|
|
10961
10420
|
}
|
|
10962
10421
|
return indexed;
|
|
10963
10422
|
}
|
|
10423
|
+
async function settledWrite(ref, deps, tokens, before, send) {
|
|
10424
|
+
try {
|
|
10425
|
+
await send();
|
|
10426
|
+
return null;
|
|
10427
|
+
} catch (err) {
|
|
10428
|
+
if (!isTransientError(err) || tokens.sent === void 0) throw err;
|
|
10429
|
+
const now = await readBack(() => deps.registry.get(ref), err, deps);
|
|
10430
|
+
if (tokens.wrote(now)) return now;
|
|
10431
|
+
if (now === null ? before === null : now.token === before?.token) throw err;
|
|
10432
|
+
const moved = new WriteConflictError(
|
|
10433
|
+
`the row of segment "${ref.segment}" changed while its retention write got no answer`
|
|
10434
|
+
);
|
|
10435
|
+
moved.cause = err;
|
|
10436
|
+
throw moved;
|
|
10437
|
+
}
|
|
10438
|
+
}
|
|
10964
10439
|
async function setSegmentRetention(ref, deps, policy) {
|
|
10965
|
-
validateUserRef(ref);
|
|
10440
|
+
validateUserRef(ref, "setRetention");
|
|
10966
10441
|
validateRetentionPolicy(policy);
|
|
10967
10442
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
10443
|
+
const created = /* @__PURE__ */ new Set();
|
|
10444
|
+
let creating = false;
|
|
10445
|
+
const tokens = writeTokens(deps.registry, (token) => {
|
|
10446
|
+
if (creating) created.add(token);
|
|
10447
|
+
});
|
|
10448
|
+
const previous = /* @__PURE__ */ new Set();
|
|
10449
|
+
const done = async (createdRow, held = policy.expiresAt) => ({
|
|
10450
|
+
...base,
|
|
10451
|
+
expiresAt: policy.expiresAt,
|
|
10452
|
+
createdRow,
|
|
10453
|
+
indexed: await reindex(ref, deps, held, previous)
|
|
10454
|
+
});
|
|
10968
10455
|
return casRetry(
|
|
10969
10456
|
deps,
|
|
10970
10457
|
ref,
|
|
@@ -10975,34 +10462,53 @@ async function setSegmentRetention(ref, deps, policy) {
|
|
|
10975
10462
|
)
|
|
10976
10463
|
},
|
|
10977
10464
|
async (record) => {
|
|
10978
|
-
|
|
10465
|
+
if (record !== null && tokens.wrote(record)) {
|
|
10466
|
+
return done(created.has(record.pointerId), readExpiresAt(record));
|
|
10467
|
+
}
|
|
10979
10468
|
if (record === null) {
|
|
10980
|
-
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
|
|
10986
|
-
|
|
10987
|
-
|
|
10988
|
-
|
|
10989
|
-
|
|
10469
|
+
creating = true;
|
|
10470
|
+
let landed2;
|
|
10471
|
+
try {
|
|
10472
|
+
landed2 = await settledWrite(
|
|
10473
|
+
ref,
|
|
10474
|
+
deps,
|
|
10475
|
+
tokens,
|
|
10476
|
+
null,
|
|
10477
|
+
() => deps.registry.create(
|
|
10478
|
+
ref,
|
|
10479
|
+
{ currentGen: null, retention: { [EXPIRES_AT]: policy.expiresAt } },
|
|
10480
|
+
tokens.options
|
|
10481
|
+
)
|
|
10482
|
+
);
|
|
10483
|
+
} finally {
|
|
10484
|
+
creating = false;
|
|
10485
|
+
}
|
|
10486
|
+
return done(true, landed2 === null ? policy.expiresAt : readExpiresAt(landed2));
|
|
10990
10487
|
}
|
|
10991
10488
|
assertNotDestroyed(record, ref);
|
|
10992
|
-
|
|
10993
|
-
|
|
10994
|
-
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
10998
|
-
|
|
10999
|
-
|
|
11000
|
-
|
|
10489
|
+
const was = readExpiresAt(record);
|
|
10490
|
+
if (was !== void 0) previous.add(was);
|
|
10491
|
+
const landed = await settledWrite(
|
|
10492
|
+
ref,
|
|
10493
|
+
deps,
|
|
10494
|
+
tokens,
|
|
10495
|
+
record,
|
|
10496
|
+
() => deps.registry.compareAndSwap(
|
|
10497
|
+
ref,
|
|
10498
|
+
record.token,
|
|
10499
|
+
{ retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt } },
|
|
10500
|
+
tokens.options
|
|
10501
|
+
)
|
|
10502
|
+
);
|
|
10503
|
+
return done(false, landed === null ? policy.expiresAt : readExpiresAt(landed));
|
|
11001
10504
|
}
|
|
11002
10505
|
);
|
|
11003
10506
|
}
|
|
11004
10507
|
async function clearSegmentRetention(ref, deps) {
|
|
11005
|
-
validateUserRef(ref);
|
|
10508
|
+
validateUserRef(ref, "clearRetention");
|
|
10509
|
+
const tokens = writeTokens(deps.registry);
|
|
10510
|
+
const previous = /* @__PURE__ */ new Set();
|
|
10511
|
+
let unanswered = false;
|
|
11006
10512
|
return casRetry(
|
|
11007
10513
|
deps,
|
|
11008
10514
|
ref,
|
|
@@ -11013,15 +10519,40 @@ async function clearSegmentRetention(ref, deps) {
|
|
|
11013
10519
|
)
|
|
11014
10520
|
},
|
|
11015
10521
|
async (record) => {
|
|
10522
|
+
if (tokens.wrote(record)) {
|
|
10523
|
+
await reindex(ref, deps, void 0, previous);
|
|
10524
|
+
return true;
|
|
10525
|
+
}
|
|
11016
10526
|
if (record === null) return false;
|
|
11017
10527
|
if (record.status === "destroyed") return false;
|
|
11018
|
-
|
|
10528
|
+
const expiresAt = readExpiresAt(record);
|
|
10529
|
+
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) {
|
|
10530
|
+
if (unanswered) await reindex(ref, deps, void 0, previous);
|
|
10531
|
+
return false;
|
|
10532
|
+
}
|
|
10533
|
+
if (expiresAt !== void 0) previous.add(expiresAt);
|
|
11019
10534
|
const rest = { ...record.retention };
|
|
11020
10535
|
delete rest[EXPIRES_AT];
|
|
11021
|
-
|
|
11022
|
-
|
|
11023
|
-
|
|
11024
|
-
|
|
10536
|
+
try {
|
|
10537
|
+
await settledWrite(
|
|
10538
|
+
ref,
|
|
10539
|
+
deps,
|
|
10540
|
+
tokens,
|
|
10541
|
+
record,
|
|
10542
|
+
() => deps.registry.compareAndSwap(
|
|
10543
|
+
ref,
|
|
10544
|
+
record.token,
|
|
10545
|
+
{ retention: Object.keys(rest).length === 0 ? void 0 : rest },
|
|
10546
|
+
tokens.options
|
|
10547
|
+
)
|
|
10548
|
+
);
|
|
10549
|
+
} catch (err) {
|
|
10550
|
+
if (isTransientError(err) || isUnsettledConflict(err) || isWriteConflictError(err) && isTransientError(err.cause)) {
|
|
10551
|
+
unanswered = true;
|
|
10552
|
+
}
|
|
10553
|
+
throw err;
|
|
10554
|
+
}
|
|
10555
|
+
await reindex(ref, deps, void 0, previous);
|
|
11025
10556
|
return true;
|
|
11026
10557
|
}
|
|
11027
10558
|
);
|
|
@@ -11029,7 +10560,7 @@ async function clearSegmentRetention(ref, deps) {
|
|
|
11029
10560
|
function assertNotDestroyed(record, ref) {
|
|
11030
10561
|
if (record.status === "destroyed") {
|
|
11031
10562
|
throw new ValidationError(
|
|
11032
|
-
`segment "${ref.segment}" is destroyed
|
|
10563
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to set a retention policy on a tombstone. Use a new segment; to delete objects a destroy left in the bucket, run dropSegment.`
|
|
11033
10564
|
);
|
|
11034
10565
|
}
|
|
11035
10566
|
}
|
|
@@ -11038,8 +10569,31 @@ function assertNotDestroyed(record, ref) {
|
|
|
11038
10569
|
var MAX_CAS_ATTEMPTS = 8;
|
|
11039
10570
|
var ERASE_CONCURRENCY = 8;
|
|
11040
10571
|
var MAX_STORAGE_SWEEPS = 3;
|
|
10572
|
+
function checkEraseDeps(name, deps) {
|
|
10573
|
+
const registry = deps === null || typeof deps !== "object" ? void 0 : deps.registry;
|
|
10574
|
+
if (registry === void 0 || registry === null || typeof registry.get !== "function") {
|
|
10575
|
+
throw new ValidationError(
|
|
10576
|
+
`${name}: \`deps.registry\` must be a registry driver, such as \`backend.registry\``
|
|
10577
|
+
);
|
|
10578
|
+
}
|
|
10579
|
+
}
|
|
10580
|
+
function checkDropOptions(options) {
|
|
10581
|
+
refuseUnknownOptions("dropSegment", options, ["confirmSegment", "dryRun", "audit"]);
|
|
10582
|
+
requireBool("dropSegment: `dryRun`", options.dryRun);
|
|
10583
|
+
checkedAuditSink(options.audit, "dropSegment");
|
|
10584
|
+
}
|
|
10585
|
+
var DESTROY_SEGMENT_KEYS = ["confirmSegment", "allowCleartext", "audit"];
|
|
10586
|
+
var ERASE_NAMESPACE_KEYS = [
|
|
10587
|
+
"confirmNamespace",
|
|
10588
|
+
"allowCleartext",
|
|
10589
|
+
"audit",
|
|
10590
|
+
"maxScanSegments"
|
|
10591
|
+
];
|
|
11041
10592
|
async function destroySegment(ref, deps, options) {
|
|
11042
|
-
validateUserRef(ref);
|
|
10593
|
+
validateUserRef(ref, "destroySegment");
|
|
10594
|
+
checkEraseDeps("destroySegment", deps);
|
|
10595
|
+
refuseUnknownOptions("destroySegment", options, DESTROY_SEGMENT_KEYS);
|
|
10596
|
+
requireBool("destroySegment: `allowCleartext`", options.allowCleartext);
|
|
11043
10597
|
checkedAuditSink(options.audit, "destroySegment");
|
|
11044
10598
|
if (options.confirmSegment !== ref.segment) {
|
|
11045
10599
|
throw new ValidationError(
|
|
@@ -11056,14 +10610,31 @@ async function destroySegment(ref, deps, options) {
|
|
|
11056
10610
|
segment: ref.segment,
|
|
11057
10611
|
...incarnationField(token)
|
|
11058
10612
|
});
|
|
10613
|
+
} else if (isCleartextTombstone(result)) {
|
|
10614
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent(tombstoneEvent(ref, token));
|
|
11059
10615
|
}
|
|
11060
10616
|
return result;
|
|
11061
10617
|
}
|
|
10618
|
+
function isCleartextTombstone(result) {
|
|
10619
|
+
return result.destroyed && !result.cryptoShredded && result.reason === void 0;
|
|
10620
|
+
}
|
|
10621
|
+
function tombstoneEvent(ref, token) {
|
|
10622
|
+
return {
|
|
10623
|
+
kind: "segment.tombstone",
|
|
10624
|
+
namespace: ref.namespace,
|
|
10625
|
+
segment: ref.segment,
|
|
10626
|
+
...incarnationField(token),
|
|
10627
|
+
objectsReadable: true
|
|
10628
|
+
};
|
|
10629
|
+
}
|
|
11062
10630
|
async function eraseNamespace(namespace, deps, options) {
|
|
11063
10631
|
if (typeof namespace !== "string" || namespace.length === 0) {
|
|
11064
10632
|
throw new ValidationError("eraseNamespace: namespace must be a non-empty string");
|
|
11065
10633
|
}
|
|
11066
10634
|
validateUserNamespace(namespace);
|
|
10635
|
+
checkEraseDeps("eraseNamespace", deps);
|
|
10636
|
+
refuseUnknownOptions("eraseNamespace", options, ERASE_NAMESPACE_KEYS);
|
|
10637
|
+
requireBool("eraseNamespace: `allowCleartext`", options.allowCleartext);
|
|
11067
10638
|
checkedAuditSink(options.audit, "eraseNamespace");
|
|
11068
10639
|
if (options.confirmNamespace !== namespace) {
|
|
11069
10640
|
throw new ValidationError(
|
|
@@ -11108,6 +10679,8 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
11108
10679
|
segment: ref.segment,
|
|
11109
10680
|
...incarnationField(token)
|
|
11110
10681
|
});
|
|
10682
|
+
} else if (isCleartextTombstone(result)) {
|
|
10683
|
+
audit.onEvent(tombstoneEvent(ref, token));
|
|
11111
10684
|
}
|
|
11112
10685
|
return result;
|
|
11113
10686
|
});
|
|
@@ -11118,8 +10691,8 @@ async function dropSegment(ref, deps, options) {
|
|
|
11118
10691
|
return await dropSegmentFor(ref, deps, options, void 0);
|
|
11119
10692
|
}
|
|
11120
10693
|
async function dropSegmentFor(ref, deps, options, sweep) {
|
|
11121
|
-
validateUserRef(ref);
|
|
11122
|
-
|
|
10694
|
+
validateUserRef(ref, "dropSegment");
|
|
10695
|
+
checkDropOptions(options);
|
|
11123
10696
|
if (options.confirmSegment !== ref.segment) {
|
|
11124
10697
|
throw new ValidationError(
|
|
11125
10698
|
`dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
|
|
@@ -11155,18 +10728,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11155
10728
|
reason: "absent"
|
|
11156
10729
|
};
|
|
11157
10730
|
}
|
|
11158
|
-
|
|
11159
|
-
const tombstone = await deps.registry.create(ref, {
|
|
11160
|
-
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
11161
|
-
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
11162
|
-
currentGen: orphans[orphans.length - 1],
|
|
11163
|
-
status: "destroyed"
|
|
11164
|
-
});
|
|
11165
|
-
shred = { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
11166
|
-
} catch (err) {
|
|
11167
|
-
if (!isWriteConflictError(err)) throw err;
|
|
11168
|
-
shred = await shredSegment(ref, deps, { allowCleartext: true, op: "dropSegment" });
|
|
11169
|
-
}
|
|
10731
|
+
shred = await claimIdentity(ref, deps, orphans[orphans.length - 1], shred);
|
|
11170
10732
|
}
|
|
11171
10733
|
if (shred.cryptoShredded) {
|
|
11172
10734
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -11182,7 +10744,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11182
10744
|
if (present.length === 0) break;
|
|
11183
10745
|
const outcomes = await mapWithConcurrency(present, ERASE_CONCURRENCY, async (generation) => {
|
|
11184
10746
|
try {
|
|
11185
|
-
await deps.storage
|
|
10747
|
+
await deleteSettled(deps.storage, { ...ref, generation }, void 0, void 0, deps);
|
|
11186
10748
|
return generation;
|
|
11187
10749
|
} catch {
|
|
11188
10750
|
return null;
|
|
@@ -11210,16 +10772,49 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11210
10772
|
return {
|
|
11211
10773
|
...base,
|
|
11212
10774
|
dropped: shred.destroyed,
|
|
11213
|
-
generationsDeleted,
|
|
10775
|
+
generationsDeleted: generationsDeleted.sort((a, b) => a - b),
|
|
11214
10776
|
generationsRemaining,
|
|
11215
10777
|
cryptoShredded: shred.cryptoShredded,
|
|
11216
10778
|
reason: shred.reason
|
|
11217
10779
|
};
|
|
11218
10780
|
}
|
|
10781
|
+
async function claimIdentity(ref, deps, newestGeneration, shred) {
|
|
10782
|
+
const createTokens = writeTokens(deps.registry);
|
|
10783
|
+
try {
|
|
10784
|
+
const tombstone = await deps.registry.create(
|
|
10785
|
+
ref,
|
|
10786
|
+
{
|
|
10787
|
+
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
10788
|
+
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
10789
|
+
currentGen: newestGeneration,
|
|
10790
|
+
status: "destroyed"
|
|
10791
|
+
},
|
|
10792
|
+
createTokens.options
|
|
10793
|
+
);
|
|
10794
|
+
return { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
10795
|
+
} catch (err) {
|
|
10796
|
+
if (!isWriteConflictError(err) && !isTransientError(err)) throw err;
|
|
10797
|
+
if (createTokens.sent === void 0 && isTransientError(err)) throw err;
|
|
10798
|
+
const now = isTransientError(err) ? await readBack(() => deps.registry.get(ref), err, deps) : await deps.registry.get(ref);
|
|
10799
|
+
if (now?.status === "destroyed" && createTokens.wrote(now)) {
|
|
10800
|
+
return { ...shred, destroyed: true, reason: void 0, token: now.token };
|
|
10801
|
+
}
|
|
10802
|
+
if (isTransientError(err) && now === null) throw err;
|
|
10803
|
+
return shredSegment(ref, deps, { allowCleartext: true, op: "dropSegment" });
|
|
10804
|
+
}
|
|
10805
|
+
}
|
|
11219
10806
|
async function shredSegment(ref, deps, options) {
|
|
11220
10807
|
const { allowCleartext, op = "destroySegment", sweep } = options;
|
|
11221
10808
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
11222
|
-
|
|
10809
|
+
const encryptedBy = /* @__PURE__ */ new Map();
|
|
10810
|
+
let sentEncrypted = false;
|
|
10811
|
+
const tokens = writeTokens(deps.registry, (token) => void encryptedBy.set(token, sentEncrypted));
|
|
10812
|
+
const ownShred = (row) => {
|
|
10813
|
+
if (row === null || row.status !== "destroyed") return null;
|
|
10814
|
+
const encrypted = encryptedBy.get(row.pointerId);
|
|
10815
|
+
return encrypted === void 0 ? null : { ...base, destroyed: true, cryptoShredded: encrypted, token: row.token };
|
|
10816
|
+
};
|
|
10817
|
+
const shred = casRetry(
|
|
11223
10818
|
deps,
|
|
11224
10819
|
ref,
|
|
11225
10820
|
{
|
|
@@ -11227,6 +10822,8 @@ async function shredSegment(ref, deps, options) {
|
|
|
11227
10822
|
contention: () => new WriteConflictError(`${op}: contention shredding "${ref.segment}" \u2014 retry`)
|
|
11228
10823
|
},
|
|
11229
10824
|
async (record) => {
|
|
10825
|
+
const own = ownShred(record);
|
|
10826
|
+
if (own !== null) return own;
|
|
11230
10827
|
if (sweep !== void 0 && (record === null || record.status === "destroyed" || !sweep.wanted(record))) {
|
|
11231
10828
|
return null;
|
|
11232
10829
|
}
|
|
@@ -11247,32 +10844,67 @@ async function shredSegment(ref, deps, options) {
|
|
|
11247
10844
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "cleartext" };
|
|
11248
10845
|
}
|
|
11249
10846
|
const unmarked = sweep === void 0 ? withoutSweepMark(record.retention) : null;
|
|
11250
|
-
|
|
11251
|
-
|
|
11252
|
-
|
|
11253
|
-
|
|
11254
|
-
|
|
11255
|
-
|
|
11256
|
-
|
|
11257
|
-
|
|
11258
|
-
|
|
11259
|
-
|
|
11260
|
-
|
|
11261
|
-
|
|
11262
|
-
|
|
11263
|
-
|
|
11264
|
-
|
|
11265
|
-
|
|
10847
|
+
sentEncrypted = encrypted;
|
|
10848
|
+
let tombstone;
|
|
10849
|
+
try {
|
|
10850
|
+
tombstone = await deps.registry.compareAndSwap(
|
|
10851
|
+
ref,
|
|
10852
|
+
record.token,
|
|
10853
|
+
tombstonePatch(record, sweep, unmarked),
|
|
10854
|
+
tokens.options
|
|
10855
|
+
);
|
|
10856
|
+
} catch (err) {
|
|
10857
|
+
if (!isTransientError(err) || tokens.sent === void 0) throw err;
|
|
10858
|
+
const now = await readBack(() => deps.registry.get(ref), err, deps);
|
|
10859
|
+
const landed = ownShred(now);
|
|
10860
|
+
if (landed !== null) return landed;
|
|
10861
|
+
if (now !== null && now.token === record.token) throw err;
|
|
10862
|
+
const moved = new WriteConflictError(
|
|
10863
|
+
`${op}: the row of "${ref.segment}" changed while its shred got no answer`
|
|
10864
|
+
);
|
|
10865
|
+
moved.cause = err;
|
|
10866
|
+
throw moved;
|
|
10867
|
+
}
|
|
11266
10868
|
return { ...base, destroyed: true, cryptoShredded: encrypted, token: tombstone.token };
|
|
11267
10869
|
}
|
|
11268
10870
|
);
|
|
10871
|
+
try {
|
|
10872
|
+
return await shred;
|
|
10873
|
+
} catch (err) {
|
|
10874
|
+
if (isWriteConflictError(err)) {
|
|
10875
|
+
const own = ownShred(await deps.registry.get(ref).catch(() => null));
|
|
10876
|
+
if (own !== null) return own;
|
|
10877
|
+
}
|
|
10878
|
+
throw err;
|
|
10879
|
+
}
|
|
10880
|
+
}
|
|
10881
|
+
function tombstonePatch(record, sweep, unmarked) {
|
|
10882
|
+
return {
|
|
10883
|
+
status: "destroyed",
|
|
10884
|
+
// ← the crypto-shred: the current row's DEK wrappings are gone (earlier versions and backups of the row keep
|
|
10885
|
+
// theirs)
|
|
10886
|
+
wrappedDeks: void 0,
|
|
10887
|
+
keyId: void 0,
|
|
10888
|
+
// The current generation's cached count and metadata go with it: sealed, they cannot be opened without
|
|
10889
|
+
// the wrappings; clear, they would outlive the segment on its tombstone.
|
|
10890
|
+
summary: void 0,
|
|
10891
|
+
// A tombstone holds nothing: a lease names a generation of a segment that now resolves none, and the ledger
|
|
10892
|
+
// vouches for objects of a segment no read resolves, which the pointer staying put would otherwise keep.
|
|
10893
|
+
knownGens: void 0,
|
|
10894
|
+
leases: void 0,
|
|
10895
|
+
// A sweep's mark, in this same write, so no other tombstone ever carries it: any other tombstone write removes
|
|
10896
|
+
// one a restored or hand-edited row carried.
|
|
10897
|
+
...sweep !== void 0 ? { retention: sweep.stamp(record.retention) } : unmarked === null ? {} : { retention: unmarked }
|
|
10898
|
+
};
|
|
11269
10899
|
}
|
|
11270
10900
|
|
|
11271
10901
|
// src/core/retention-sweep.ts
|
|
11272
10902
|
var DEFAULT_RETIRE_LIMIT = 100;
|
|
11273
10903
|
var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
11274
10904
|
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
10905
|
+
var MIN_TOMBSTONE_GRACE_MS = 36e5;
|
|
11275
10906
|
var DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
10907
|
+
var MAX_LOOKBACK_BUCKETS = 3660;
|
|
11276
10908
|
async function rowsFromDueIndex(registry, options) {
|
|
11277
10909
|
const rows = [];
|
|
11278
10910
|
const pointers = /* @__PURE__ */ new Map();
|
|
@@ -11348,10 +10980,10 @@ async function rowsFromFleet(registry, options) {
|
|
|
11348
10980
|
}
|
|
11349
10981
|
return { rows, pointers, litter };
|
|
11350
10982
|
}
|
|
11351
|
-
async function forgetDuePointer(
|
|
10983
|
+
async function forgetDuePointer(deps, ref, expiresAt, onFault) {
|
|
11352
10984
|
if (!canIndex(ref)) return;
|
|
11353
10985
|
try {
|
|
11354
|
-
await registry
|
|
10986
|
+
await rowDeleteSettled(deps.registry, dueIndexRef(dueBucket(expiresAt), ref), void 0, deps);
|
|
11355
10987
|
} catch (err) {
|
|
11356
10988
|
onFault(err);
|
|
11357
10989
|
}
|
|
@@ -11363,22 +10995,22 @@ async function filePurgePointer(registry, ref, purgeAt) {
|
|
|
11363
10995
|
} catch {
|
|
11364
10996
|
}
|
|
11365
10997
|
}
|
|
11366
|
-
async function forgetFoundPointers(
|
|
10998
|
+
async function forgetFoundPointers(deps, found2, onFault) {
|
|
11367
10999
|
for (const pointer of found2) {
|
|
11368
11000
|
try {
|
|
11369
|
-
await registry
|
|
11001
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11370
11002
|
} catch (err) {
|
|
11371
11003
|
onFault(err);
|
|
11372
11004
|
}
|
|
11373
11005
|
}
|
|
11374
11006
|
}
|
|
11375
|
-
async function forgetPurgePointers(
|
|
11007
|
+
async function forgetPurgePointers(deps, ref, purgeAt, found2, onFault) {
|
|
11376
11008
|
if (!canIndex(ref)) return;
|
|
11377
11009
|
const filed = dueIndexRef(dueBucket(purgeAt), ref);
|
|
11378
11010
|
const all = found2.some((p) => p.ref.namespace === filed.namespace) ? found2 : [...found2, { ref: filed, target: ref }];
|
|
11379
11011
|
for (const pointer of all) {
|
|
11380
11012
|
try {
|
|
11381
|
-
await registry
|
|
11013
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11382
11014
|
} catch (err) {
|
|
11383
11015
|
onFault(err);
|
|
11384
11016
|
}
|
|
@@ -11394,10 +11026,38 @@ function resolveRetireOptions(options) {
|
|
|
11394
11026
|
if (!Array.isArray(options.shards)) {
|
|
11395
11027
|
throw new ValidationError("retireExpired: shards must be an array of shard numbers");
|
|
11396
11028
|
}
|
|
11029
|
+
if (options.shards.length === 0) {
|
|
11030
|
+
throw new ValidationError(
|
|
11031
|
+
"retireExpired: shards is empty, so this sweep would own no segment; pass at least one shard number, or leave shards out to sweep every one"
|
|
11032
|
+
);
|
|
11033
|
+
}
|
|
11034
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11397
11035
|
for (const shard of options.shards) {
|
|
11398
|
-
requireInteger(shard, { min: 0, max: total - 1, what: "retireExpired: a shard" });
|
|
11036
|
+
const n = requireInteger(shard, { min: 0, max: total - 1, what: "retireExpired: a shard" });
|
|
11037
|
+
if (seen.has(n)) {
|
|
11038
|
+
throw new ValidationError(
|
|
11039
|
+
`retireExpired: shard ${n} is listed twice; shards is a set of shard numbers`
|
|
11040
|
+
);
|
|
11041
|
+
}
|
|
11042
|
+
seen.add(n);
|
|
11399
11043
|
}
|
|
11400
11044
|
}
|
|
11045
|
+
const scan = options.scan === void 0 ? "fleet" : options.scan;
|
|
11046
|
+
if (scan !== "fleet" && scan !== "index") {
|
|
11047
|
+
throw new ValidationError(
|
|
11048
|
+
`retireExpired: \`scan\` must be 'fleet' or 'index'; got ${typeof scan === "string" ? JSON.stringify(scan) : scan === null ? "null" : typeof scan}`
|
|
11049
|
+
);
|
|
11050
|
+
}
|
|
11051
|
+
const lookbackBuckets = options.lookbackBuckets === void 0 ? DEFAULT_LOOKBACK_BUCKETS : options.lookbackBuckets;
|
|
11052
|
+
requireInteger(lookbackBuckets, {
|
|
11053
|
+
min: 0,
|
|
11054
|
+
max: MAX_LOOKBACK_BUCKETS,
|
|
11055
|
+
what: "retireExpired: `lookbackBuckets`",
|
|
11056
|
+
note: "days; older rows are the fleet scan's to find"
|
|
11057
|
+
});
|
|
11058
|
+
if (options.shards === void 0 && options.totalShards !== void 0) {
|
|
11059
|
+
requirePositiveInteger(options.totalShards, "retireExpired: `totalShards`");
|
|
11060
|
+
}
|
|
11401
11061
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
11402
11062
|
const now = options.now;
|
|
11403
11063
|
requireFiniteNumber(now, { what: "retireExpired: `now`", note: "epoch milliseconds" });
|
|
@@ -11406,11 +11066,15 @@ function resolveRetireOptions(options) {
|
|
|
11406
11066
|
`retireExpired: \`now\` (${now}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds. A sweep with a wrong-units clock treats every policy as expired.`
|
|
11407
11067
|
);
|
|
11408
11068
|
}
|
|
11409
|
-
const limit = options.limit
|
|
11069
|
+
const limit = options.limit === void 0 ? DEFAULT_RETIRE_LIMIT : options.limit;
|
|
11410
11070
|
requirePositiveInteger(limit, "retireExpired: `limit`");
|
|
11411
11071
|
const purgeTombstones = options.purgeTombstones ?? true;
|
|
11412
|
-
const grace = options.tombstoneGraceMs
|
|
11413
|
-
requireFiniteNumber(grace, {
|
|
11072
|
+
const grace = options.tombstoneGraceMs === void 0 ? DEFAULT_TOMBSTONE_GRACE_MS : options.tombstoneGraceMs;
|
|
11073
|
+
requireFiniteNumber(grace, {
|
|
11074
|
+
min: MIN_TOMBSTONE_GRACE_MS,
|
|
11075
|
+
what: "retireExpired: `tombstoneGraceMs`",
|
|
11076
|
+
note: "one hour: the tombstone must outlast a create sent again after it got no answer"
|
|
11077
|
+
});
|
|
11414
11078
|
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11415
11079
|
requirePositiveInteger(maxScanSegments, "retireExpired: `maxScanSegments`");
|
|
11416
11080
|
return {
|
|
@@ -11422,8 +11086,8 @@ function resolveRetireOptions(options) {
|
|
|
11422
11086
|
grace,
|
|
11423
11087
|
maxScanSegments,
|
|
11424
11088
|
dryRun: options.dryRun === true,
|
|
11425
|
-
scan
|
|
11426
|
-
lookbackBuckets
|
|
11089
|
+
scan,
|
|
11090
|
+
lookbackBuckets,
|
|
11427
11091
|
shards: options.shards,
|
|
11428
11092
|
totalShards: options.totalShards ?? 0
|
|
11429
11093
|
};
|
|
@@ -11480,7 +11144,7 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11480
11144
|
try {
|
|
11481
11145
|
const live = await deps.registry.get(ref);
|
|
11482
11146
|
if (live?.status === "destroyed" && retirementStamp(live.retention) === null) {
|
|
11483
|
-
await forgetFoundPointers(deps
|
|
11147
|
+
await forgetFoundPointers(deps, found2, tally.noteFault);
|
|
11484
11148
|
}
|
|
11485
11149
|
} catch (err) {
|
|
11486
11150
|
tally.noteFault(err);
|
|
@@ -11500,7 +11164,7 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11500
11164
|
if (!await isFullyReclaimed(deps, ref)) {
|
|
11501
11165
|
let collectFault;
|
|
11502
11166
|
if (!dryRun) {
|
|
11503
|
-
await gcOrphanGenerations(ref, deps).catch((error) => {
|
|
11167
|
+
await gcOrphanGenerations(ref, deps, { incarnation: rec }).catch((error) => {
|
|
11504
11168
|
collectFault = { error };
|
|
11505
11169
|
});
|
|
11506
11170
|
}
|
|
@@ -11518,12 +11182,12 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11518
11182
|
}
|
|
11519
11183
|
}
|
|
11520
11184
|
if (!dryRun) {
|
|
11521
|
-
await deps.registry
|
|
11185
|
+
await rowDeleteSettled(deps.registry, ref, rec.token, deps);
|
|
11522
11186
|
tally.tombstonesPurged += 1;
|
|
11523
11187
|
tally.purgeRun = 0;
|
|
11524
11188
|
if (removesRows) {
|
|
11525
11189
|
await forgetPurgePointers(
|
|
11526
|
-
deps
|
|
11190
|
+
deps,
|
|
11527
11191
|
ref,
|
|
11528
11192
|
retiredAt + grace,
|
|
11529
11193
|
pointers.get(segmentKey(ref)) ?? [],
|
|
@@ -11554,8 +11218,9 @@ async function retireLive(rec, ctx) {
|
|
|
11554
11218
|
tally.limited = true;
|
|
11555
11219
|
return null;
|
|
11556
11220
|
}
|
|
11221
|
+
let live = null;
|
|
11557
11222
|
try {
|
|
11558
|
-
|
|
11223
|
+
live = await deps.registry.get(ref);
|
|
11559
11224
|
const livePolicy = live === null ? null : readRetentionPolicy(live.retention);
|
|
11560
11225
|
if (live === null || live.status === "destroyed" || livePolicy === null || livePolicy === "invalid" || livePolicy.expiresAt > now) {
|
|
11561
11226
|
return { ...base, action: "skipped", reason: "policy-changed" };
|
|
@@ -11586,11 +11251,11 @@ async function retireLive(rec, ctx) {
|
|
|
11586
11251
|
}
|
|
11587
11252
|
tally.retired += 1;
|
|
11588
11253
|
tally.add({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
11589
|
-
await forgetDuePointer(deps
|
|
11254
|
+
await forgetDuePointer(deps, ref, livePolicy.expiresAt, tally.noteFault);
|
|
11590
11255
|
if (opts.purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
11591
11256
|
const tombstone = await deps.registry.get(ref).catch(() => null);
|
|
11592
11257
|
if (tombstone?.status === "destroyed") {
|
|
11593
|
-
const deleted = await deps.registry
|
|
11258
|
+
const deleted = await rowDeleteSettled(deps.registry, ref, tombstone.token, deps).then(
|
|
11594
11259
|
() => true,
|
|
11595
11260
|
(err) => {
|
|
11596
11261
|
tally.noteFault(err);
|
|
@@ -11603,28 +11268,58 @@ async function retireLive(rec, ctx) {
|
|
|
11603
11268
|
if (removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
11604
11269
|
return null;
|
|
11605
11270
|
} catch (err) {
|
|
11606
|
-
const after = await deps
|
|
11607
|
-
if (after?.status === "destroyed" && after.retention?.[SWEEP_MARK] === now) {
|
|
11608
|
-
|
|
11609
|
-
|
|
11271
|
+
const after = await rowAfterFault(deps, ref, err);
|
|
11272
|
+
if (after?.status === "destroyed" && after.retention?.[SWEEP_MARK] === now && live !== null && sameIncarnation(live, after)) {
|
|
11273
|
+
return retiredAfterFault(live, policy.expiresAt, err, ctx);
|
|
11274
|
+
}
|
|
11275
|
+
if (after === void 0) {
|
|
11610
11276
|
return {
|
|
11611
11277
|
...base,
|
|
11612
|
-
action: "
|
|
11613
|
-
|
|
11614
|
-
result: {
|
|
11615
|
-
...base,
|
|
11616
|
-
dropped: true,
|
|
11617
|
-
generationsDeleted: [],
|
|
11618
|
-
generationsRemaining: [],
|
|
11619
|
-
cryptoShredded: false,
|
|
11620
|
-
reason: void 0
|
|
11621
|
-
},
|
|
11622
|
-
fault: failureReason(err)
|
|
11278
|
+
action: "skipped",
|
|
11279
|
+
reason: `${failureReason(err)} (the row could not be read afterwards: the tombstone may have landed, and the segment may be shredded)`
|
|
11623
11280
|
};
|
|
11624
11281
|
}
|
|
11625
11282
|
return { ...base, action: "skipped", reason: failureReason(err) };
|
|
11626
11283
|
}
|
|
11627
11284
|
}
|
|
11285
|
+
async function rowAfterFault(deps, ref, err) {
|
|
11286
|
+
try {
|
|
11287
|
+
return await readBack(
|
|
11288
|
+
() => deps.registry.get(ref),
|
|
11289
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
11290
|
+
deps
|
|
11291
|
+
);
|
|
11292
|
+
} catch {
|
|
11293
|
+
return void 0;
|
|
11294
|
+
}
|
|
11295
|
+
}
|
|
11296
|
+
async function retiredAfterFault(live, expiresAt, err, ctx) {
|
|
11297
|
+
const { deps, opts, removesRows, tally } = ctx;
|
|
11298
|
+
const ref = { namespace: live.namespace, segment: live.segment };
|
|
11299
|
+
const base = { segment: live.segment, namespace: live.namespace };
|
|
11300
|
+
if (removesRows) await filePurgePointer(deps.registry, ref, opts.now + opts.grace);
|
|
11301
|
+
tally.retired += 1;
|
|
11302
|
+
const remaining = await generationsOf(deps.storage, ref).catch(() => namedBy(live));
|
|
11303
|
+
return {
|
|
11304
|
+
...base,
|
|
11305
|
+
action: "retired",
|
|
11306
|
+
expiresAt,
|
|
11307
|
+
result: {
|
|
11308
|
+
...base,
|
|
11309
|
+
dropped: true,
|
|
11310
|
+
generationsDeleted: [],
|
|
11311
|
+
generationsRemaining: remaining,
|
|
11312
|
+
cryptoShredded: live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0,
|
|
11313
|
+
reason: void 0
|
|
11314
|
+
},
|
|
11315
|
+
fault: failureReason(err)
|
|
11316
|
+
};
|
|
11317
|
+
}
|
|
11318
|
+
function namedBy(row) {
|
|
11319
|
+
const names = new Set((row.knownGens ?? []).map((k) => k.generation));
|
|
11320
|
+
if (row.currentGen !== null) names.add(row.currentGen);
|
|
11321
|
+
return [...names].sort((a, b) => a - b);
|
|
11322
|
+
}
|
|
11628
11323
|
async function retireExpired(deps, options) {
|
|
11629
11324
|
const opts = resolveRetireOptions(options);
|
|
11630
11325
|
const { now, limit, dryRun, shards, totalShards } = opts;
|
|
@@ -11656,7 +11351,7 @@ async function retireExpired(deps, options) {
|
|
|
11656
11351
|
attemptedLitter += 1;
|
|
11657
11352
|
try {
|
|
11658
11353
|
if (await deps.registry.get(pointer.target) !== null) continue;
|
|
11659
|
-
await deps.registry
|
|
11354
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11660
11355
|
} catch (err) {
|
|
11661
11356
|
tally.noteFault(err);
|
|
11662
11357
|
}
|
|
@@ -11694,6 +11389,411 @@ function failureReason(err) {
|
|
|
11694
11389
|
return `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
11695
11390
|
}
|
|
11696
11391
|
|
|
11392
|
+
// src/core/preflight.ts
|
|
11393
|
+
var PREFLIGHT_NAMESPACE = `${RESERVED_NAMESPACE_PREFIX}preflight`;
|
|
11394
|
+
var PREFLIGHT_SEGMENT = "probe";
|
|
11395
|
+
var LIFECYCLE_TIMEOUT_MS = 3e4;
|
|
11396
|
+
var LIFECYCLE_TICK_MS = 250;
|
|
11397
|
+
var MARKER_A = ascii("cloudbitmaps preflight probe A");
|
|
11398
|
+
var MARKER_B = ascii("cloudbitmaps preflight probe B");
|
|
11399
|
+
var TAIL_BYTES = 64;
|
|
11400
|
+
var MAX_MESSAGE_CHARS = 300;
|
|
11401
|
+
var MAX_LEFTOVERS_NAMED = 3;
|
|
11402
|
+
var MAX_LIFECYCLE_DETAIL_CHARS = 1e3;
|
|
11403
|
+
function ascii(text) {
|
|
11404
|
+
const bytes = new Uint8Array(text.length);
|
|
11405
|
+
for (let i = 0; i < text.length; i += 1) bytes[i] = text.charCodeAt(i);
|
|
11406
|
+
return bytes;
|
|
11407
|
+
}
|
|
11408
|
+
function sameBytes2(a, b) {
|
|
11409
|
+
if (a.length !== b.length) return false;
|
|
11410
|
+
for (let i = 0; i < a.length; i += 1) if (a[i] !== b[i]) return false;
|
|
11411
|
+
return true;
|
|
11412
|
+
}
|
|
11413
|
+
function probeNumber(entropy) {
|
|
11414
|
+
const bytes = entropy(6);
|
|
11415
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < 6) {
|
|
11416
|
+
throw new TypeError("preflight needs an entropy source that returns the bytes it is asked for");
|
|
11417
|
+
}
|
|
11418
|
+
let n = 0;
|
|
11419
|
+
for (let i = 0; i < 6; i += 1) n = n * 256 + (bytes[i] ?? 0);
|
|
11420
|
+
return n;
|
|
11421
|
+
}
|
|
11422
|
+
function describeError(err) {
|
|
11423
|
+
let name = "Error";
|
|
11424
|
+
let message = "";
|
|
11425
|
+
try {
|
|
11426
|
+
if (typeof err === "object" && err !== null) {
|
|
11427
|
+
const n = err.name;
|
|
11428
|
+
const m = err.message;
|
|
11429
|
+
if (typeof n === "string" && n !== "") name = n;
|
|
11430
|
+
if (typeof m === "string") message = m;
|
|
11431
|
+
} else {
|
|
11432
|
+
message = String(err);
|
|
11433
|
+
}
|
|
11434
|
+
} catch {
|
|
11435
|
+
}
|
|
11436
|
+
const cut = message.length > MAX_MESSAGE_CHARS ? `${message.slice(0, MAX_MESSAGE_CHARS)}...` : message;
|
|
11437
|
+
return cut === "" ? name : `${name}: ${cut}`;
|
|
11438
|
+
}
|
|
11439
|
+
function failed(doing, err) {
|
|
11440
|
+
let transient = false;
|
|
11441
|
+
try {
|
|
11442
|
+
transient = isTransientError(err);
|
|
11443
|
+
} catch {
|
|
11444
|
+
}
|
|
11445
|
+
return `${doing} failed with ${describeError(err)}.${transient ? " This is a transient fault: run preflight() again." : ""}`;
|
|
11446
|
+
}
|
|
11447
|
+
function isConflict(err) {
|
|
11448
|
+
try {
|
|
11449
|
+
return isWriteConflictError(err);
|
|
11450
|
+
} catch {
|
|
11451
|
+
return false;
|
|
11452
|
+
}
|
|
11453
|
+
}
|
|
11454
|
+
var REFUSAL_NAMES = /* @__PURE__ */ new Set([
|
|
11455
|
+
"AccessDenied",
|
|
11456
|
+
"Forbidden",
|
|
11457
|
+
"AuthorizationFailure",
|
|
11458
|
+
"AuthorizationPermissionMismatch"
|
|
11459
|
+
]);
|
|
11460
|
+
function provesRefused(err) {
|
|
11461
|
+
try {
|
|
11462
|
+
for (let e = err, depth = 0; typeof e === "object" && e !== null && depth < 2; depth += 1) {
|
|
11463
|
+
if (isValidationError(e)) return true;
|
|
11464
|
+
const shape = e;
|
|
11465
|
+
if (typeof shape.name === "string" && REFUSAL_NAMES.has(shape.name)) return true;
|
|
11466
|
+
const status = shape.$metadata?.httpStatusCode ?? shape.statusCode ?? shape.status ?? shape.code;
|
|
11467
|
+
if (status === 401 || status === 403) return true;
|
|
11468
|
+
e = shape.cause;
|
|
11469
|
+
}
|
|
11470
|
+
} catch {
|
|
11471
|
+
}
|
|
11472
|
+
return false;
|
|
11473
|
+
}
|
|
11474
|
+
var ORDER = [
|
|
11475
|
+
"registry",
|
|
11476
|
+
"write",
|
|
11477
|
+
"write-once",
|
|
11478
|
+
"read",
|
|
11479
|
+
"list",
|
|
11480
|
+
"delete",
|
|
11481
|
+
"lifecycle"
|
|
11482
|
+
];
|
|
11483
|
+
async function lifecycleStep(deps) {
|
|
11484
|
+
const check = deps.checkLifecycle;
|
|
11485
|
+
if (check === void 0) {
|
|
11486
|
+
return {
|
|
11487
|
+
status: "unchecked",
|
|
11488
|
+
detail: "This backend does not offer the lifecycle check. Look at the bucket lifecycle rules by hand: none may delete or archive the store objects or the registry rows."
|
|
11489
|
+
};
|
|
11490
|
+
}
|
|
11491
|
+
let settled2 = false;
|
|
11492
|
+
const run = (async () => {
|
|
11493
|
+
try {
|
|
11494
|
+
return { kind: "answered", verdict: await check() };
|
|
11495
|
+
} catch (error) {
|
|
11496
|
+
return { kind: "threw", error };
|
|
11497
|
+
} finally {
|
|
11498
|
+
settled2 = true;
|
|
11499
|
+
}
|
|
11500
|
+
})();
|
|
11501
|
+
const bound = (async () => {
|
|
11502
|
+
try {
|
|
11503
|
+
for (let waited = 0; waited < LIFECYCLE_TIMEOUT_MS && !settled2; waited += LIFECYCLE_TICK_MS) {
|
|
11504
|
+
await deps.clock.sleep(LIFECYCLE_TICK_MS);
|
|
11505
|
+
}
|
|
11506
|
+
} catch (error) {
|
|
11507
|
+
return { kind: "clock", error };
|
|
11508
|
+
}
|
|
11509
|
+
return { kind: "timeout" };
|
|
11510
|
+
})();
|
|
11511
|
+
let outcome = await Promise.race([run, bound]);
|
|
11512
|
+
if (outcome.kind === "timeout" && settled2) outcome = await run;
|
|
11513
|
+
if (outcome.kind === "timeout") {
|
|
11514
|
+
return {
|
|
11515
|
+
status: "unchecked",
|
|
11516
|
+
detail: `The lifecycle check did not answer within ${LIFECYCLE_TIMEOUT_MS / 1e3} seconds. Run preflight() again, or look at the bucket lifecycle rules by hand.`
|
|
11517
|
+
};
|
|
11518
|
+
}
|
|
11519
|
+
if (outcome.kind === "clock") {
|
|
11520
|
+
return {
|
|
11521
|
+
status: "unchecked",
|
|
11522
|
+
detail: `The store's clock failed while the lifecycle check was timed: ${describeError(outcome.error)}. Run preflight() again, or look at the bucket lifecycle rules by hand.`
|
|
11523
|
+
};
|
|
11524
|
+
}
|
|
11525
|
+
if (outcome.kind === "threw") {
|
|
11526
|
+
return {
|
|
11527
|
+
status: "unchecked",
|
|
11528
|
+
detail: `${failed("The lifecycle check", outcome.error)} Look at the bucket lifecycle rules by hand.`
|
|
11529
|
+
};
|
|
11530
|
+
}
|
|
11531
|
+
return readVerdict(outcome.verdict);
|
|
11532
|
+
}
|
|
11533
|
+
function readVerdict(answer) {
|
|
11534
|
+
const unreadable = {
|
|
11535
|
+
status: "unchecked",
|
|
11536
|
+
detail: "The lifecycle check gave an answer preflight could not read. Look at the bucket lifecycle rules by hand."
|
|
11537
|
+
};
|
|
11538
|
+
try {
|
|
11539
|
+
const verdict = answer;
|
|
11540
|
+
const status = verdict?.status;
|
|
11541
|
+
if (status !== "ok" && status !== "problem" && status !== "unchecked") return unreadable;
|
|
11542
|
+
const text = verdict?.detail;
|
|
11543
|
+
const detail = typeof text === "string" ? text : "";
|
|
11544
|
+
return {
|
|
11545
|
+
status,
|
|
11546
|
+
detail: detail.length > MAX_LIFECYCLE_DETAIL_CHARS ? `${detail.slice(0, MAX_LIFECYCLE_DETAIL_CHARS)}...` : detail
|
|
11547
|
+
};
|
|
11548
|
+
} catch {
|
|
11549
|
+
return unreadable;
|
|
11550
|
+
}
|
|
11551
|
+
}
|
|
11552
|
+
async function registryStep(registry) {
|
|
11553
|
+
const row = await registry.get({ namespace: PREFLIGHT_NAMESPACE, segment: PREFLIGHT_SEGMENT });
|
|
11554
|
+
if (row !== null) {
|
|
11555
|
+
return {
|
|
11556
|
+
status: "problem",
|
|
11557
|
+
detail: `A registry row exists at ${PREFLIGHT_NAMESPACE}/${PREFLIGHT_SEGMENT}, an address only the library uses and never writes a row at. The registry is not the one this store expects, or something else writes there.`
|
|
11558
|
+
};
|
|
11559
|
+
}
|
|
11560
|
+
if (registry.capabilities().canWrite === false) {
|
|
11561
|
+
return {
|
|
11562
|
+
status: "problem",
|
|
11563
|
+
detail: "The registry reads but cannot write in this runtime (no Web Crypto to draw its tokens from), so a load would be refused."
|
|
11564
|
+
};
|
|
11565
|
+
}
|
|
11566
|
+
return { status: "ok", detail: "The registry answered a read, and can write." };
|
|
11567
|
+
}
|
|
11568
|
+
async function guarded(run, onThrow) {
|
|
11569
|
+
try {
|
|
11570
|
+
return await run();
|
|
11571
|
+
} catch (err) {
|
|
11572
|
+
return onThrow(err);
|
|
11573
|
+
}
|
|
11574
|
+
}
|
|
11575
|
+
var skipped = (because) => ({
|
|
11576
|
+
status: "unchecked",
|
|
11577
|
+
detail: `Not run: ${because}.`
|
|
11578
|
+
});
|
|
11579
|
+
async function writeStep(probe) {
|
|
11580
|
+
try {
|
|
11581
|
+
await probe.storage.putImmutable(probe.key, async (sink) => sink.write(MARKER_A));
|
|
11582
|
+
probe.found.set("write", { status: "ok", detail: "A new object was written." });
|
|
11583
|
+
return { wrote: true, notOurs: false };
|
|
11584
|
+
} catch (err) {
|
|
11585
|
+
const notOurs = isConflict(err);
|
|
11586
|
+
probe.refused = !notOurs && provesRefused(err);
|
|
11587
|
+
probe.found.set("write", {
|
|
11588
|
+
status: "problem",
|
|
11589
|
+
detail: notOurs ? "The probe object already existed: another run drew the same number. Run preflight() again." : failed("Writing a new object", err)
|
|
11590
|
+
});
|
|
11591
|
+
return { wrote: false, notOurs };
|
|
11592
|
+
}
|
|
11593
|
+
}
|
|
11594
|
+
async function writeOnceStep(probe) {
|
|
11595
|
+
try {
|
|
11596
|
+
await probe.storage.putImmutable(probe.key, async (sink) => sink.write(MARKER_B));
|
|
11597
|
+
probe.accepted = MARKER_B;
|
|
11598
|
+
probe.found.set("write-once", {
|
|
11599
|
+
status: "problem",
|
|
11600
|
+
detail: "A second write to the same key succeeded. The storage does not refuse an overwrite, so a generation could be replaced under a reader."
|
|
11601
|
+
});
|
|
11602
|
+
} catch (err) {
|
|
11603
|
+
probe.found.set(
|
|
11604
|
+
"write-once",
|
|
11605
|
+
isConflict(err) ? { status: "ok", detail: "A second write to the same key was refused." } : {
|
|
11606
|
+
status: "problem",
|
|
11607
|
+
detail: `${failed("The second write to the same key", err)} Whether it was refused is not known.`
|
|
11608
|
+
}
|
|
11609
|
+
);
|
|
11610
|
+
}
|
|
11611
|
+
}
|
|
11612
|
+
async function readStep(probe) {
|
|
11613
|
+
try {
|
|
11614
|
+
const tail = await probe.storage.getTail(probe.key, TAIL_BYTES);
|
|
11615
|
+
probe.version = tail.version;
|
|
11616
|
+
const same = sameBytes2(tail.bytes, probe.accepted) && tail.size === probe.accepted.length;
|
|
11617
|
+
probe.found.set(
|
|
11618
|
+
"read",
|
|
11619
|
+
same ? { status: "ok", detail: "The object just written read back as written." } : {
|
|
11620
|
+
status: "problem",
|
|
11621
|
+
detail: `The object just written did not read back as written: the bytes or the size differ.${probe.found.get("write-once")?.status === "problem" ? " It was compared against the bytes the storage accepted last, so look at the write-once check first." : ""}`
|
|
11622
|
+
}
|
|
11623
|
+
);
|
|
11624
|
+
} catch (err) {
|
|
11625
|
+
probe.found.set("read", {
|
|
11626
|
+
status: "problem",
|
|
11627
|
+
detail: failed("Reading the object just written", err)
|
|
11628
|
+
});
|
|
11629
|
+
}
|
|
11630
|
+
}
|
|
11631
|
+
function leftoversSentence(others, named) {
|
|
11632
|
+
if (others === 0) return "";
|
|
11633
|
+
const more = others > named.length ? ", ..." : "";
|
|
11634
|
+
const one = others === 1;
|
|
11635
|
+
return ` ${others} other probe object${one ? "" : "s"} (generation ${named.join(", ")}${more}) ${one ? "was" : "were"} left alone: ${one ? "it may be" : "they may be"} a run in progress, or one that was cut off.`;
|
|
11636
|
+
}
|
|
11637
|
+
async function listStep(probe) {
|
|
11638
|
+
try {
|
|
11639
|
+
let seen = false;
|
|
11640
|
+
let others = 0;
|
|
11641
|
+
const named = [];
|
|
11642
|
+
for await (const listed2 of probe.storage.list({
|
|
11643
|
+
namespace: PREFLIGHT_NAMESPACE,
|
|
11644
|
+
segment: PREFLIGHT_SEGMENT
|
|
11645
|
+
})) {
|
|
11646
|
+
if (listed2.generation === probe.key.generation) {
|
|
11647
|
+
seen = true;
|
|
11648
|
+
} else {
|
|
11649
|
+
others += 1;
|
|
11650
|
+
if (named.length < MAX_LEFTOVERS_NAMED) named.push(listed2.generation);
|
|
11651
|
+
}
|
|
11652
|
+
}
|
|
11653
|
+
const leftovers = leftoversSentence(others, named);
|
|
11654
|
+
probe.found.set(
|
|
11655
|
+
"list",
|
|
11656
|
+
seen ? { status: "ok", detail: `The listing named the object just written.${leftovers}` } : {
|
|
11657
|
+
status: "problem",
|
|
11658
|
+
detail: `The listing did not name the object just written, so a listing can lag a write.${leftovers}`
|
|
11659
|
+
}
|
|
11660
|
+
);
|
|
11661
|
+
} catch (err) {
|
|
11662
|
+
probe.found.set("list", {
|
|
11663
|
+
status: "problem",
|
|
11664
|
+
detail: failed("Listing the probe segment", err)
|
|
11665
|
+
});
|
|
11666
|
+
}
|
|
11667
|
+
}
|
|
11668
|
+
async function deleteStep(probe, wrote) {
|
|
11669
|
+
try {
|
|
11670
|
+
await probe.storage.delete(
|
|
11671
|
+
probe.key,
|
|
11672
|
+
probe.version === void 0 ? void 0 : { ifVersion: probe.version }
|
|
11673
|
+
);
|
|
11674
|
+
probe.found.set(
|
|
11675
|
+
"delete",
|
|
11676
|
+
wrote ? { status: "ok", detail: "The object was deleted." } : {
|
|
11677
|
+
status: "unchecked",
|
|
11678
|
+
detail: "The write failed, so this check was not made. The run deleted its key in case the write had landed; that delete succeeded, which it also does when no object was there."
|
|
11679
|
+
}
|
|
11680
|
+
);
|
|
11681
|
+
} catch (err) {
|
|
11682
|
+
if (wrote) {
|
|
11683
|
+
probe.found.set("delete", {
|
|
11684
|
+
status: "problem",
|
|
11685
|
+
detail: `${failed("Deleting the object just written", err)} The object ${probe.object} was left behind: delete it by hand.`
|
|
11686
|
+
});
|
|
11687
|
+
return;
|
|
11688
|
+
}
|
|
11689
|
+
const write = probe.found.get("write");
|
|
11690
|
+
if (probe.refused) {
|
|
11691
|
+
probe.found.set("delete", {
|
|
11692
|
+
status: "unchecked",
|
|
11693
|
+
detail: `The write failed, so this check was not made. The run tried to delete its key in case the write had landed, and that failed with ${describeError(err)}. The write was most likely refused and stored nothing; a later run's \`list\` names any probe it finds.`
|
|
11694
|
+
});
|
|
11695
|
+
return;
|
|
11696
|
+
}
|
|
11697
|
+
probe.found.set("delete", {
|
|
11698
|
+
status: "unchecked",
|
|
11699
|
+
detail: `The write failed, so this check was not made. The run tried to delete its key in case the write had landed, and that failed with ${describeError(err)}.`
|
|
11700
|
+
});
|
|
11701
|
+
if (write !== void 0) {
|
|
11702
|
+
probe.found.set("write", {
|
|
11703
|
+
status: write.status,
|
|
11704
|
+
detail: `${write.detail} The write may have landed, and the delete of ${probe.object} failed, so it may have been left behind.`
|
|
11705
|
+
});
|
|
11706
|
+
}
|
|
11707
|
+
}
|
|
11708
|
+
}
|
|
11709
|
+
async function objectSteps(probe) {
|
|
11710
|
+
const { wrote, notOurs } = await writeStep(probe);
|
|
11711
|
+
try {
|
|
11712
|
+
if (wrote) {
|
|
11713
|
+
await writeOnceStep(probe);
|
|
11714
|
+
await readStep(probe);
|
|
11715
|
+
await listStep(probe);
|
|
11716
|
+
} else {
|
|
11717
|
+
const why = skipped("no object was written");
|
|
11718
|
+
for (const name of ["write-once", "read", "list"]) probe.found.set(name, why);
|
|
11719
|
+
}
|
|
11720
|
+
} finally {
|
|
11721
|
+
if (notOurs) {
|
|
11722
|
+
probe.found.set(
|
|
11723
|
+
"delete",
|
|
11724
|
+
skipped("the object under the key was not this run's, so it was left alone")
|
|
11725
|
+
);
|
|
11726
|
+
} else {
|
|
11727
|
+
await deleteStep(probe, wrote);
|
|
11728
|
+
}
|
|
11729
|
+
}
|
|
11730
|
+
}
|
|
11731
|
+
function drawKey(entropy) {
|
|
11732
|
+
try {
|
|
11733
|
+
return {
|
|
11734
|
+
namespace: PREFLIGHT_NAMESPACE,
|
|
11735
|
+
segment: PREFLIGHT_SEGMENT,
|
|
11736
|
+
generation: probeNumber(entropy)
|
|
11737
|
+
};
|
|
11738
|
+
} catch (error) {
|
|
11739
|
+
return { error };
|
|
11740
|
+
}
|
|
11741
|
+
}
|
|
11742
|
+
function noNumber(error) {
|
|
11743
|
+
const found2 = /* @__PURE__ */ new Map();
|
|
11744
|
+
found2.set("write", {
|
|
11745
|
+
status: "problem",
|
|
11746
|
+
detail: `No number could be drawn for the probe's name, so nothing was written: ${describeError(error)}. Give the store an entropy source (the runtime's Web Crypto, or the \`entropy\` seam).`
|
|
11747
|
+
});
|
|
11748
|
+
const why = skipped("no object was written");
|
|
11749
|
+
for (const name of ["write-once", "read", "list", "delete"]) found2.set(name, why);
|
|
11750
|
+
return found2;
|
|
11751
|
+
}
|
|
11752
|
+
async function objectFindings(deps, key) {
|
|
11753
|
+
const probe = {
|
|
11754
|
+
storage: deps.storage,
|
|
11755
|
+
key,
|
|
11756
|
+
object: `${PREFLIGHT_NAMESPACE}/segments/${PREFLIGHT_SEGMENT}.${key.generation}.crbm (under the store's prefix)`,
|
|
11757
|
+
found: /* @__PURE__ */ new Map(),
|
|
11758
|
+
accepted: MARKER_A,
|
|
11759
|
+
refused: false
|
|
11760
|
+
};
|
|
11761
|
+
try {
|
|
11762
|
+
await objectSteps(probe);
|
|
11763
|
+
} catch (err) {
|
|
11764
|
+
for (const name of ["write", "write-once", "read", "list", "delete"]) {
|
|
11765
|
+
if (!probe.found.has(name)) {
|
|
11766
|
+
probe.found.set(name, { status: "problem", detail: failed("The check", err) });
|
|
11767
|
+
}
|
|
11768
|
+
}
|
|
11769
|
+
}
|
|
11770
|
+
return probe.found;
|
|
11771
|
+
}
|
|
11772
|
+
async function runPreflight(deps) {
|
|
11773
|
+
const registry = await guarded(
|
|
11774
|
+
() => registryStep(deps.registry),
|
|
11775
|
+
(err) => ({ status: "problem", detail: failed("Reading the registry", err) })
|
|
11776
|
+
);
|
|
11777
|
+
const key = drawKey(deps.entropy);
|
|
11778
|
+
const found2 = "error" in key ? noNumber(key.error) : await objectFindings(deps, key);
|
|
11779
|
+
found2.set("registry", registry);
|
|
11780
|
+
found2.set(
|
|
11781
|
+
"lifecycle",
|
|
11782
|
+
await guarded(
|
|
11783
|
+
() => lifecycleStep(deps),
|
|
11784
|
+
(err) => ({
|
|
11785
|
+
status: "unchecked",
|
|
11786
|
+
detail: `${failed("The lifecycle check", err)} Look at the bucket lifecycle rules by hand.`
|
|
11787
|
+
})
|
|
11788
|
+
)
|
|
11789
|
+
);
|
|
11790
|
+
const checks = ORDER.map((name) => ({
|
|
11791
|
+
name,
|
|
11792
|
+
...found2.get(name) ?? skipped("the check did not run")
|
|
11793
|
+
}));
|
|
11794
|
+
return { ok: !checks.some((c) => c.status === "problem"), checks };
|
|
11795
|
+
}
|
|
11796
|
+
|
|
11697
11797
|
// src/core/consistency.ts
|
|
11698
11798
|
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
11699
11799
|
var NOTHING = { issues: [] };
|
|
@@ -11838,7 +11938,7 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
11838
11938
|
if (options.generations !== void 0 && typeof options.generations !== "boolean") {
|
|
11839
11939
|
throw new ValidationError(`generations must be a boolean; got ${String(options.generations)}`);
|
|
11840
11940
|
}
|
|
11841
|
-
const concurrency = options.concurrency
|
|
11941
|
+
const concurrency = options.concurrency === void 0 ? DEFAULT_CHECK_CONCURRENCY : options.concurrency;
|
|
11842
11942
|
requirePositiveInteger(concurrency, "concurrency");
|
|
11843
11943
|
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11844
11944
|
const recs = await drainRegistry(deps.registry, {
|
|
@@ -11997,19 +12097,15 @@ export {
|
|
|
11997
12097
|
CrbmStorageChunkSource,
|
|
11998
12098
|
DEFAULT_BUDGET,
|
|
11999
12099
|
DEFAULT_RETRY_POLICY,
|
|
12000
|
-
InProcessKeystore,
|
|
12001
12100
|
IntegrityError,
|
|
12002
12101
|
KeyUnavailableError,
|
|
12003
12102
|
LEASE_SKEW_MS,
|
|
12004
12103
|
LeaseExpiredError,
|
|
12005
12104
|
LeaseLimitError,
|
|
12006
|
-
LocalFsStorage,
|
|
12007
12105
|
MAX_LEASES_PER_SEGMENT,
|
|
12008
12106
|
MAX_LEASE_MS,
|
|
12009
12107
|
MIN_EXPIRES_AT_MS,
|
|
12010
|
-
MemoryStorage,
|
|
12011
12108
|
NOOP_METRICS,
|
|
12012
|
-
NodeAead,
|
|
12013
12109
|
NotFoundError,
|
|
12014
12110
|
PinnedStorageChunkSource,
|
|
12015
12111
|
RecordingAuditSink,
|
|
@@ -12032,6 +12128,7 @@ export {
|
|
|
12032
12128
|
eraseNamespace,
|
|
12033
12129
|
errorKind,
|
|
12034
12130
|
excludingReservedRows,
|
|
12131
|
+
findIdUnderTombstone,
|
|
12035
12132
|
getSegmentRetention,
|
|
12036
12133
|
isStorageBackend,
|
|
12037
12134
|
judgeLoad,
|
|
@@ -12042,6 +12139,7 @@ export {
|
|
|
12042
12139
|
mapWithConcurrency,
|
|
12043
12140
|
readRetentionPolicy,
|
|
12044
12141
|
rebindCombineMany,
|
|
12142
|
+
refuseUnknownRefKeys,
|
|
12045
12143
|
releaseLease,
|
|
12046
12144
|
resolveBudget,
|
|
12047
12145
|
resolvePerOpBudget,
|
|
@@ -12050,6 +12148,7 @@ export {
|
|
|
12050
12148
|
runCombineMany,
|
|
12051
12149
|
runConsistencyCheck,
|
|
12052
12150
|
runExport,
|
|
12151
|
+
runPreflight,
|
|
12053
12152
|
safeMetrics,
|
|
12054
12153
|
segmentExists,
|
|
12055
12154
|
segmentKey,
|