@cloudbitmaps/core 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/chunk-7GT632UF.js +641 -0
- package/dist/chunk-7GT632UF.js.map +7 -0
- package/dist/chunk-CGILDXGP.js +73 -0
- package/dist/chunk-CGILDXGP.js.map +7 -0
- package/dist/chunk-GKHERVVQ.js +121 -0
- package/dist/chunk-GKHERVVQ.js.map +7 -0
- package/dist/{chunk-OSU7W2Y3.js → chunk-MVUD4R72.js} +200 -756
- package/dist/chunk-MVUD4R72.js.map +7 -0
- package/dist/core/audit.d.ts +53 -15
- package/dist/core/bit-route.d.ts +2 -0
- package/dist/core/blob.d.ts +4 -0
- package/dist/core/budget.d.ts +8 -3
- package/dist/core/check.d.ts +49 -0
- package/dist/core/combine-many.d.ts +11 -0
- package/dist/core/consistency.d.ts +11 -2
- package/dist/core/cooperative.d.ts +7 -0
- package/dist/core/crbm-storage-source.d.ts +7 -4
- package/dist/core/crypto.d.ts +5 -0
- package/dist/core/discover.d.ts +5 -3
- package/dist/core/engine.d.ts +4 -4
- package/dist/core/erase-id.d.ts +35 -12
- package/dist/core/erasure.d.ts +61 -27
- package/dist/core/errors.d.ts +30 -0
- package/dist/core/leases.d.ts +2 -0
- package/dist/core/load-input.d.ts +48 -0
- package/dist/core/load.d.ts +23 -2
- package/dist/core/metrics.d.ts +21 -0
- package/dist/core/pinned-storage-source.d.ts +27 -8
- package/dist/core/pointer-id.d.ts +9 -0
- package/dist/core/ports.d.ts +100 -7
- package/dist/core/preflight.d.ts +80 -0
- package/dist/core/retention-sweep.d.ts +60 -15
- package/dist/core/retention.d.ts +17 -6
- package/dist/core/retry.d.ts +1 -2
- package/dist/core/rollback.d.ts +22 -12
- package/dist/core/validate.d.ts +33 -2
- package/dist/driver-kit.d.ts +3 -3
- package/dist/driver-kit.js +61 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +24 -1
- package/dist/drivers/_shared/options.d.ts +15 -7
- package/dist/drivers/backends.d.ts +17 -2
- package/dist/drivers/crypto.d.ts +17 -2
- package/dist/export/index.d.ts +16 -2
- package/dist/index.d.ts +10 -9
- package/dist/index.js +1743 -1701
- package/dist/index.js.map +4 -4
- package/dist/node.d.ts +14 -0
- package/dist/node.js +1064 -0
- package/dist/node.js.map +7 -0
- package/package.json +11 -2
- package/dist/chunk-OSU7W2Y3.js.map +0 -7
package/dist/index.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
|
+
import {
|
|
2
|
+
boundedAll,
|
|
3
|
+
mapWithConcurrency
|
|
4
|
+
} from "./chunk-CGILDXGP.js";
|
|
5
|
+
import {
|
|
6
|
+
BufferReader,
|
|
7
|
+
KeptSegmentKeys,
|
|
8
|
+
chunkKeyUnder,
|
|
9
|
+
segmentKey,
|
|
10
|
+
segmentPrefix,
|
|
11
|
+
shardOf
|
|
12
|
+
} from "./chunk-GKHERVVQ.js";
|
|
1
13
|
import {
|
|
2
14
|
BudgetExceededError,
|
|
3
15
|
CHUNK_COUNT,
|
|
4
16
|
CONTENTION_NEXT_STEP,
|
|
5
17
|
CapabilityError,
|
|
6
18
|
CloudRoaringError,
|
|
7
|
-
DEFAULT_NAMESPACE,
|
|
8
19
|
DEFAULT_RETRY_POLICY,
|
|
9
20
|
FOOTER_BYTES,
|
|
10
21
|
IntegrityError,
|
|
@@ -20,6 +31,7 @@ import {
|
|
|
20
31
|
MAX_REMAINDER,
|
|
21
32
|
MAX_STORED_KNOWN_GENERATIONS,
|
|
22
33
|
NotFoundError,
|
|
34
|
+
PORT_KEYS,
|
|
23
35
|
PREAMBLE_BYTES,
|
|
24
36
|
RESERVED_NAMESPACE_PREFIX,
|
|
25
37
|
StaleOperandError,
|
|
@@ -31,23 +43,14 @@ import {
|
|
|
31
43
|
ValidationError,
|
|
32
44
|
WriteConflictError,
|
|
33
45
|
aadFor,
|
|
34
|
-
applyRegistryPatch,
|
|
35
46
|
ascending,
|
|
36
47
|
assertRegistryCanWrite,
|
|
37
|
-
|
|
38
|
-
brandAsBackend,
|
|
39
|
-
checkNow,
|
|
48
|
+
checkRequireEncryption,
|
|
40
49
|
checkedRetryPolicy,
|
|
41
50
|
compactFingerprint,
|
|
42
51
|
copiedMetadata,
|
|
43
|
-
decodeNameFromPath,
|
|
44
52
|
describingKnown,
|
|
45
|
-
drawIncarnation,
|
|
46
|
-
drawWrite,
|
|
47
|
-
encodeNameForKey,
|
|
48
|
-
encodeNameForPath,
|
|
49
53
|
encodedNameLength,
|
|
50
|
-
entropyIsAvailable,
|
|
51
54
|
errorKind,
|
|
52
55
|
fallbackBound,
|
|
53
56
|
fingerprintFor,
|
|
@@ -65,6 +68,7 @@ import {
|
|
|
65
68
|
isReservedNamespace,
|
|
66
69
|
isStorageBackend,
|
|
67
70
|
isTransientError,
|
|
71
|
+
isUnsettledConflict,
|
|
68
72
|
isValidationError,
|
|
69
73
|
isWriteConflictError,
|
|
70
74
|
jitteredWait,
|
|
@@ -72,24 +76,22 @@ import {
|
|
|
72
76
|
keptWindow,
|
|
73
77
|
knownGensShapeFault,
|
|
74
78
|
leaseChurn,
|
|
75
|
-
mapWithConcurrency,
|
|
76
79
|
metadataBytes,
|
|
77
80
|
metadataFromBytes,
|
|
78
81
|
metadataToCarry,
|
|
79
|
-
namespacePathPart,
|
|
80
|
-
newIncarnationToken,
|
|
81
82
|
nextKnown,
|
|
82
|
-
nextRegistryToken,
|
|
83
83
|
onlyLeasesDiffer,
|
|
84
84
|
ownSummary,
|
|
85
|
-
parseRegistryEnvelope,
|
|
86
85
|
pointerIdOf,
|
|
86
|
+
pointerSetBy,
|
|
87
87
|
pruneKnown,
|
|
88
|
-
|
|
88
|
+
refusalOfId,
|
|
89
89
|
refuseUnknownOptions,
|
|
90
|
+
refuseUnknownRefKeys,
|
|
90
91
|
releaseLease,
|
|
91
92
|
renewPointer,
|
|
92
93
|
renewsPointer,
|
|
94
|
+
requireBool,
|
|
93
95
|
requireFiniteNumber,
|
|
94
96
|
requireInteger,
|
|
95
97
|
requirePositiveInteger,
|
|
@@ -97,7 +99,6 @@ import {
|
|
|
97
99
|
rollbackKnown,
|
|
98
100
|
sameIncarnation,
|
|
99
101
|
sameKeys,
|
|
100
|
-
serializeRegistryEnvelope,
|
|
101
102
|
sizeOfFingerprint,
|
|
102
103
|
sizePart,
|
|
103
104
|
splitId,
|
|
@@ -109,23 +110,20 @@ import {
|
|
|
109
110
|
validateChunkKey,
|
|
110
111
|
validateChunkKeyOrder,
|
|
111
112
|
validateChunkRef,
|
|
112
|
-
validateNewRegistryRecord,
|
|
113
|
-
validateRegistryPatch,
|
|
114
113
|
validateSegmentRef,
|
|
115
114
|
validateUserNamespace,
|
|
116
115
|
validateUserRef,
|
|
117
|
-
webCryptoEntropy,
|
|
118
116
|
windowOf,
|
|
119
117
|
withRetry
|
|
120
|
-
} from "./chunk-
|
|
118
|
+
} from "./chunk-MVUD4R72.js";
|
|
121
119
|
|
|
122
120
|
// src/core/budget.ts
|
|
123
121
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
124
122
|
function resolveBudget(opt, fallback) {
|
|
125
123
|
if (opt === false) return null;
|
|
126
|
-
if (opt !== void 0
|
|
127
|
-
if (typeof opt !== "object" || Array.isArray(opt)) {
|
|
128
|
-
const got = Array.isArray(opt) ? "an array" : `a ${typeof opt}`;
|
|
124
|
+
if (opt !== void 0) {
|
|
125
|
+
if (opt === null || typeof opt !== "object" || Array.isArray(opt)) {
|
|
126
|
+
const got = opt === null ? "null" : Array.isArray(opt) ? "an array" : `a ${typeof opt}`;
|
|
129
127
|
throw new ValidationError(`budget must be { maxRequests } or false; got ${got}`);
|
|
130
128
|
}
|
|
131
129
|
const unknown = Object.keys(opt).filter(
|
|
@@ -137,12 +135,12 @@ function resolveBudget(opt, fallback) {
|
|
|
137
135
|
);
|
|
138
136
|
}
|
|
139
137
|
}
|
|
140
|
-
const maxRequests = opt?.maxRequests
|
|
138
|
+
const maxRequests = opt?.maxRequests === void 0 ? fallback.maxRequests : opt.maxRequests;
|
|
141
139
|
requirePositiveInteger(maxRequests, "budget.maxRequests");
|
|
142
140
|
return { maxRequests };
|
|
143
141
|
}
|
|
144
142
|
function resolvePerOpBudget(opt, storeBudget) {
|
|
145
|
-
if (opt === void 0
|
|
143
|
+
if (opt === void 0) return storeBudget;
|
|
146
144
|
return resolveBudget(opt, storeBudget ?? DEFAULT_BUDGET);
|
|
147
145
|
}
|
|
148
146
|
function checkBudget(budget, projectedRequests, op) {
|
|
@@ -175,8 +173,9 @@ async function refuseAbsent(op, candidates, limit, exists, nameOf) {
|
|
|
175
173
|
const absent = candidates.filter((_, i) => !found2[i]);
|
|
176
174
|
if (absent.length === 0) return;
|
|
177
175
|
const named = absent.map((c) => `"${nameOf(c)}"`).join(", ");
|
|
176
|
+
const [operand, names, does, was, it] = absent.length > 1 ? ["operands", "name segments", "do", "were", "they"] : ["operand", "names a segment", "does", "was", "it"];
|
|
178
177
|
throw new ValidationError(
|
|
179
|
-
`${op}: operand ${named} names
|
|
178
|
+
`${op}: ${operand} ${named} ${names} that ${does} not exist or ${was} dropped, so ${it} would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
180
179
|
);
|
|
181
180
|
}
|
|
182
181
|
|
|
@@ -389,17 +388,17 @@ var AEAD_TAG_BYTES = 16;
|
|
|
389
388
|
var DEFAULT_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BITMAP_BYTES + AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
|
|
390
389
|
|
|
391
390
|
// src/core/id-window.ts
|
|
392
|
-
function assertIdBound(name, bound) {
|
|
391
|
+
function assertIdBound(name, bound, op) {
|
|
393
392
|
if (bound !== void 0 && (!Number.isInteger(bound) || bound < 0 || bound > U32_MAX)) {
|
|
394
|
-
throw new ValidationError(`${name} must be an integer from 0 to ${U32_MAX}`);
|
|
393
|
+
throw new ValidationError(`${op}: ${name} must be an integer from 0 to ${U32_MAX}`);
|
|
395
394
|
}
|
|
396
395
|
}
|
|
397
|
-
function windowOf2(range) {
|
|
396
|
+
function windowOf2(range, op) {
|
|
398
397
|
const after = range?.after;
|
|
399
398
|
const through = range?.through;
|
|
400
399
|
if (after === void 0 && through === void 0) return null;
|
|
401
|
-
assertIdBound("after", after);
|
|
402
|
-
assertIdBound("through", through);
|
|
400
|
+
assertIdBound("after", after, op);
|
|
401
|
+
assertIdBound("through", through, op);
|
|
403
402
|
const lo = after === void 0 ? 0 : after + 1;
|
|
404
403
|
const hi = through ?? U32_MAX;
|
|
405
404
|
if (lo > hi) return "empty";
|
|
@@ -525,62 +524,6 @@ var InvalidationScope = class extends InvalidationRecord {
|
|
|
525
524
|
}
|
|
526
525
|
};
|
|
527
526
|
|
|
528
|
-
// src/core/keys.ts
|
|
529
|
-
var FIELD = " ";
|
|
530
|
-
var NO_NAMESPACE = "/";
|
|
531
|
-
function segmentKey(ref) {
|
|
532
|
-
const ns = ref.namespace === void 0 ? NO_NAMESPACE : encodeNameForKey(ref.namespace);
|
|
533
|
-
return `${ns}${FIELD}${encodeNameForKey(ref.segment)}`;
|
|
534
|
-
}
|
|
535
|
-
function segmentPrefix(ref) {
|
|
536
|
-
return `${segmentKey(ref)}${FIELD}`;
|
|
537
|
-
}
|
|
538
|
-
var KEPT_SEGMENT_KEYS = 1024;
|
|
539
|
-
var KeptSegmentKeys = class {
|
|
540
|
-
constructor(encode, max = KEPT_SEGMENT_KEYS) {
|
|
541
|
-
this.encode = encode;
|
|
542
|
-
this.max = max;
|
|
543
|
-
}
|
|
544
|
-
encode;
|
|
545
|
-
max;
|
|
546
|
-
byNamespace = /* @__PURE__ */ new Map();
|
|
547
|
-
kept = 0;
|
|
548
|
-
/** The segment's encoded key. */
|
|
549
|
-
of(ref) {
|
|
550
|
-
let names = this.byNamespace.get(ref.namespace);
|
|
551
|
-
const found2 = names?.get(ref.segment);
|
|
552
|
-
if (found2 !== void 0) return found2;
|
|
553
|
-
const encoded = this.encode(ref);
|
|
554
|
-
if (this.kept >= this.max) {
|
|
555
|
-
this.byNamespace.clear();
|
|
556
|
-
this.kept = 0;
|
|
557
|
-
names = void 0;
|
|
558
|
-
}
|
|
559
|
-
if (names === void 0) {
|
|
560
|
-
names = /* @__PURE__ */ new Map();
|
|
561
|
-
this.byNamespace.set(ref.namespace, names);
|
|
562
|
-
}
|
|
563
|
-
names.set(ref.segment, encoded);
|
|
564
|
-
this.kept += 1;
|
|
565
|
-
return encoded;
|
|
566
|
-
}
|
|
567
|
-
/** How many encoded keys are kept. */
|
|
568
|
-
get size() {
|
|
569
|
-
return this.kept;
|
|
570
|
-
}
|
|
571
|
-
};
|
|
572
|
-
function chunkKeyUnder(prefix, chunkKey, version) {
|
|
573
|
-
return version === void 0 ? `${prefix}${chunkKey}` : `${prefix}${chunkKey}${FIELD}${version}`;
|
|
574
|
-
}
|
|
575
|
-
function shardOf(key, totalShards) {
|
|
576
|
-
let h = 2166136261;
|
|
577
|
-
for (let i = 0; i < key.length; i++) {
|
|
578
|
-
h ^= key.charCodeAt(i);
|
|
579
|
-
h = Math.imul(h, 16777619);
|
|
580
|
-
}
|
|
581
|
-
return (h >>> 0) % totalShards;
|
|
582
|
-
}
|
|
583
|
-
|
|
584
527
|
// src/core/audit.ts
|
|
585
528
|
var NOOP_AUDIT = {
|
|
586
529
|
onEvent() {
|
|
@@ -613,6 +556,7 @@ function ignoreRejection(returned) {
|
|
|
613
556
|
}
|
|
614
557
|
var RecordingAuditSink = class {
|
|
615
558
|
recorded = [];
|
|
559
|
+
/** Records the event, in emission order. */
|
|
616
560
|
onEvent(event) {
|
|
617
561
|
this.recorded.push(event);
|
|
618
562
|
}
|
|
@@ -622,6 +566,7 @@ var RecordingAuditSink = class {
|
|
|
622
566
|
(e) => e.kind === "segment.collect" ? { ...e, collected: [...e.collected] } : { ...e }
|
|
623
567
|
);
|
|
624
568
|
}
|
|
569
|
+
/** Forgets every recorded event. */
|
|
625
570
|
reset() {
|
|
626
571
|
this.recorded.length = 0;
|
|
627
572
|
}
|
|
@@ -667,6 +612,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
667
612
|
for (const name of OP_NAMES) ops[name] = { count: 0, totalMs: 0 };
|
|
668
613
|
return ops;
|
|
669
614
|
}
|
|
615
|
+
/** Adds the event to the totals. An `advisory` event is a note, not a quantity, and adds nothing. */
|
|
670
616
|
onEvent(event) {
|
|
671
617
|
switch (event.kind) {
|
|
672
618
|
case "storage.get":
|
|
@@ -701,6 +647,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
701
647
|
}
|
|
702
648
|
}
|
|
703
649
|
}
|
|
650
|
+
/** A copy of the totals so far, which later events do not change. */
|
|
704
651
|
snapshot() {
|
|
705
652
|
const ops = /* @__PURE__ */ Object.create(null);
|
|
706
653
|
for (const name of OP_NAMES) ops[name] = { ...this.ops[name] };
|
|
@@ -712,6 +659,7 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
712
659
|
ops
|
|
713
660
|
};
|
|
714
661
|
}
|
|
662
|
+
/** Sets every total back to zero. */
|
|
715
663
|
reset() {
|
|
716
664
|
this.storage = { gets: 0, bytes: 0, totalMs: 0 };
|
|
717
665
|
this.cache = { hits: 0, misses: 0 };
|
|
@@ -1003,7 +951,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1003
951
|
}
|
|
1004
952
|
}
|
|
1005
953
|
async *iterateRange(seg, range) {
|
|
1006
|
-
const w = windowOf2(range) ?? WHOLE_ID_SPACE;
|
|
954
|
+
const w = windowOf2(range, "iterate") ?? WHOLE_ID_SPACE;
|
|
1007
955
|
if (w === "empty") return;
|
|
1008
956
|
const run = await this.planIterate(seg, w);
|
|
1009
957
|
try {
|
|
@@ -1027,7 +975,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1027
975
|
* 65,536 ids (256 KiB) and is the caller's to keep.
|
|
1028
976
|
*/
|
|
1029
977
|
async *iterateBatches(seg, range) {
|
|
1030
|
-
const w = windowOf2(range);
|
|
978
|
+
const w = windowOf2(range, "iterate");
|
|
1031
979
|
if (w === "empty") return;
|
|
1032
980
|
const run = await this.planIterate(seg, w);
|
|
1033
981
|
try {
|
|
@@ -1167,9 +1115,9 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1167
1115
|
* `after` turn out to be fewer than `n`, and one that cuts its last chunk may read it and yield nothing.
|
|
1168
1116
|
*/
|
|
1169
1117
|
async *everyNth(seg, n, range) {
|
|
1170
|
-
requirePositiveInteger(n, "n");
|
|
1118
|
+
requirePositiveInteger(n, "everyNth: n");
|
|
1171
1119
|
const epoch = this.scope.epoch;
|
|
1172
|
-
const w = windowOf2(range);
|
|
1120
|
+
const w = windowOf2(range, "everyNth");
|
|
1173
1121
|
if (w === "empty") return;
|
|
1174
1122
|
const span = w ?? WHOLE_ID_SPACE;
|
|
1175
1123
|
const { loKey, loRem, hiKey, hiRem } = span;
|
|
@@ -1207,10 +1155,10 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1207
1155
|
const hits = [];
|
|
1208
1156
|
let before = 0;
|
|
1209
1157
|
for (const key of keys) {
|
|
1210
|
-
const
|
|
1211
|
-
const size = counts.get(key) -
|
|
1158
|
+
const skipped2 = key === loKey && cutsFirst ? below : 0;
|
|
1159
|
+
const size = counts.get(key) - skipped2;
|
|
1212
1160
|
const next = (Math.floor(before / n) + 1) * n;
|
|
1213
|
-
if (next <= before + size) hits.push({ key, index:
|
|
1161
|
+
if (next <= before + size) hits.push({ key, index: skipped2 + next - before - 1 });
|
|
1214
1162
|
before += size;
|
|
1215
1163
|
}
|
|
1216
1164
|
if (hits.length === 0) return;
|
|
@@ -1324,33 +1272,30 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1324
1272
|
* for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
|
|
1325
1273
|
* errors are the id reads' own.
|
|
1326
1274
|
*/
|
|
1327
|
-
intersectChunks(segs, options) {
|
|
1328
|
-
return this.combineChunks(segs, options?.exclude ?? [], "all", "intersect", options);
|
|
1275
|
+
intersectChunks(segs, options, label) {
|
|
1276
|
+
return this.combineChunks(segs, options?.exclude ?? [], "all", "intersect", options, label);
|
|
1329
1277
|
}
|
|
1330
1278
|
/** {@link union}, as chunks: see {@link intersectChunks}. */
|
|
1331
|
-
unionChunks(segs, options) {
|
|
1332
|
-
return this.combineChunks(segs, options?.exclude ?? [], "any", "union", options);
|
|
1279
|
+
unionChunks(segs, options, label) {
|
|
1280
|
+
return this.combineChunks(segs, options?.exclude ?? [], "any", "union", options, label);
|
|
1333
1281
|
}
|
|
1334
1282
|
/** {@link andNot}, as chunks: see {@link intersectChunks}. */
|
|
1335
|
-
andNotChunks(seg, excludes, options) {
|
|
1336
|
-
return this.combineChunks([seg], excludes, "all", "andNot", options);
|
|
1283
|
+
andNotChunks(seg, excludes, options, label) {
|
|
1284
|
+
return this.combineChunks([seg], excludes, "all", "andNot", options, label);
|
|
1337
1285
|
}
|
|
1338
1286
|
/**
|
|
1339
1287
|
* Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
|
|
1340
1288
|
* absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
|
|
1341
1289
|
* read and the batch read share it, so the two cannot disagree about what is read or charged.
|
|
1342
1290
|
*/
|
|
1343
|
-
async combinePlan(segs, excludeSegs, mode, op, options) {
|
|
1291
|
+
async combinePlan(segs, excludeSegs, mode, op, options, label = op) {
|
|
1344
1292
|
if (segs.length === 0) {
|
|
1345
1293
|
throw new ValidationError(`${op} requires at least one segment`);
|
|
1346
1294
|
}
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
}
|
|
1350
|
-
const limit = options?.concurrency ?? DEFAULT_INTERSECT_CONCURRENCY;
|
|
1351
|
-
requirePositiveInteger(limit, "concurrency");
|
|
1295
|
+
const limit = options?.concurrency === void 0 ? DEFAULT_INTERSECT_CONCURRENCY : options.concurrency;
|
|
1296
|
+
requirePositiveInteger(limit, `${label}: concurrency`);
|
|
1352
1297
|
const budget = resolvePerOpBudget(options?.budget, this.budget);
|
|
1353
|
-
const w = windowOf2(options);
|
|
1298
|
+
const w = windowOf2(options, label);
|
|
1354
1299
|
if (w === "empty") return null;
|
|
1355
1300
|
const epoch = this.scope.epoch;
|
|
1356
1301
|
const [operands, excludes] = await this.combineOperands(segs, excludeSegs, w, -1);
|
|
@@ -1412,7 +1357,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1412
1357
|
const streams = [];
|
|
1413
1358
|
const batches = [];
|
|
1414
1359
|
const streamExcludes = mode === "any" || operands.length === 1;
|
|
1415
|
-
const
|
|
1360
|
+
const open = (o, wanted) => {
|
|
1416
1361
|
o.streamed = this.openStreamed(
|
|
1417
1362
|
o.seg,
|
|
1418
1363
|
wanted === void 0 ? keys : keys.filter(wanted),
|
|
@@ -1424,8 +1369,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1424
1369
|
);
|
|
1425
1370
|
if (o.streamed) streams.push(o.streamed);
|
|
1426
1371
|
};
|
|
1427
|
-
for (const o of operands)
|
|
1428
|
-
if (streamExcludes) for (const e of excludes)
|
|
1372
|
+
for (const o of operands) open(o, mode === "all" ? void 0 : (k) => o.keys.has(k));
|
|
1373
|
+
if (streamExcludes) for (const e of excludes) open(e, (k) => e.keys.has(k));
|
|
1429
1374
|
else {
|
|
1430
1375
|
const shared = /* @__PURE__ */ new Map();
|
|
1431
1376
|
for (const e of excludes) this.openBatch(e, moves, limit, batches, shared);
|
|
@@ -1541,8 +1486,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1541
1486
|
}
|
|
1542
1487
|
}
|
|
1543
1488
|
/** The chunk read: the plan every combine shares, then each key's result, cut to the range at its edges. */
|
|
1544
|
-
async *combineChunks(segs, excludeSegs, mode, op, options) {
|
|
1545
|
-
const run = await this.combinePlan(segs, excludeSegs, mode, op, options);
|
|
1489
|
+
async *combineChunks(segs, excludeSegs, mode, op, options, label) {
|
|
1490
|
+
const run = await this.combinePlan(segs, excludeSegs, mode, op, options, label);
|
|
1546
1491
|
if (run === null) return;
|
|
1547
1492
|
const { w } = run;
|
|
1548
1493
|
try {
|
|
@@ -1651,8 +1596,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1651
1596
|
askBatch(batch, key) {
|
|
1652
1597
|
if (batch.closed)
|
|
1653
1598
|
return Promise.reject(new Error("the combine this read belongs to has stopped"));
|
|
1654
|
-
return new Promise((
|
|
1655
|
-
batch.pending.push({ key, resolve
|
|
1599
|
+
return new Promise((resolve, reject) => {
|
|
1600
|
+
batch.pending.push({ key, resolve, reject });
|
|
1656
1601
|
batch.held.add(key);
|
|
1657
1602
|
batch.oldest = Math.min(batch.oldest, batch.window?.slotOf(key) ?? 0);
|
|
1658
1603
|
if (batch.held.size >= batch.half) this.flushBatch(batch);
|
|
@@ -1725,8 +1670,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
1725
1670
|
const streams = /* @__PURE__ */ new Set();
|
|
1726
1671
|
const streamKeys = [];
|
|
1727
1672
|
for (const ask of asks) {
|
|
1728
|
-
const
|
|
1729
|
-
if (!
|
|
1673
|
+
const open = this.scope.hasOpenRead(this.chunkCacheKey({ ...seg, chunkKey: ask.key }, gen));
|
|
1674
|
+
if (!open && streamKeys[streamKeys.length - 1] !== ask.key) {
|
|
1730
1675
|
streams.add(ask);
|
|
1731
1676
|
streamKeys.push(ask.key);
|
|
1732
1677
|
}
|
|
@@ -2000,7 +1945,7 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
2000
1945
|
* the read listed the segment's keys: the read then lists them again, and caches as a read begun after the call.
|
|
2001
1946
|
*
|
|
2002
1947
|
* The decoded-chunk cache is keyed by generation, which handles a *publish* (the new generation misses) but
|
|
2003
|
-
* not a *destruction*: a segment that was erased from, dropped,
|
|
1948
|
+
* not a *destruction*: a segment that was erased from, dropped, destroyed or retired has no newer generation
|
|
2004
1949
|
* whose key would miss, so its decoded chunks stay resident and keep answering. That is why the erasing
|
|
2005
1950
|
* process itself could report `erased: true` and then still answer `true` for the same id, out of RAM, with
|
|
2006
1951
|
* no storage read to intercept.
|
|
@@ -2065,8 +2010,8 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
2065
2010
|
}
|
|
2066
2011
|
if (report && this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
2067
2012
|
}
|
|
2068
|
-
const
|
|
2069
|
-
if (
|
|
2013
|
+
const open = this.scope.openRead(cacheKey);
|
|
2014
|
+
if (open) return open;
|
|
2070
2015
|
const token = {};
|
|
2071
2016
|
const entry = { token, read: this.fetchChunk(ref, gen, cacheKey, token) };
|
|
2072
2017
|
this.scope.beginRead(cacheKey, entry);
|
|
@@ -2374,8 +2319,8 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2374
2319
|
const ext = format === "roaring" ? ".roaring" : ".ndjson";
|
|
2375
2320
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
2376
2321
|
const segments = [];
|
|
2377
|
-
const
|
|
2378
|
-
const
|
|
2322
|
+
const failed2 = [];
|
|
2323
|
+
const skipped2 = [];
|
|
2379
2324
|
let totalIds = 0;
|
|
2380
2325
|
if (typeof reader?.segment !== "function") {
|
|
2381
2326
|
throw new ValidationError(
|
|
@@ -2393,7 +2338,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2393
2338
|
await writer.abort?.();
|
|
2394
2339
|
} catch {
|
|
2395
2340
|
}
|
|
2396
|
-
|
|
2341
|
+
skipped2.push({ segment: ref.segment, namespace: ref.namespace, reason: "destroyed" });
|
|
2397
2342
|
};
|
|
2398
2343
|
const exportRef = async (ref) => {
|
|
2399
2344
|
let writer;
|
|
@@ -2451,13 +2396,13 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2451
2396
|
} catch {
|
|
2452
2397
|
}
|
|
2453
2398
|
}
|
|
2454
|
-
|
|
2399
|
+
failed2.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2455
2400
|
return;
|
|
2456
2401
|
}
|
|
2457
2402
|
try {
|
|
2458
2403
|
await writer.close();
|
|
2459
2404
|
} catch (err) {
|
|
2460
|
-
|
|
2405
|
+
failed2.push({ segment: ref.segment, namespace: ref.namespace, error: errMessage(err) });
|
|
2461
2406
|
return;
|
|
2462
2407
|
}
|
|
2463
2408
|
segments.push({ segment: ref.segment, namespace: ref.namespace, count, bytes });
|
|
@@ -2466,7 +2411,7 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2466
2411
|
for await (const rec of registry.list(options.namespace)) {
|
|
2467
2412
|
if (options.namespace === void 0 && isReservedRow(rec)) continue;
|
|
2468
2413
|
if (rec.status === "destroyed") {
|
|
2469
|
-
|
|
2414
|
+
skipped2.push({ segment: rec.segment, namespace: rec.namespace, reason: "destroyed" });
|
|
2470
2415
|
continue;
|
|
2471
2416
|
}
|
|
2472
2417
|
await exportRef({ segment: rec.segment, namespace: rec.namespace });
|
|
@@ -2477,14 +2422,14 @@ async function runExport(reader, registry, sink, options = {}) {
|
|
|
2477
2422
|
totalSegments: segments.length,
|
|
2478
2423
|
totalIds,
|
|
2479
2424
|
segments,
|
|
2480
|
-
failed,
|
|
2481
|
-
skipped
|
|
2425
|
+
failed: failed2,
|
|
2426
|
+
skipped: skipped2
|
|
2482
2427
|
};
|
|
2483
2428
|
}
|
|
2484
2429
|
|
|
2485
2430
|
// src/core/discover.ts
|
|
2486
2431
|
async function segmentExists(ref, registry) {
|
|
2487
|
-
validateUserRef(ref);
|
|
2432
|
+
validateUserRef(ref, "exists");
|
|
2488
2433
|
const record = await registry.get(ref);
|
|
2489
2434
|
return record !== null && record.status !== "destroyed" && record.currentGen !== null;
|
|
2490
2435
|
}
|
|
@@ -2502,1105 +2447,210 @@ async function* listSegments(registry, options = {}) {
|
|
|
2502
2447
|
}
|
|
2503
2448
|
}
|
|
2504
2449
|
|
|
2505
|
-
// src/
|
|
2506
|
-
|
|
2450
|
+
// src/core/reader-defaults.ts
|
|
2451
|
+
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
2452
|
+
var REFRESH_RETRY_MS = 500;
|
|
2453
|
+
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
2454
|
+
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
2455
|
+
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
2456
|
+
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
2507
2457
|
|
|
2508
|
-
// src/
|
|
2509
|
-
|
|
2458
|
+
// src/core/crbm/integrity.ts
|
|
2459
|
+
var crbmIntegrity = (message) => new IntegrityError(`${message}; re-load from source or roll back`);
|
|
2510
2460
|
|
|
2511
|
-
// src/core/
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
length = 0;
|
|
2520
|
-
write(bytes) {
|
|
2521
|
-
this.chunks.push(new Uint8Array(bytes));
|
|
2522
|
-
this.length += bytes.length;
|
|
2523
|
-
return Promise.resolve();
|
|
2524
|
-
}
|
|
2525
|
-
bytes() {
|
|
2526
|
-
const out = new Uint8Array(this.length);
|
|
2527
|
-
let offset = 0;
|
|
2528
|
-
for (const chunk of this.chunks) {
|
|
2529
|
-
out.set(chunk, offset);
|
|
2530
|
-
offset += chunk.length;
|
|
2531
|
-
}
|
|
2532
|
-
return out;
|
|
2533
|
-
}
|
|
2534
|
-
};
|
|
2535
|
-
var BufferReader = class {
|
|
2536
|
-
constructor(buffer) {
|
|
2537
|
-
this.buffer = buffer;
|
|
2538
|
-
}
|
|
2539
|
-
buffer;
|
|
2540
|
-
getRange(offset, length) {
|
|
2541
|
-
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0 || offset + length > this.buffer.length) {
|
|
2542
|
-
throw new ValidationError(
|
|
2543
|
-
`range [${offset}, ${offset + length}) out of bounds for ${this.buffer.length}-byte blob`
|
|
2544
|
-
);
|
|
2545
|
-
}
|
|
2546
|
-
return Promise.resolve(this.buffer.subarray(offset, offset + length));
|
|
2461
|
+
// src/core/crbm/crc32c.ts
|
|
2462
|
+
var TABLES = (() => {
|
|
2463
|
+
const POLY = 2197175160;
|
|
2464
|
+
const t0 = new Uint32Array(256);
|
|
2465
|
+
for (let n = 0; n < 256; n++) {
|
|
2466
|
+
let c = n;
|
|
2467
|
+
for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
|
|
2468
|
+
t0[n] = c >>> 0;
|
|
2547
2469
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2470
|
+
const tables = [t0];
|
|
2471
|
+
for (let k = 1; k < 8; k++) {
|
|
2472
|
+
const prev = tables[k - 1];
|
|
2473
|
+
const next = new Uint32Array(256);
|
|
2474
|
+
for (let n = 0; n < 256; n++) {
|
|
2475
|
+
const v = prev[n];
|
|
2476
|
+
next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
|
|
2553
2477
|
}
|
|
2554
|
-
|
|
2555
|
-
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
2556
|
-
return Promise.resolve({ bytes: this.buffer.subarray(size - take), size });
|
|
2557
|
-
}
|
|
2558
|
-
};
|
|
2559
|
-
|
|
2560
|
-
// src/drivers/memory.ts
|
|
2561
|
-
var MemoryRegistryDriver = class {
|
|
2562
|
-
rows = /* @__PURE__ */ new Map();
|
|
2563
|
-
now;
|
|
2564
|
-
entropy;
|
|
2565
|
-
seq = 0;
|
|
2566
|
-
constructor(options = {}) {
|
|
2567
|
-
this.now = options.now ?? (() => Date.now());
|
|
2568
|
-
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
2569
|
-
}
|
|
2570
|
-
nextToken(incarnation) {
|
|
2571
|
-
this.seq += 1;
|
|
2572
|
-
return `${incarnation}.${this.seq}.${drawWrite(this.entropy)}`;
|
|
2573
|
-
}
|
|
2574
|
-
/** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
|
|
2575
|
-
capabilities() {
|
|
2576
|
-
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
2577
|
-
}
|
|
2578
|
-
/** Every write reports its token to `onToken` before it is applied. */
|
|
2579
|
-
get reportsWriteTokens() {
|
|
2580
|
-
return true;
|
|
2581
|
-
}
|
|
2582
|
-
async get(ref) {
|
|
2583
|
-
validateSegmentRef(ref);
|
|
2584
|
-
const row = this.rows.get(segmentKey(ref));
|
|
2585
|
-
return row ? structuredClone(row) : null;
|
|
2478
|
+
tables.push(next);
|
|
2586
2479
|
}
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2480
|
+
return tables;
|
|
2481
|
+
})();
|
|
2482
|
+
var T0 = TABLES[0];
|
|
2483
|
+
var T1 = TABLES[1];
|
|
2484
|
+
var T2 = TABLES[2];
|
|
2485
|
+
var T3 = TABLES[3];
|
|
2486
|
+
var T4 = TABLES[4];
|
|
2487
|
+
var T5 = TABLES[5];
|
|
2488
|
+
var T6 = TABLES[6];
|
|
2489
|
+
var T7 = TABLES[7];
|
|
2490
|
+
function crc32c(bytes, seed = 0) {
|
|
2491
|
+
let crc = (seed ^ 4294967295) >>> 0;
|
|
2492
|
+
let i = 0;
|
|
2493
|
+
const n8 = bytes.length - bytes.length % 8;
|
|
2494
|
+
for (; i < n8; i += 8) {
|
|
2495
|
+
crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
|
|
2496
|
+
crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
|
|
2598
2497
|
}
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
* as the write, in memory, so a stale one fails exactly as a lost race does.
|
|
2602
|
-
*/
|
|
2603
|
-
async compareAndSwap(ref, expected, patch, options) {
|
|
2604
|
-
validateSegmentRef(ref);
|
|
2605
|
-
const checked = validateRegistryPatch(patch);
|
|
2606
|
-
const key = segmentKey(ref);
|
|
2607
|
-
const existing = this.rows.get(key);
|
|
2608
|
-
if (!existing || existing.token !== expected) {
|
|
2609
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2610
|
-
}
|
|
2611
|
-
const token = this.nextToken(incarnationOf(existing.token));
|
|
2612
|
-
options?.onToken?.(token);
|
|
2613
|
-
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
2614
|
-
return { token };
|
|
2498
|
+
for (; i < bytes.length; i++) {
|
|
2499
|
+
crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
|
|
2615
2500
|
}
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2501
|
+
return (crc ^ 4294967295) >>> 0;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
// src/core/crbm/plan-reads.ts
|
|
2505
|
+
var MAX_COALESCE_GAP_BYTES = 256 * 1024;
|
|
2506
|
+
var MAX_COALESCED_READ_BYTES = DEFAULT_MAX_BITMAP_BYTES;
|
|
2507
|
+
var MAX_RANGES_IN_FLIGHT = 32;
|
|
2508
|
+
function* planChunkReads(extents, region) {
|
|
2509
|
+
let chunks = [];
|
|
2510
|
+
let start = 0;
|
|
2511
|
+
let end = 0;
|
|
2512
|
+
let previousEnd = region.start;
|
|
2513
|
+
for (const extent of extents) {
|
|
2514
|
+
const extentEnd = extent.offset + extent.length;
|
|
2515
|
+
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
2516
|
+
throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
|
|
2620
2517
|
}
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2518
|
+
previousEnd = extentEnd;
|
|
2519
|
+
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
2520
|
+
chunks.push(extent);
|
|
2521
|
+
end = extentEnd;
|
|
2522
|
+
continue;
|
|
2627
2523
|
}
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
objects = /* @__PURE__ */ new Map();
|
|
2633
|
-
stored = 0;
|
|
2634
|
-
capabilities() {
|
|
2635
|
-
return {
|
|
2636
|
-
rangeRead: true,
|
|
2637
|
-
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
2638
|
-
conditionalPut: true,
|
|
2639
|
-
conditionalDelete: true
|
|
2640
|
-
};
|
|
2641
|
-
}
|
|
2642
|
-
async putImmutable(key, write) {
|
|
2643
|
-
const k = genObjectKey(key);
|
|
2644
|
-
const sink = new BufferSink();
|
|
2645
|
-
await write(sink);
|
|
2646
|
-
const body = sink.bytes();
|
|
2647
|
-
if (this.objects.has(k)) {
|
|
2648
|
-
throw new WriteConflictError(
|
|
2649
|
-
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
2650
|
-
);
|
|
2524
|
+
if (chunks.length > 0) {
|
|
2525
|
+
const done = chunks;
|
|
2526
|
+
chunks = [];
|
|
2527
|
+
yield { offset: start, length: end - start, chunks: done };
|
|
2651
2528
|
}
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2529
|
+
chunks.push(extent);
|
|
2530
|
+
start = extent.offset;
|
|
2531
|
+
end = extentEnd;
|
|
2655
2532
|
}
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
}
|
|
2665
|
-
return body.slice(offset, offset + length);
|
|
2533
|
+
if (chunks.length > 0) yield { offset: start, length: end - start, chunks };
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
// src/core/crbm/varint.ts
|
|
2537
|
+
var U32_CEIL = U32_MAX + 1;
|
|
2538
|
+
function varintLength(value) {
|
|
2539
|
+
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
2540
|
+
throw new RangeError(`varint value must be a u32; got ${value}`);
|
|
2666
2541
|
}
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2542
|
+
if (value < 128) return 1;
|
|
2543
|
+
if (value < 16384) return 2;
|
|
2544
|
+
if (value < 2097152) return 3;
|
|
2545
|
+
if (value < 268435456) return 4;
|
|
2546
|
+
return 5;
|
|
2547
|
+
}
|
|
2548
|
+
function writeVarintAt(out, pos, value) {
|
|
2549
|
+
varintLength(value);
|
|
2550
|
+
let v = value;
|
|
2551
|
+
let at = pos;
|
|
2552
|
+
while (v > 127) {
|
|
2553
|
+
out[at++] = v & 127 | 128;
|
|
2554
|
+
v >>>= 7;
|
|
2672
2555
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2556
|
+
out[at++] = v;
|
|
2557
|
+
return at;
|
|
2558
|
+
}
|
|
2559
|
+
function readVarint(bytes, offset) {
|
|
2560
|
+
let result = 0;
|
|
2561
|
+
let shift = 0;
|
|
2562
|
+
let pos = offset;
|
|
2563
|
+
for (let i = 0; i < 5; i++) {
|
|
2564
|
+
if (pos >= bytes.length) {
|
|
2565
|
+
throw crbmIntegrity("varint truncated past end of buffer");
|
|
2681
2566
|
}
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
const prefix = `${segmentKey(ref)} `;
|
|
2687
|
-
for (const k of this.objects.keys()) {
|
|
2688
|
-
if (k.startsWith(prefix)) {
|
|
2689
|
-
yield {
|
|
2690
|
-
namespace: ref.namespace,
|
|
2691
|
-
segment: ref.segment,
|
|
2692
|
-
generation: Number(k.slice(prefix.length))
|
|
2693
|
-
};
|
|
2694
|
-
}
|
|
2567
|
+
const byte = bytes[pos];
|
|
2568
|
+
pos += 1;
|
|
2569
|
+
if (i === 4 && byte > 15) {
|
|
2570
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
2695
2571
|
}
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
if (stored === void 0) {
|
|
2700
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
2572
|
+
result = (result | (byte & 127) << shift) >>> 0;
|
|
2573
|
+
if ((byte & 128) === 0) {
|
|
2574
|
+
return { value: result, next: pos };
|
|
2701
2575
|
}
|
|
2702
|
-
|
|
2576
|
+
shift += 7;
|
|
2703
2577
|
}
|
|
2704
|
-
|
|
2705
|
-
function genObjectKey(key) {
|
|
2706
|
-
validateSegmentRef(key);
|
|
2707
|
-
requireInteger(key.generation, { min: 0, what: "generation" });
|
|
2708
|
-
return `${segmentKey(key)} ${key.generation}`;
|
|
2578
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
2709
2579
|
}
|
|
2710
2580
|
|
|
2711
|
-
// src/
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
// src/drivers/localfs/paths.ts
|
|
2718
|
-
import { join } from "node:path";
|
|
2719
|
-
var MAX_COMPONENT_BYTES = 255;
|
|
2720
|
-
var TEMP_SUFFIX_BYTES = 1 + 36 + 4;
|
|
2721
|
-
function assertFits(component, extraBytes) {
|
|
2722
|
-
if (component.length + extraBytes > MAX_COMPONENT_BYTES) {
|
|
2723
|
-
throw new ValidationError(
|
|
2724
|
-
`a name is too long for a file name on this backend: the namespace, or the segment with its generation and the temp file's suffix, must fit in ${MAX_COMPONENT_BYTES} bytes once encoded`
|
|
2725
|
-
);
|
|
2726
|
-
}
|
|
2581
|
+
// src/core/crbm/reader.ts
|
|
2582
|
+
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
2583
|
+
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
2584
|
+
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
2585
|
+
function indexCapacity(indexLength) {
|
|
2586
|
+
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
2727
2587
|
}
|
|
2728
|
-
function
|
|
2729
|
-
|
|
2730
|
-
assertFits(namespacePathPart(key.namespace), 0);
|
|
2731
|
-
assertFits(storageObjectFilename(key.segment, key.generation), TEMP_SUFFIX_BYTES);
|
|
2732
|
-
}
|
|
2733
|
-
function assertRegistryNamesFit(ref) {
|
|
2734
|
-
validateSegmentRef(ref);
|
|
2735
|
-
assertFits(namespacePathPart(ref.namespace), 0);
|
|
2736
|
-
assertFits(`${encodeNameForPath(ref.segment)}${REGISTRY_SUFFIX}`, TEMP_SUFFIX_BYTES);
|
|
2737
|
-
}
|
|
2738
|
-
function segmentsDir(root, ref) {
|
|
2739
|
-
validateSegmentRef(ref);
|
|
2740
|
-
return join(root, namespacePathPart(ref.namespace), "segments");
|
|
2741
|
-
}
|
|
2742
|
-
function storageObjectPath(root, key) {
|
|
2743
|
-
validateSegmentRef(key);
|
|
2744
|
-
requireInteger(key.generation, { min: 0, what: "generation" });
|
|
2745
|
-
return join(segmentsDir(root, key), storageObjectFilename(key.segment, key.generation));
|
|
2746
|
-
}
|
|
2747
|
-
function storageObjectFilename(segment, generation) {
|
|
2748
|
-
return `${encodeNameForPath(segment)}.${generation}.crbm`;
|
|
2749
|
-
}
|
|
2750
|
-
function parseGeneration(segment, filename) {
|
|
2751
|
-
const prefix = `${encodeNameForPath(segment)}.`;
|
|
2752
|
-
const suffix = ".crbm";
|
|
2753
|
-
if (!filename.startsWith(prefix) || !filename.endsWith(suffix)) return null;
|
|
2754
|
-
const middle = filename.slice(prefix.length, filename.length - suffix.length);
|
|
2755
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
2756
|
-
const gen = Number(middle);
|
|
2757
|
-
return Number.isSafeInteger(gen) ? gen : null;
|
|
2758
|
-
}
|
|
2759
|
-
var REGISTRY_SUFFIX = ".reg";
|
|
2760
|
-
function registryDir(root, namespace) {
|
|
2761
|
-
if (namespace !== void 0) validateSegmentRef({ segment: "x", namespace });
|
|
2762
|
-
return join(root, namespacePathPart(namespace), "registry");
|
|
2763
|
-
}
|
|
2764
|
-
function registryRowPath(root, ref) {
|
|
2765
|
-
validateSegmentRef(ref);
|
|
2766
|
-
return join(
|
|
2767
|
-
registryDir(root, ref.namespace),
|
|
2768
|
-
`${encodeNameForPath(ref.segment)}${REGISTRY_SUFFIX}`
|
|
2769
|
-
);
|
|
2588
|
+
function magicMatches(bytes, offset) {
|
|
2589
|
+
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
2770
2590
|
}
|
|
2771
|
-
function
|
|
2772
|
-
if (!
|
|
2773
|
-
|
|
2774
|
-
const segment = decodeNameFromPath(stem);
|
|
2775
|
-
if (encodeNameForPath(segment) !== stem) return null;
|
|
2776
|
-
try {
|
|
2777
|
-
validateSegmentRef({ segment });
|
|
2778
|
-
} catch {
|
|
2779
|
-
return null;
|
|
2591
|
+
function checkSize(tail, size) {
|
|
2592
|
+
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
2593
|
+
throw crbmIntegrity(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
2780
2594
|
}
|
|
2781
|
-
return segment;
|
|
2782
2595
|
}
|
|
2783
|
-
function
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
try {
|
|
2788
|
-
validateSegmentRef({ segment: "x", namespace });
|
|
2789
|
-
} catch {
|
|
2790
|
-
return null;
|
|
2596
|
+
function checkedFooter(tail, size) {
|
|
2597
|
+
checkSize(tail, size);
|
|
2598
|
+
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
2599
|
+
throw crbmIntegrity(`.crbm too small: ${size}B`);
|
|
2791
2600
|
}
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
// src/drivers/localfs/exact-case.ts
|
|
2796
|
-
import { readdir, realpath, stat } from "node:fs/promises";
|
|
2797
|
-
import { dirname as dirname2, join as join3, parse, relative, resolve, sep } from "node:path";
|
|
2798
|
-
|
|
2799
|
-
// src/drivers/localfs/fs-util.ts
|
|
2800
|
-
import { constants as FS } from "node:fs";
|
|
2801
|
-
import { randomUUID } from "node:crypto";
|
|
2802
|
-
import { lstat, mkdir, opendir, open, unlink } from "node:fs/promises";
|
|
2803
|
-
import { dirname, join as join2 } from "node:path";
|
|
2804
|
-
var isCode = (err, code) => err instanceof Error && err.code === code;
|
|
2805
|
-
var isAbsent = (err) => isCode(err, "ENOENT") || isCode(err, "ENOTDIR") || isCode(err, "ENAMETOOLONG");
|
|
2806
|
-
var TRANSIENT_FS_CODES = /* @__PURE__ */ new Set(["EBUSY", "EAGAIN", "EMFILE", "ENFILE", "ETIMEDOUT"]);
|
|
2807
|
-
var isFsTransient = (err) => err instanceof Error && TRANSIENT_FS_CODES.has(err.code ?? "");
|
|
2808
|
-
var mapFsError = (err) => isFsTransient(err) ? new TransientError(`transient filesystem fault: ${err.code}`, { cause: err }) : err;
|
|
2809
|
-
var O_NOFOLLOW = FS.O_NOFOLLOW ?? 0;
|
|
2810
|
-
var FILE_MODE = 384;
|
|
2811
|
-
var DIR_MODE = 448;
|
|
2812
|
-
async function fsyncDir(dir) {
|
|
2813
|
-
try {
|
|
2814
|
-
const handle = await open(dir, "r");
|
|
2815
|
-
try {
|
|
2816
|
-
await handle.sync();
|
|
2817
|
-
} finally {
|
|
2818
|
-
await handle.close();
|
|
2819
|
-
}
|
|
2820
|
-
} catch {
|
|
2601
|
+
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
2602
|
+
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
2603
|
+
throw crbmIntegrity(".crbm end magic mismatch");
|
|
2821
2604
|
}
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset);
|
|
2827
|
-
if (bytesWritten <= 0) {
|
|
2828
|
-
throw Object.assign(new Error("a write made no progress: the device is full"), {
|
|
2829
|
-
code: "ENOSPC"
|
|
2830
|
-
});
|
|
2831
|
-
}
|
|
2832
|
-
offset += bytesWritten;
|
|
2605
|
+
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
2606
|
+
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
2607
|
+
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
2608
|
+
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
2833
2609
|
}
|
|
2610
|
+
return { footer, fview, storedFooterCrc };
|
|
2834
2611
|
}
|
|
2835
|
-
async function
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2612
|
+
async function readFooter(blob) {
|
|
2613
|
+
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2614
|
+
const { fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
2615
|
+
return {
|
|
2616
|
+
encrypted: (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0,
|
|
2617
|
+
footerCrc: storedFooterCrc
|
|
2618
|
+
};
|
|
2841
2619
|
}
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
const last = lastSwept.get(dir);
|
|
2850
|
-
if (last !== void 0 && now - last < SWEEP_INTERVAL_MS) return;
|
|
2851
|
-
lastSwept.set(dir, now);
|
|
2852
|
-
try {
|
|
2853
|
-
const entries = await opendir(dir);
|
|
2854
|
-
for await (const entry of entries) {
|
|
2855
|
-
if (!TEMP_NAME.test(entry.name)) continue;
|
|
2856
|
-
const path = join2(dir, entry.name);
|
|
2857
|
-
const info = await lstat(path).catch(() => void 0);
|
|
2858
|
-
if (info?.isFile() === true && info.mtimeMs < now - ORPHAN_TEMP_AGE_MS) {
|
|
2859
|
-
await unlink(path).catch(() => {
|
|
2860
|
-
});
|
|
2861
|
-
}
|
|
2862
|
-
}
|
|
2863
|
-
} catch {
|
|
2620
|
+
async function footerSaysEncrypted(blob) {
|
|
2621
|
+
return (await readFooter(blob)).encrypted;
|
|
2622
|
+
}
|
|
2623
|
+
function readU64(view, offset, field) {
|
|
2624
|
+
const big = view.getBigUint64(offset, true);
|
|
2625
|
+
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
2626
|
+
throw crbmIntegrity(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
2864
2627
|
}
|
|
2628
|
+
return Number(big);
|
|
2865
2629
|
}
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
}
|
|
2875
|
-
return -1;
|
|
2876
|
-
}
|
|
2877
|
-
var swapCase = (s) => [...s].map((c) => c === c.toLowerCase() ? c.toUpperCase() : c.toLowerCase()).join("");
|
|
2878
|
-
var components = (path) => path.split(sep).filter((c) => c !== "");
|
|
2879
|
-
function caseVariantError() {
|
|
2880
|
-
return new ValidationError(
|
|
2881
|
-
"names differ only by case on a case-insensitive filesystem: a name that differs only by case from one already stored is the same file here, so it is refused rather than written over"
|
|
2882
|
-
);
|
|
2630
|
+
var keepLimits = /* @__PURE__ */ new WeakMap();
|
|
2631
|
+
function openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo) {
|
|
2632
|
+
const asked = { ...options };
|
|
2633
|
+
if (keepChunkBytesUpTo !== void 0) keepLimits.set(asked, keepChunkBytesUpTo);
|
|
2634
|
+
return CrbmReader.open(blob, asked);
|
|
2635
|
+
}
|
|
2636
|
+
function openTailBytes(options) {
|
|
2637
|
+
return Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
2883
2638
|
}
|
|
2884
|
-
var
|
|
2885
|
-
constructor(
|
|
2886
|
-
this.
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
try {
|
|
2900
|
-
names = await readdir(this.root);
|
|
2901
|
-
} catch (err) {
|
|
2902
|
-
if (isAbsent(err)) return this.foldsAtAncestor();
|
|
2903
|
-
throw err;
|
|
2904
|
-
}
|
|
2905
|
-
for (const name of names) {
|
|
2906
|
-
const swapped = swapCase(name);
|
|
2907
|
-
if (swapped === name) continue;
|
|
2908
|
-
const own = await stat(join3(this.root, name), { bigint: true }).catch(() => void 0);
|
|
2909
|
-
if (own === void 0) continue;
|
|
2910
|
-
const other = await stat(join3(this.root, swapped), { bigint: true }).catch(() => void 0);
|
|
2911
|
-
this.insensitive = other !== void 0 && other.ino === own.ino && other.dev === own.dev;
|
|
2912
|
-
return this.insensitive;
|
|
2913
|
-
}
|
|
2914
|
-
return this.foldsAtAncestor();
|
|
2915
|
-
}
|
|
2916
|
-
/**
|
|
2917
|
-
* The same question asked of the root's own name, or the nearest existing directory above it, when the root holds
|
|
2918
|
-
* nothing to try: a root with no entries has none to alias yet, but two names created at once still would. Not
|
|
2919
|
-
* remembered, since it is the volume above the root that answers.
|
|
2920
|
-
*/
|
|
2921
|
-
async foldsAtAncestor() {
|
|
2922
|
-
let dir = resolve(this.root);
|
|
2923
|
-
for (; ; ) {
|
|
2924
|
-
const top = parse(dir).root;
|
|
2925
|
-
const parts = components(relative(top, dir));
|
|
2926
|
-
let at = parts.length - 1;
|
|
2927
|
-
while (at >= 0 && swapCase(parts[at]) === parts[at]) at--;
|
|
2928
|
-
if (at === -1) return false;
|
|
2929
|
-
const swapped = [...parts];
|
|
2930
|
-
swapped[at] = swapCase(parts[at]);
|
|
2931
|
-
let own;
|
|
2932
|
-
try {
|
|
2933
|
-
own = await stat(dir, { bigint: true });
|
|
2934
|
-
} catch (err) {
|
|
2935
|
-
if (!isAbsent(err)) return false;
|
|
2936
|
-
const parent = dirname2(dir);
|
|
2937
|
-
if (parent === dir) return false;
|
|
2938
|
-
dir = parent;
|
|
2939
|
-
continue;
|
|
2940
|
-
}
|
|
2941
|
-
const other = await stat(join3(top, ...swapped), { bigint: true }).catch(() => void 0);
|
|
2942
|
-
return other !== void 0 && other.ino === own.ino && other.dev === own.dev;
|
|
2943
|
-
}
|
|
2944
|
-
}
|
|
2945
|
-
/** Whether the root's filesystem folds case: names that differ only by case are one file there. */
|
|
2946
|
-
folds() {
|
|
2947
|
-
return this.foldsCase();
|
|
2948
|
-
}
|
|
2949
|
-
/**
|
|
2950
|
-
* `true` when `path` (inside the root) is, on disk, a path whose name differs only by case from the one given:
|
|
2951
|
-
* the deepest part of it that exists is resolved to its real name and compared exactly. A path that does not exist
|
|
2952
|
-
* yet, or that nothing differs from, is `false`.
|
|
2953
|
-
*/
|
|
2954
|
-
async differs(path) {
|
|
2955
|
-
if (!await this.foldsCase()) return false;
|
|
2956
|
-
const rootAbs = resolve(this.root);
|
|
2957
|
-
let probe = resolve(path);
|
|
2958
|
-
let real;
|
|
2959
|
-
for (; ; ) {
|
|
2960
|
-
if (probe === rootAbs || !relative(rootAbs, probe) || relative(rootAbs, probe).startsWith("..")) {
|
|
2961
|
-
return false;
|
|
2962
|
-
}
|
|
2963
|
-
try {
|
|
2964
|
-
real = await realpath(probe);
|
|
2965
|
-
break;
|
|
2966
|
-
} catch (err) {
|
|
2967
|
-
if (!isAbsent(err)) throw err;
|
|
2968
|
-
probe = dirname2(probe);
|
|
2969
|
-
}
|
|
2970
|
-
}
|
|
2971
|
-
const realRoot = await realpath(rootAbs).catch(() => void 0);
|
|
2972
|
-
if (realRoot === void 0) return false;
|
|
2973
|
-
const actual = relative(realRoot, real);
|
|
2974
|
-
if (actual.startsWith("..")) return false;
|
|
2975
|
-
return firstCaseDifference(components(relative(rootAbs, probe)), components(actual)) !== -1;
|
|
2976
|
-
}
|
|
2977
|
-
/** Refuse a write to `path` when its real name differs from it only by case. */
|
|
2978
|
-
async refuseVariant(path) {
|
|
2979
|
-
if (await this.differs(path)) throw caseVariantError();
|
|
2980
|
-
}
|
|
2981
|
-
};
|
|
2982
|
-
|
|
2983
|
-
// src/drivers/localfs/storage.ts
|
|
2984
|
-
async function readWhole(handle, key, length, offset) {
|
|
2985
|
-
const buf = new Uint8Array(length);
|
|
2986
|
-
let filled = 0;
|
|
2987
|
-
while (filled < length) {
|
|
2988
|
-
const { bytesRead } = await handle.read(buf, filled, length - filled, offset + filled);
|
|
2989
|
-
if (bytesRead === 0) {
|
|
2990
|
-
throw new IntegrityError(
|
|
2991
|
-
`short read of ${key.segment}.${key.generation}: wanted ${length}B at ${offset}, got ${filled}B`
|
|
2992
|
-
);
|
|
2993
|
-
}
|
|
2994
|
-
filled += bytesRead;
|
|
2995
|
-
}
|
|
2996
|
-
return buf;
|
|
2997
|
-
}
|
|
2998
|
-
var LocalFsStorageDriver = class {
|
|
2999
|
-
/**
|
|
3000
|
-
* `options.privateRoot` is the directory this one lives under, from which down directories are created private; it
|
|
3001
|
-
* defaults to `root`.
|
|
3002
|
-
*/
|
|
3003
|
-
constructor(root, options = {}) {
|
|
3004
|
-
this.root = root;
|
|
3005
|
-
this.options = options;
|
|
3006
|
-
this.exactCase = new ExactCase(root);
|
|
3007
|
-
}
|
|
3008
|
-
root;
|
|
3009
|
-
options;
|
|
3010
|
-
exactCase;
|
|
3011
|
-
capabilities() {
|
|
3012
|
-
return {
|
|
3013
|
-
rangeRead: true,
|
|
3014
|
-
maxObjectBytes: Number.MAX_SAFE_INTEGER,
|
|
3015
|
-
conditionalPut: false,
|
|
3016
|
-
conditionalDelete: false
|
|
3017
|
-
};
|
|
3018
|
-
}
|
|
3019
|
-
async putImmutable(key, write) {
|
|
3020
|
-
const finalPath = storageObjectPath(this.root, key);
|
|
3021
|
-
assertStorageNamesFit(key);
|
|
3022
|
-
await this.exactCase.refuseVariant(finalPath);
|
|
3023
|
-
await makeDirs(dirname3(finalPath), this.options.privateRoot ?? this.root);
|
|
3024
|
-
void sweepOrphanTemps(dirname3(finalPath));
|
|
3025
|
-
const tmpPath = tempPathFor(finalPath);
|
|
3026
|
-
const handle = await open2(
|
|
3027
|
-
tmpPath,
|
|
3028
|
-
FS2.O_CREAT | FS2.O_EXCL | FS2.O_WRONLY | O_NOFOLLOW,
|
|
3029
|
-
FILE_MODE
|
|
3030
|
-
);
|
|
3031
|
-
const hash = createHash2("sha256");
|
|
3032
|
-
let size = 0;
|
|
3033
|
-
const sink = {
|
|
3034
|
-
async write(bytes) {
|
|
3035
|
-
await writeAll(handle, bytes);
|
|
3036
|
-
hash.update(bytes);
|
|
3037
|
-
size += bytes.length;
|
|
3038
|
-
}
|
|
3039
|
-
};
|
|
3040
|
-
try {
|
|
3041
|
-
await write(sink);
|
|
3042
|
-
await handle.sync();
|
|
3043
|
-
} catch (err) {
|
|
3044
|
-
await handle.close().catch(() => {
|
|
3045
|
-
});
|
|
3046
|
-
await unlink2(tmpPath).catch(() => {
|
|
3047
|
-
});
|
|
3048
|
-
throw mapFsError(err);
|
|
3049
|
-
}
|
|
3050
|
-
await handle.close();
|
|
3051
|
-
try {
|
|
3052
|
-
await link(tmpPath, finalPath);
|
|
3053
|
-
} catch (err) {
|
|
3054
|
-
await unlink2(tmpPath).catch(() => {
|
|
3055
|
-
});
|
|
3056
|
-
if (isCode(err, "ENOENT")) {
|
|
3057
|
-
throw new TransientError(
|
|
3058
|
-
`the temporary file of ${key.segment}.${key.generation} was removed mid-write`,
|
|
3059
|
-
{
|
|
3060
|
-
cause: err
|
|
3061
|
-
}
|
|
3062
|
-
);
|
|
3063
|
-
}
|
|
3064
|
-
if (isCode(err, "EEXIST")) {
|
|
3065
|
-
throw new WriteConflictError(
|
|
3066
|
-
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
3067
|
-
);
|
|
3068
|
-
}
|
|
3069
|
-
throw mapFsError(err);
|
|
3070
|
-
}
|
|
3071
|
-
await unlink2(tmpPath).catch(() => {
|
|
3072
|
-
});
|
|
3073
|
-
await fsyncDir(dirname3(finalPath));
|
|
3074
|
-
return { size, sha256: hash.digest("hex") };
|
|
3075
|
-
}
|
|
3076
|
-
async getRange(key, offset, length) {
|
|
3077
|
-
requireInteger(offset, { min: 0, what: "range offset" });
|
|
3078
|
-
requireInteger(length, { min: 0, what: "range length" });
|
|
3079
|
-
const handle = await this.openRead(key);
|
|
3080
|
-
try {
|
|
3081
|
-
const { size } = await handle.stat();
|
|
3082
|
-
if (offset + length > size) {
|
|
3083
|
-
throw new ValidationError(
|
|
3084
|
-
`range [${offset}, ${offset + length}) out of bounds for ${size}B`
|
|
3085
|
-
);
|
|
3086
|
-
}
|
|
3087
|
-
return await readWhole(handle, key, length, offset);
|
|
3088
|
-
} finally {
|
|
3089
|
-
await handle.close();
|
|
3090
|
-
}
|
|
3091
|
-
}
|
|
3092
|
-
async getTail(key, maxBytes) {
|
|
3093
|
-
checkTailLength(maxBytes);
|
|
3094
|
-
const handle = await this.openRead(key);
|
|
3095
|
-
try {
|
|
3096
|
-
const { size } = await handle.stat();
|
|
3097
|
-
const take = Math.min(Math.max(maxBytes, 0), size);
|
|
3098
|
-
return { bytes: await readWhole(handle, key, take, size - take), size };
|
|
3099
|
-
} finally {
|
|
3100
|
-
await handle.close();
|
|
3101
|
-
}
|
|
3102
|
-
}
|
|
3103
|
-
/**
|
|
3104
|
-
* Idempotent. Ignores `options.ifVersion`, as `conditionalDelete: false` says: see the note on the class for why it
|
|
3105
|
-
* cannot apply one. The parameter is declared so that a wrapper of this driver can hand the port's options on.
|
|
3106
|
-
*/
|
|
3107
|
-
async delete(key, options) {
|
|
3108
|
-
void options;
|
|
3109
|
-
const path = storageObjectPath(this.root, key);
|
|
3110
|
-
if (await this.exactCase.differs(path)) return;
|
|
3111
|
-
await unlink2(path).catch((err) => {
|
|
3112
|
-
if (!isAbsent(err)) throw mapFsError(err);
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
async *list(ref) {
|
|
3116
|
-
const dir = segmentsDir(this.root, ref);
|
|
3117
|
-
if (await this.exactCase.differs(dir)) return;
|
|
3118
|
-
let names;
|
|
3119
|
-
try {
|
|
3120
|
-
names = await readdir2(dir);
|
|
3121
|
-
} catch (err) {
|
|
3122
|
-
if (isAbsent(err)) return;
|
|
3123
|
-
throw mapFsError(err);
|
|
3124
|
-
}
|
|
3125
|
-
for (const name of names) {
|
|
3126
|
-
const generation = parseGeneration(ref.segment, name);
|
|
3127
|
-
if (generation !== null) {
|
|
3128
|
-
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3129
|
-
}
|
|
3130
|
-
}
|
|
3131
|
-
}
|
|
3132
|
-
async openRead(key) {
|
|
3133
|
-
const path = storageObjectPath(this.root, key);
|
|
3134
|
-
if (await this.exactCase.differs(path)) {
|
|
3135
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3136
|
-
}
|
|
3137
|
-
try {
|
|
3138
|
-
return await open2(path, FS2.O_RDONLY | O_NOFOLLOW);
|
|
3139
|
-
} catch (err) {
|
|
3140
|
-
if (isAbsent(err)) {
|
|
3141
|
-
throw new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
3142
|
-
}
|
|
3143
|
-
if (isCode(err, "ELOOP")) {
|
|
3144
|
-
throw new NotFoundError(
|
|
3145
|
-
`generation path is a symlink, refusing: ${key.segment}.${key.generation}`
|
|
3146
|
-
);
|
|
3147
|
-
}
|
|
3148
|
-
throw mapFsError(err);
|
|
3149
|
-
}
|
|
3150
|
-
}
|
|
3151
|
-
};
|
|
3152
|
-
|
|
3153
|
-
// src/drivers/localfs/registry.ts
|
|
3154
|
-
import { constants as FS3 } from "node:fs";
|
|
3155
|
-
import { open as open3, opendir as opendir2, readdir as readdir3, realpath as realpath2, rename, unlink as unlink3 } from "node:fs/promises";
|
|
3156
|
-
import { basename, dirname as dirname4, join as join4, relative as relative2, resolve as resolve2 } from "node:path";
|
|
3157
|
-
var DEFAULT_MAX_ROW_BYTES = 1 * 1024 * 1024;
|
|
3158
|
-
var rowChains = /* @__PURE__ */ new Map();
|
|
3159
|
-
async function rowLockKey(path) {
|
|
3160
|
-
const abs = resolve2(path);
|
|
3161
|
-
const tail = [basename(abs)];
|
|
3162
|
-
let dir = dirname4(abs);
|
|
3163
|
-
for (; ; ) {
|
|
3164
|
-
try {
|
|
3165
|
-
return join4(await realpath2(dir), ...tail);
|
|
3166
|
-
} catch (err) {
|
|
3167
|
-
const parent = dirname4(dir);
|
|
3168
|
-
if (!isAbsent(err) || parent === dir) throw mapFsError(err);
|
|
3169
|
-
tail.unshift(basename(dir));
|
|
3170
|
-
dir = parent;
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
}
|
|
3174
|
-
var LocalFsRegistryDriver = class {
|
|
3175
|
-
constructor(root, options = {}) {
|
|
3176
|
-
this.root = root;
|
|
3177
|
-
this.options = options;
|
|
3178
|
-
this.exactCase = new ExactCase(root);
|
|
3179
|
-
this.now = options.now ?? (() => Date.now());
|
|
3180
|
-
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
3181
|
-
}
|
|
3182
|
-
root;
|
|
3183
|
-
options;
|
|
3184
|
-
now;
|
|
3185
|
-
entropy;
|
|
3186
|
-
exactCase;
|
|
3187
|
-
/**
|
|
3188
|
-
* `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
|
|
3189
|
-
* the root in this process takes, so no write can land between the check and the unlink.
|
|
3190
|
-
*/
|
|
3191
|
-
capabilities() {
|
|
3192
|
-
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
3193
|
-
}
|
|
3194
|
-
/** Every write reports its token to `onToken` before the row's file is written. */
|
|
3195
|
-
get reportsWriteTokens() {
|
|
3196
|
-
return true;
|
|
3197
|
-
}
|
|
3198
|
-
async get(ref) {
|
|
3199
|
-
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
3200
|
-
return env && !env.deleted ? env.record : null;
|
|
3201
|
-
}
|
|
3202
|
-
async create(ref, record, options) {
|
|
3203
|
-
const checked = validateNewRegistryRecord(record);
|
|
3204
|
-
const path = registryRowPath(this.root, ref);
|
|
3205
|
-
assertRegistryNamesFit(ref);
|
|
3206
|
-
return this.withRowLock(path, async () => {
|
|
3207
|
-
const current = await this.readRow(path, ref);
|
|
3208
|
-
if (current !== null && !current.deleted) {
|
|
3209
|
-
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
3210
|
-
}
|
|
3211
|
-
const token = newIncarnationToken(this.entropy, current?.record);
|
|
3212
|
-
options?.onToken?.(token);
|
|
3213
|
-
await this.writeRow(path, recordFromNew(ref, checked, this.now(), token));
|
|
3214
|
-
return { token };
|
|
3215
|
-
});
|
|
3216
|
-
}
|
|
3217
|
-
/**
|
|
3218
|
-
* A caller's `held` row (see `RegistryWriteOptions`) has no read to spare here: the row's file is read under its lock and its token compared
|
|
3219
|
-
* there, a local read and no request, so a stale one fails exactly as a lost race does.
|
|
3220
|
-
*/
|
|
3221
|
-
async compareAndSwap(ref, expected, patch, options) {
|
|
3222
|
-
const checked = validateRegistryPatch(patch);
|
|
3223
|
-
const path = registryRowPath(this.root, ref);
|
|
3224
|
-
assertRegistryNamesFit(ref);
|
|
3225
|
-
return this.withRowLock(path, async () => {
|
|
3226
|
-
const current = await this.readRow(path, ref);
|
|
3227
|
-
if (current === null || current.deleted || current.record.token !== expected) {
|
|
3228
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3229
|
-
}
|
|
3230
|
-
const token = nextRegistryToken(current.record, this.entropy);
|
|
3231
|
-
options?.onToken?.(token);
|
|
3232
|
-
await this.writeRow(path, applyRegistryPatch(current.record, checked, this.now(), token));
|
|
3233
|
-
return { token };
|
|
3234
|
-
});
|
|
3235
|
-
}
|
|
3236
|
-
async *list(namespace) {
|
|
3237
|
-
for (const ns of await this.namespaceDirs(namespace)) {
|
|
3238
|
-
const dir = registryDir(this.root, ns);
|
|
3239
|
-
if (await this.exactCase.differs(dir)) continue;
|
|
3240
|
-
let entries;
|
|
3241
|
-
try {
|
|
3242
|
-
entries = await opendir2(dir);
|
|
3243
|
-
} catch (err) {
|
|
3244
|
-
if (isAbsent(err)) continue;
|
|
3245
|
-
throw mapFsError(err);
|
|
3246
|
-
}
|
|
3247
|
-
for await (const entry of entries) {
|
|
3248
|
-
const segment = parseRegistryRow(entry.name);
|
|
3249
|
-
if (segment === null) continue;
|
|
3250
|
-
const ref = { namespace: ns, segment };
|
|
3251
|
-
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
3252
|
-
if (env && !env.deleted) yield env.record;
|
|
3253
|
-
}
|
|
3254
|
-
}
|
|
3255
|
-
}
|
|
3256
|
-
async delete(ref, expected) {
|
|
3257
|
-
const path = registryRowPath(this.root, ref);
|
|
3258
|
-
return this.withRowLock(path, async () => {
|
|
3259
|
-
const current = await this.readRow(path, ref);
|
|
3260
|
-
if (expected !== void 0) {
|
|
3261
|
-
if (current === null || current.deleted || current.record.token !== expected) {
|
|
3262
|
-
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
3263
|
-
}
|
|
3264
|
-
} else if (current === null || current.deleted) {
|
|
3265
|
-
return;
|
|
3266
|
-
}
|
|
3267
|
-
await this.unlinkRow(path);
|
|
3268
|
-
});
|
|
3269
|
-
}
|
|
3270
|
-
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
3271
|
-
async unlinkRow(path) {
|
|
3272
|
-
try {
|
|
3273
|
-
await unlink3(path);
|
|
3274
|
-
} catch (err) {
|
|
3275
|
-
if (!isCode(err, "ENOENT")) throw mapFsError(err);
|
|
3276
|
-
}
|
|
3277
|
-
await fsyncDir(dirname4(path));
|
|
3278
|
-
}
|
|
3279
|
-
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
3280
|
-
async namespaceDirs(namespace) {
|
|
3281
|
-
if (namespace !== void 0) return [namespace];
|
|
3282
|
-
let entries;
|
|
3283
|
-
try {
|
|
3284
|
-
entries = await readdir3(this.root);
|
|
3285
|
-
} catch (err) {
|
|
3286
|
-
if (isCode(err, "ENOENT")) return [];
|
|
3287
|
-
throw mapFsError(err);
|
|
3288
|
-
}
|
|
3289
|
-
const namespaces = [];
|
|
3290
|
-
for (const entry of entries) {
|
|
3291
|
-
const parsed = parseNamespaceDir(entry);
|
|
3292
|
-
if (parsed !== null) namespaces.push(parsed.namespace);
|
|
3293
|
-
}
|
|
3294
|
-
return namespaces;
|
|
3295
|
-
}
|
|
3296
|
-
/** Read the row at `path`, which must be the row of `ref`: one that names another segment is refused. */
|
|
3297
|
-
async readRow(path, ref) {
|
|
3298
|
-
let handle;
|
|
3299
|
-
try {
|
|
3300
|
-
if (await this.exactCase.differs(path)) return null;
|
|
3301
|
-
handle = await open3(path, FS3.O_RDONLY | O_NOFOLLOW);
|
|
3302
|
-
} catch (err) {
|
|
3303
|
-
if (isAbsent(err) || isCode(err, "ELOOP")) return null;
|
|
3304
|
-
throw mapFsError(err);
|
|
3305
|
-
}
|
|
3306
|
-
try {
|
|
3307
|
-
const { size } = await handle.stat();
|
|
3308
|
-
if (size > DEFAULT_MAX_ROW_BYTES) {
|
|
3309
|
-
throw new IntegrityError(`registry row ${size}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`);
|
|
3310
|
-
}
|
|
3311
|
-
const text = (await handle.readFile()).toString("utf8");
|
|
3312
|
-
return parseRegistryEnvelope(text, relative2(this.root, path), ref);
|
|
3313
|
-
} finally {
|
|
3314
|
-
await handle.close();
|
|
3315
|
-
}
|
|
3316
|
-
}
|
|
3317
|
-
async writeRow(path, record) {
|
|
3318
|
-
await this.exactCase.refuseVariant(path);
|
|
3319
|
-
await makeDirs(dirname4(path), this.options.privateRoot ?? this.root);
|
|
3320
|
-
void sweepOrphanTemps(dirname4(path));
|
|
3321
|
-
const out = Buffer.from(serializeRegistryEnvelope({ deleted: false, record }), "utf8");
|
|
3322
|
-
if (out.length > DEFAULT_MAX_ROW_BYTES) {
|
|
3323
|
-
throw new ValidationError(
|
|
3324
|
-
`registry row ${out.length}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`
|
|
3325
|
-
);
|
|
3326
|
-
}
|
|
3327
|
-
const tmp = tempPathFor(path);
|
|
3328
|
-
let handle;
|
|
3329
|
-
try {
|
|
3330
|
-
handle = await open3(tmp, FS3.O_CREAT | FS3.O_EXCL | FS3.O_WRONLY | O_NOFOLLOW, FILE_MODE);
|
|
3331
|
-
} catch (err) {
|
|
3332
|
-
throw mapFsError(err);
|
|
3333
|
-
}
|
|
3334
|
-
try {
|
|
3335
|
-
await writeAll(handle, out);
|
|
3336
|
-
await handle.sync();
|
|
3337
|
-
} catch (err) {
|
|
3338
|
-
await handle.close().catch(() => {
|
|
3339
|
-
});
|
|
3340
|
-
await unlink3(tmp).catch(() => {
|
|
3341
|
-
});
|
|
3342
|
-
throw mapFsError(err);
|
|
3343
|
-
}
|
|
3344
|
-
await handle.close();
|
|
3345
|
-
await rename(tmp, path).catch(async (err) => {
|
|
3346
|
-
await unlink3(tmp).catch(() => {
|
|
3347
|
-
});
|
|
3348
|
-
if (isCode(err, "ENOENT")) {
|
|
3349
|
-
throw new TransientError("the temporary file of a registry row was removed mid-write", {
|
|
3350
|
-
cause: err
|
|
3351
|
-
});
|
|
3352
|
-
}
|
|
3353
|
-
throw mapFsError(err);
|
|
3354
|
-
});
|
|
3355
|
-
await fsyncDir(dirname4(path));
|
|
3356
|
-
}
|
|
3357
|
-
/** Serialize callbacks for a row so read-modify-write is atomic across every instance in the process. */
|
|
3358
|
-
async withRowLock(path, fn) {
|
|
3359
|
-
const resolved = await rowLockKey(path);
|
|
3360
|
-
const key = await this.exactCase.folds() ? resolved.toLowerCase() : resolved;
|
|
3361
|
-
const prev = rowChains.get(key) ?? Promise.resolve();
|
|
3362
|
-
const result = prev.then(fn, fn);
|
|
3363
|
-
const tail = result.then(
|
|
3364
|
-
() => void 0,
|
|
3365
|
-
() => void 0
|
|
3366
|
-
);
|
|
3367
|
-
rowChains.set(key, tail);
|
|
3368
|
-
void tail.then(() => {
|
|
3369
|
-
if (rowChains.get(key) === tail) rowChains.delete(key);
|
|
3370
|
-
});
|
|
3371
|
-
return result;
|
|
3372
|
-
}
|
|
3373
|
-
};
|
|
3374
|
-
|
|
3375
|
-
// src/drivers/backends.ts
|
|
3376
|
-
var CLOCK_OPTION_KEYS = ["now"];
|
|
3377
|
-
var MemoryStorage = class {
|
|
3378
|
-
storage;
|
|
3379
|
-
registry;
|
|
3380
|
-
constructor(options = {}) {
|
|
3381
|
-
refuseUnknownOptions("MemoryStorage", options, CLOCK_OPTION_KEYS);
|
|
3382
|
-
checkNow("MemoryStorage", options.now);
|
|
3383
|
-
this.storage = new MemoryStorageDriver();
|
|
3384
|
-
this.registry = new MemoryRegistryDriver(options.now === void 0 ? {} : { now: options.now });
|
|
3385
|
-
brandAsBackend(this);
|
|
3386
|
-
}
|
|
3387
|
-
};
|
|
3388
|
-
var LocalFsStorage = class {
|
|
3389
|
-
constructor(root, options = {}) {
|
|
3390
|
-
this.root = root;
|
|
3391
|
-
if (typeof root !== "string" || root.length === 0) {
|
|
3392
|
-
throw new ValidationError(
|
|
3393
|
-
`LocalFsStorage needs a root directory: a non-empty string, got ${root === "" ? "an empty string" : typeof root}`
|
|
3394
|
-
);
|
|
3395
|
-
}
|
|
3396
|
-
refuseUnknownOptions("LocalFsStorage", options, CLOCK_OPTION_KEYS);
|
|
3397
|
-
checkNow("LocalFsStorage", options.now);
|
|
3398
|
-
this.storage = new LocalFsStorageDriver(join5(root, "storage"), { privateRoot: root });
|
|
3399
|
-
this.registry = new LocalFsRegistryDriver(
|
|
3400
|
-
join5(root, "registry"),
|
|
3401
|
-
options.now === void 0 ? { privateRoot: root } : { now: options.now, privateRoot: root }
|
|
3402
|
-
);
|
|
3403
|
-
brandAsBackend(this);
|
|
3404
|
-
}
|
|
3405
|
-
root;
|
|
3406
|
-
storage;
|
|
3407
|
-
registry;
|
|
3408
|
-
};
|
|
3409
|
-
|
|
3410
|
-
// src/core/reader-defaults.ts
|
|
3411
|
-
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
3412
|
-
var REFRESH_RETRY_MS = 500;
|
|
3413
|
-
var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
|
|
3414
|
-
var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
3415
|
-
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
3416
|
-
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
3417
|
-
|
|
3418
|
-
// src/core/crbm/integrity.ts
|
|
3419
|
-
var crbmIntegrity = (message) => new IntegrityError(`${message}; re-load from source or roll back`);
|
|
3420
|
-
|
|
3421
|
-
// src/core/crbm/crc32c.ts
|
|
3422
|
-
var TABLES = (() => {
|
|
3423
|
-
const POLY = 2197175160;
|
|
3424
|
-
const t0 = new Uint32Array(256);
|
|
3425
|
-
for (let n = 0; n < 256; n++) {
|
|
3426
|
-
let c = n;
|
|
3427
|
-
for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
|
|
3428
|
-
t0[n] = c >>> 0;
|
|
3429
|
-
}
|
|
3430
|
-
const tables = [t0];
|
|
3431
|
-
for (let k = 1; k < 8; k++) {
|
|
3432
|
-
const prev = tables[k - 1];
|
|
3433
|
-
const next = new Uint32Array(256);
|
|
3434
|
-
for (let n = 0; n < 256; n++) {
|
|
3435
|
-
const v = prev[n];
|
|
3436
|
-
next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
|
|
3437
|
-
}
|
|
3438
|
-
tables.push(next);
|
|
3439
|
-
}
|
|
3440
|
-
return tables;
|
|
3441
|
-
})();
|
|
3442
|
-
var T0 = TABLES[0];
|
|
3443
|
-
var T1 = TABLES[1];
|
|
3444
|
-
var T2 = TABLES[2];
|
|
3445
|
-
var T3 = TABLES[3];
|
|
3446
|
-
var T4 = TABLES[4];
|
|
3447
|
-
var T5 = TABLES[5];
|
|
3448
|
-
var T6 = TABLES[6];
|
|
3449
|
-
var T7 = TABLES[7];
|
|
3450
|
-
function crc32c(bytes, seed = 0) {
|
|
3451
|
-
let crc = (seed ^ 4294967295) >>> 0;
|
|
3452
|
-
let i = 0;
|
|
3453
|
-
const n8 = bytes.length - bytes.length % 8;
|
|
3454
|
-
for (; i < n8; i += 8) {
|
|
3455
|
-
crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
|
|
3456
|
-
crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
|
|
3457
|
-
}
|
|
3458
|
-
for (; i < bytes.length; i++) {
|
|
3459
|
-
crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
|
|
3460
|
-
}
|
|
3461
|
-
return (crc ^ 4294967295) >>> 0;
|
|
3462
|
-
}
|
|
3463
|
-
|
|
3464
|
-
// src/core/crbm/plan-reads.ts
|
|
3465
|
-
var MAX_COALESCE_GAP_BYTES = 256 * 1024;
|
|
3466
|
-
var MAX_COALESCED_READ_BYTES = DEFAULT_MAX_BITMAP_BYTES;
|
|
3467
|
-
var MAX_RANGES_IN_FLIGHT = 32;
|
|
3468
|
-
function* planChunkReads(extents, region) {
|
|
3469
|
-
let chunks = [];
|
|
3470
|
-
let start = 0;
|
|
3471
|
-
let end = 0;
|
|
3472
|
-
let previousEnd = region.start;
|
|
3473
|
-
for (const extent of extents) {
|
|
3474
|
-
const extentEnd = extent.offset + extent.length;
|
|
3475
|
-
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
3476
|
-
throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
|
|
3477
|
-
}
|
|
3478
|
-
previousEnd = extentEnd;
|
|
3479
|
-
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
3480
|
-
chunks.push(extent);
|
|
3481
|
-
end = extentEnd;
|
|
3482
|
-
continue;
|
|
3483
|
-
}
|
|
3484
|
-
if (chunks.length > 0) {
|
|
3485
|
-
const done = chunks;
|
|
3486
|
-
chunks = [];
|
|
3487
|
-
yield { offset: start, length: end - start, chunks: done };
|
|
3488
|
-
}
|
|
3489
|
-
chunks.push(extent);
|
|
3490
|
-
start = extent.offset;
|
|
3491
|
-
end = extentEnd;
|
|
3492
|
-
}
|
|
3493
|
-
if (chunks.length > 0) yield { offset: start, length: end - start, chunks };
|
|
3494
|
-
}
|
|
3495
|
-
|
|
3496
|
-
// src/core/crbm/varint.ts
|
|
3497
|
-
var U32_CEIL = U32_MAX + 1;
|
|
3498
|
-
function writeVarint(out, value) {
|
|
3499
|
-
if (!Number.isInteger(value) || value < 0 || value >= U32_CEIL) {
|
|
3500
|
-
throw new RangeError(`varint value must be a u32; got ${value}`);
|
|
3501
|
-
}
|
|
3502
|
-
let v = value;
|
|
3503
|
-
while (v > 127) {
|
|
3504
|
-
out.push(v & 127 | 128);
|
|
3505
|
-
v >>>= 7;
|
|
3506
|
-
}
|
|
3507
|
-
out.push(v);
|
|
3508
|
-
}
|
|
3509
|
-
function readVarint(bytes, offset) {
|
|
3510
|
-
let result = 0;
|
|
3511
|
-
let shift = 0;
|
|
3512
|
-
let pos = offset;
|
|
3513
|
-
for (let i = 0; i < 5; i++) {
|
|
3514
|
-
if (pos >= bytes.length) {
|
|
3515
|
-
throw crbmIntegrity("varint truncated past end of buffer");
|
|
3516
|
-
}
|
|
3517
|
-
const byte = bytes[pos];
|
|
3518
|
-
pos += 1;
|
|
3519
|
-
if (i === 4 && byte > 15) {
|
|
3520
|
-
throw crbmIntegrity("varint exceeds u32 range");
|
|
3521
|
-
}
|
|
3522
|
-
result = (result | (byte & 127) << shift) >>> 0;
|
|
3523
|
-
if ((byte & 128) === 0) {
|
|
3524
|
-
return { value: result, next: pos };
|
|
3525
|
-
}
|
|
3526
|
-
shift += 7;
|
|
3527
|
-
}
|
|
3528
|
-
throw crbmIntegrity("varint exceeds u32 range");
|
|
3529
|
-
}
|
|
3530
|
-
|
|
3531
|
-
// src/core/crbm/reader.ts
|
|
3532
|
-
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
3533
|
-
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
3534
|
-
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
3535
|
-
function indexCapacity(indexLength) {
|
|
3536
|
-
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
3537
|
-
}
|
|
3538
|
-
function magicMatches(bytes, offset) {
|
|
3539
|
-
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
3540
|
-
}
|
|
3541
|
-
function checkSize(tail, size) {
|
|
3542
|
-
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3543
|
-
throw crbmIntegrity(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
3544
|
-
}
|
|
3545
|
-
}
|
|
3546
|
-
function checkedFooter(tail, size) {
|
|
3547
|
-
checkSize(tail, size);
|
|
3548
|
-
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
3549
|
-
throw crbmIntegrity(`.crbm too small: ${size}B`);
|
|
3550
|
-
}
|
|
3551
|
-
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
3552
|
-
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
3553
|
-
throw crbmIntegrity(".crbm end magic mismatch");
|
|
3554
|
-
}
|
|
3555
|
-
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
3556
|
-
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
3557
|
-
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
3558
|
-
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
3559
|
-
}
|
|
3560
|
-
return { footer, fview, storedFooterCrc };
|
|
3561
|
-
}
|
|
3562
|
-
async function readFooter(blob) {
|
|
3563
|
-
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
3564
|
-
const { fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
3565
|
-
return {
|
|
3566
|
-
encrypted: (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0,
|
|
3567
|
-
footerCrc: storedFooterCrc
|
|
3568
|
-
};
|
|
3569
|
-
}
|
|
3570
|
-
async function footerSaysEncrypted(blob) {
|
|
3571
|
-
return (await readFooter(blob)).encrypted;
|
|
3572
|
-
}
|
|
3573
|
-
function readU64(view, offset, field) {
|
|
3574
|
-
const big = view.getBigUint64(offset, true);
|
|
3575
|
-
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
3576
|
-
throw crbmIntegrity(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
3577
|
-
}
|
|
3578
|
-
return Number(big);
|
|
3579
|
-
}
|
|
3580
|
-
var keepLimits = /* @__PURE__ */ new WeakMap();
|
|
3581
|
-
function openCrbmReaderKeeping(blob, options, keepChunkBytesUpTo) {
|
|
3582
|
-
const asked = { ...options };
|
|
3583
|
-
if (keepChunkBytesUpTo !== void 0) keepLimits.set(asked, keepChunkBytesUpTo);
|
|
3584
|
-
return CrbmReader.open(blob, asked);
|
|
3585
|
-
}
|
|
3586
|
-
function openTailBytes(options) {
|
|
3587
|
-
return Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
3588
|
-
}
|
|
3589
|
-
var CrbmReader = class _CrbmReader {
|
|
3590
|
-
constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc, metadata, metadataWeight, payloadEnd, kept) {
|
|
3591
|
-
this.blob = blob;
|
|
3592
|
-
this.objectSize = objectSize;
|
|
3593
|
-
this.generation = generation;
|
|
3594
|
-
this.lineage = lineage;
|
|
3595
|
-
this.totalCardinality = totalCardinality;
|
|
3596
|
-
this.index = index;
|
|
3597
|
-
this.servedFromTail = servedFromTail;
|
|
3598
|
-
this.crypto = crypto;
|
|
3599
|
-
this.footerCrc = footerCrc;
|
|
3600
|
-
this.metadata = metadata;
|
|
3601
|
-
this.metadataWeight = metadataWeight;
|
|
3602
|
-
this.payloadEnd = payloadEnd;
|
|
3603
|
-
this.kept = kept;
|
|
2639
|
+
var CrbmReader = class _CrbmReader {
|
|
2640
|
+
constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc, metadata, metadataWeight, payloadEnd, kept) {
|
|
2641
|
+
this.blob = blob;
|
|
2642
|
+
this.objectSize = objectSize;
|
|
2643
|
+
this.generation = generation;
|
|
2644
|
+
this.lineage = lineage;
|
|
2645
|
+
this.totalCardinality = totalCardinality;
|
|
2646
|
+
this.index = index;
|
|
2647
|
+
this.servedFromTail = servedFromTail;
|
|
2648
|
+
this.crypto = crypto;
|
|
2649
|
+
this.footerCrc = footerCrc;
|
|
2650
|
+
this.metadata = metadata;
|
|
2651
|
+
this.metadataWeight = metadataWeight;
|
|
2652
|
+
this.payloadEnd = payloadEnd;
|
|
2653
|
+
this.kept = kept;
|
|
3604
2654
|
}
|
|
3605
2655
|
blob;
|
|
3606
2656
|
objectSize;
|
|
@@ -4028,13 +3078,13 @@ var CrbmReader = class _CrbmReader {
|
|
|
4028
3078
|
* error.
|
|
4029
3079
|
*/
|
|
4030
3080
|
readChunks(chunkKeys, options = {}) {
|
|
4031
|
-
const
|
|
4032
|
-
const stream = this.streamChunks(chunkKeys, options,
|
|
3081
|
+
const open = { window: void 0 };
|
|
3082
|
+
const stream = this.streamChunks(chunkKeys, options, open);
|
|
4033
3083
|
return Object.assign(stream, {
|
|
4034
|
-
settled: () =>
|
|
3084
|
+
settled: () => open.window?.settle() ?? Promise.resolve()
|
|
4035
3085
|
});
|
|
4036
3086
|
}
|
|
4037
|
-
async *streamChunks(chunkKeys, options,
|
|
3087
|
+
async *streamChunks(chunkKeys, options, open) {
|
|
4038
3088
|
const width = options.concurrency ?? MAX_RANGES_IN_FLIGHT;
|
|
4039
3089
|
requirePositiveInteger(width, "concurrency");
|
|
4040
3090
|
validateChunkKeyOrder(chunkKeys);
|
|
@@ -4082,7 +3132,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
4082
3132
|
ramp !== false,
|
|
4083
3133
|
typeof ramp === "number" ? ramp : 1
|
|
4084
3134
|
);
|
|
4085
|
-
|
|
3135
|
+
open.window = window;
|
|
4086
3136
|
const cur = { range: void 0, at: 0 };
|
|
4087
3137
|
let last;
|
|
4088
3138
|
try {
|
|
@@ -4587,16 +3637,16 @@ function viaRetry(retry, request) {
|
|
|
4587
3637
|
return retry === void 0 ? request() : retry(request);
|
|
4588
3638
|
}
|
|
4589
3639
|
function validateRead(ref, keys) {
|
|
4590
|
-
validateUserRef(ref);
|
|
3640
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
4591
3641
|
if (keys.length > 0) validateSegmentRef({ ...ref });
|
|
4592
3642
|
for (const chunkKey of keys) validateChunkKey(chunkKey);
|
|
4593
3643
|
validateChunkKeyOrder(keys);
|
|
4594
3644
|
}
|
|
4595
3645
|
var Snapshot = class _Snapshot {
|
|
4596
|
-
constructor(resolution, target,
|
|
3646
|
+
constructor(resolution, target, open, prior) {
|
|
4597
3647
|
this.resolution = resolution;
|
|
4598
3648
|
this.target = target;
|
|
4599
|
-
this.open =
|
|
3649
|
+
this.open = open;
|
|
4600
3650
|
this.prior = prior;
|
|
4601
3651
|
}
|
|
4602
3652
|
resolution;
|
|
@@ -4607,8 +3657,8 @@ var Snapshot = class _Snapshot {
|
|
|
4607
3657
|
onReader;
|
|
4608
3658
|
opened;
|
|
4609
3659
|
outcome;
|
|
4610
|
-
static live(resolution, target,
|
|
4611
|
-
return new _Snapshot(resolution, target,
|
|
3660
|
+
static live(resolution, target, open, prior) {
|
|
3661
|
+
return new _Snapshot(resolution, target, open, prior);
|
|
4612
3662
|
}
|
|
4613
3663
|
static eager(reader) {
|
|
4614
3664
|
const snap = new _Snapshot(void 0, void 0, void 0, void 0);
|
|
@@ -4698,6 +3748,11 @@ var CrbmStorageChunkSource = class {
|
|
|
4698
3748
|
maxOpenIndexBytes,
|
|
4699
3749
|
...readerOptions
|
|
4700
3750
|
} = options;
|
|
3751
|
+
checkRequireEncryption(requireEncryption);
|
|
3752
|
+
if (maxOpenSegments !== void 0) requirePositiveInteger(maxOpenSegments, "maxOpenSegments");
|
|
3753
|
+
if (maxOpenIndexBytes !== void 0) {
|
|
3754
|
+
requirePositiveInteger(maxOpenIndexBytes, "maxOpenIndexBytes");
|
|
3755
|
+
}
|
|
4701
3756
|
if (keystore !== void 0 && registry === void 0) {
|
|
4702
3757
|
throw new CapabilityError(
|
|
4703
3758
|
"a keystore needs a storage backend \u2014 that is where the wrapped DEKs live. A bare IStorageDriver has no registry to hold them"
|
|
@@ -4710,7 +3765,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4710
3765
|
this.keystore = keystore;
|
|
4711
3766
|
this.requireEncryption = requireEncryption ?? false;
|
|
4712
3767
|
this.readerOptions = readerOptions;
|
|
4713
|
-
if (currentGenTtlMs !== void 0
|
|
3768
|
+
if (currentGenTtlMs !== void 0) {
|
|
4714
3769
|
requireFiniteNumber(currentGenTtlMs, {
|
|
4715
3770
|
min: 0,
|
|
4716
3771
|
what: "cache.genTtlMs",
|
|
@@ -5040,6 +4095,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5040
4095
|
* storage side can close the window; with no registry, no clock or `cache.genTtlMs: 0`, nothing bounds it at all.
|
|
5041
4096
|
*/
|
|
5042
4097
|
invalidate(ref) {
|
|
4098
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5043
4099
|
const key = segmentKey(ref);
|
|
5044
4100
|
this.invalidated.invalidate(key);
|
|
5045
4101
|
this.snapshots.delete(key);
|
|
@@ -5109,7 +4165,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5109
4165
|
* once the swept one is gone is exactly what a pin taken now should hold. One retry, then propagate.
|
|
5110
4166
|
*/
|
|
5111
4167
|
async pinGeneration(ref, lease) {
|
|
5112
|
-
validateUserRef(ref);
|
|
4168
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5113
4169
|
for (let moved = 0; ; moved++) {
|
|
5114
4170
|
try {
|
|
5115
4171
|
try {
|
|
@@ -5141,7 +4197,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5141
4197
|
* of the row's summary against it, since that check is made where the row's own generation is read.
|
|
5142
4198
|
*/
|
|
5143
4199
|
async pinGenerationAt(ref, at, lease) {
|
|
5144
|
-
validateUserRef(ref);
|
|
4200
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5145
4201
|
if (typeof at?.fingerprint !== "string")
|
|
5146
4202
|
throw new ValidationError(
|
|
5147
4203
|
`a pin of segment "${ref.segment}" needs the fingerprint of the object it names`
|
|
@@ -5291,7 +4347,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5291
4347
|
* the one read path with no resilience.
|
|
5292
4348
|
*
|
|
5293
4349
|
* **`status` is re-checked here**, on every open rather than once at pin time. A pin taken before a
|
|
5294
|
-
* crypto-shred must not keep unwrapping a DEK the shred
|
|
4350
|
+
* crypto-shred must not keep unwrapping a DEK whose wrapping the shred removed; a destroyed segment resolves no
|
|
4351
|
+
* generation
|
|
5295
4352
|
* for anyone, and a pin's read of one fails. That holds from the pin's next open: a reader already open keeps
|
|
5296
4353
|
* the key it unwrapped until it is evicted or invalidated.
|
|
5297
4354
|
*
|
|
@@ -5358,7 +4415,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5358
4415
|
* object, and the read fails if the generation is now another object.
|
|
5359
4416
|
*/
|
|
5360
4417
|
async getChunkAt(ref, generation, held) {
|
|
5361
|
-
validateUserRef(ref);
|
|
4418
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5362
4419
|
const reader = await this.readerAt(ref, generation, held);
|
|
5363
4420
|
if (reader === null) return null;
|
|
5364
4421
|
if (held !== void 0 && this.replacedPins.get(this.heldKey(ref, held.version, held.fingerprint)) !== void 0)
|
|
@@ -5416,7 +4473,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5416
4473
|
}
|
|
5417
4474
|
/** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
|
|
5418
4475
|
async listChunkKeysAt(ref, generation, held) {
|
|
5419
|
-
validateUserRef(ref);
|
|
4476
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5420
4477
|
const reader = await this.readerAt(ref, generation, held);
|
|
5421
4478
|
return reader === null ? [] : reader.chunkKeys();
|
|
5422
4479
|
}
|
|
@@ -5425,27 +4482,28 @@ var CrbmStorageChunkSource = class {
|
|
|
5425
4482
|
* for {@link getChunkAt}.
|
|
5426
4483
|
*/
|
|
5427
4484
|
async cardinalitiesAt(ref, generation, held) {
|
|
5428
|
-
validateUserRef(ref);
|
|
4485
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5429
4486
|
const reader = await this.readerAt(ref, generation, held);
|
|
5430
4487
|
return reader === null ? null : reader.cardinalities();
|
|
5431
4488
|
}
|
|
5432
4489
|
/** Size of a specific generation. `held` as for {@link getChunkAt}. */
|
|
5433
4490
|
async sizeOfAt(ref, generation, held) {
|
|
5434
|
-
validateUserRef(ref);
|
|
4491
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5435
4492
|
const reader = await this.readerAt(ref, generation, held);
|
|
5436
4493
|
return reader === null ? null : { sizeBytes: reader.sizeBytes };
|
|
5437
4494
|
}
|
|
5438
4495
|
/**
|
|
5439
4496
|
* Whether the segment is a registered name at all. With a registry: a row that is not a `destroyed` tombstone,
|
|
5440
4497
|
* including one with no generation yet (minted by `setRetention` before the first load), which somebody deliberately
|
|
5441
|
-
* created.
|
|
4498
|
+
* created. No generation resolves through a tombstone, so it answers `false`. Without a registry there is no
|
|
4499
|
+
* registration to
|
|
5442
4500
|
* consult, so the honest answer is whether the bucket holds any generation of it.
|
|
5443
4501
|
*
|
|
5444
4502
|
* Deliberately NOT served from the snapshot memo: the memo resolves to `null` for both states this exists to
|
|
5445
4503
|
* tell apart.
|
|
5446
4504
|
*/
|
|
5447
4505
|
async exists(ref) {
|
|
5448
|
-
validateUserRef(ref);
|
|
4506
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5449
4507
|
if (this.registry !== void 0) {
|
|
5450
4508
|
const row = await this.registry.get(ref);
|
|
5451
4509
|
return row !== null && row.status !== "destroyed";
|
|
@@ -5739,10 +4797,10 @@ var CrbmStorageChunkSource = class {
|
|
|
5739
4797
|
async *getChunksAt(ref, generation, keys, held, options) {
|
|
5740
4798
|
validateRead(ref, keys);
|
|
5741
4799
|
const retry = options?.retry;
|
|
5742
|
-
const
|
|
4800
|
+
const open = () => this.readerAt(ref, generation, held);
|
|
5743
4801
|
const segment = segmentKey(ref);
|
|
5744
4802
|
let since = this.invalidated.epoch;
|
|
5745
|
-
const reader = await viaRetry(retry,
|
|
4803
|
+
const reader = await viaRetry(retry, open);
|
|
5746
4804
|
if (reader === null) {
|
|
5747
4805
|
for (const key of keys) yield { key, bytes: null };
|
|
5748
4806
|
return;
|
|
@@ -5755,7 +4813,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5755
4813
|
while (await pull.advance()) {
|
|
5756
4814
|
if (this.invalidated.invalidatedSince(segment, since)) {
|
|
5757
4815
|
since = this.invalidated.epoch;
|
|
5758
|
-
if (await viaRetry(retry,
|
|
4816
|
+
if (await viaRetry(retry, open) === null) throw goneUnder(ref, generation);
|
|
5759
4817
|
}
|
|
5760
4818
|
yield pull.take((chunk) => chunk);
|
|
5761
4819
|
}
|
|
@@ -5771,11 +4829,11 @@ var CrbmStorageChunkSource = class {
|
|
|
5771
4829
|
}
|
|
5772
4830
|
}
|
|
5773
4831
|
async listChunkKeys(ref) {
|
|
5774
|
-
validateUserRef(ref);
|
|
4832
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5775
4833
|
return this.withFreshSnapshot(ref, (reader) => reader.chunkKeys(), []);
|
|
5776
4834
|
}
|
|
5777
4835
|
async sizeOf(ref) {
|
|
5778
|
-
validateUserRef(ref);
|
|
4836
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5779
4837
|
return this.withFreshSnapshot(
|
|
5780
4838
|
ref,
|
|
5781
4839
|
(reader) => ({ sizeBytes: reader.sizeBytes }),
|
|
@@ -5783,7 +4841,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5783
4841
|
);
|
|
5784
4842
|
}
|
|
5785
4843
|
async cardinalities(ref) {
|
|
5786
|
-
validateUserRef(ref);
|
|
4844
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5787
4845
|
return this.withFreshSnapshot(
|
|
5788
4846
|
ref,
|
|
5789
4847
|
(reader) => reader.cardinalities(),
|
|
@@ -5804,7 +4862,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5804
4862
|
* believed until the next time the object is opened, which holds it against the summary.
|
|
5805
4863
|
*/
|
|
5806
4864
|
async summary(ref) {
|
|
5807
|
-
validateUserRef(ref);
|
|
4865
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5808
4866
|
return this.described(
|
|
5809
4867
|
ref,
|
|
5810
4868
|
(generation, row) => summaryOfGeneration(generation, row),
|
|
@@ -5837,7 +4895,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5837
4895
|
}
|
|
5838
4896
|
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5839
4897
|
async summaryAt(ref, generation, held) {
|
|
5840
|
-
validateUserRef(ref);
|
|
4898
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5841
4899
|
const reader = await this.readerAt(ref, generation, held);
|
|
5842
4900
|
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
5843
4901
|
}
|
|
@@ -5851,7 +4909,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5851
4909
|
* are the generation's the row names, and `checkConsistency` is what finds such a row.
|
|
5852
4910
|
*/
|
|
5853
4911
|
async stat(ref) {
|
|
5854
|
-
validateUserRef(ref);
|
|
4912
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5855
4913
|
return this.described(
|
|
5856
4914
|
ref,
|
|
5857
4915
|
(generation, row) => ({
|
|
@@ -5863,7 +4921,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5863
4921
|
}
|
|
5864
4922
|
/** {@link stat} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
5865
4923
|
async statAt(ref, generation, held) {
|
|
5866
|
-
validateUserRef(ref);
|
|
4924
|
+
validateUserRef(ref, "read", PORT_KEYS);
|
|
5867
4925
|
const reader = await this.readerAt(ref, generation, held);
|
|
5868
4926
|
return reader === null ? null : sizedSummary(reader);
|
|
5869
4927
|
}
|
|
@@ -5885,8 +4943,9 @@ var CrbmStorageChunkSource = class {
|
|
|
5885
4943
|
*
|
|
5886
4944
|
* A *second* NotFound is pathological (GC outrunning resolution) and propagates rather than fabricating an
|
|
5887
4945
|
* absent answer — never return a wrong result. `ifGone` is returned when the segment has no committed
|
|
5888
|
-
* generation to serve: storage is empty, or the row
|
|
5889
|
-
*
|
|
4946
|
+
* generation to serve: storage is empty, or the row is destroyed (crypto-shredded, destroyed in the clear or
|
|
4947
|
+
* dropped), in which case
|
|
4948
|
+
* reading empty is the documented outcome rather than a failure.
|
|
5890
4949
|
*
|
|
5891
4950
|
* **Bounded to exactly two resolve-and-open round trips.** That is a cost contract, not a detail: the retry
|
|
5892
4951
|
* re-reads the *registry*, so an unbounded one would hammer the shared, throttle-prone resource in a
|
|
@@ -5984,9 +5043,10 @@ var PinnedStorageChunkSource = class {
|
|
|
5984
5043
|
this.inner = inner;
|
|
5985
5044
|
this.pins = pins;
|
|
5986
5045
|
for (const [key, pin] of pins) {
|
|
5987
|
-
|
|
5046
|
+
const held = pin;
|
|
5047
|
+
if (held.generation !== null && (typeof held.version !== "string" || typeof held.fingerprint !== "string"))
|
|
5988
5048
|
throw new ValidationError(
|
|
5989
|
-
`pin for "${key}" names generation ${
|
|
5049
|
+
`pin for "${key}" names generation ${held.generation} without its version and object fingerprint`
|
|
5990
5050
|
);
|
|
5991
5051
|
}
|
|
5992
5052
|
}
|
|
@@ -6093,6 +5153,7 @@ var PinnedStorageChunkSource = class {
|
|
|
6093
5153
|
|
|
6094
5154
|
// src/core/cooperative.ts
|
|
6095
5155
|
var YIELD_EVERY = 1024;
|
|
5156
|
+
var YIELD_EVERY_IDS = 1 << 14;
|
|
6096
5157
|
function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
6097
5158
|
if (clock === void 0) return () => null;
|
|
6098
5159
|
const sleep = clock.sleep;
|
|
@@ -6120,6 +5181,51 @@ var ChunkLoadInput = class {
|
|
|
6120
5181
|
}
|
|
6121
5182
|
chunks;
|
|
6122
5183
|
};
|
|
5184
|
+
var sourceErrors = /* @__PURE__ */ new WeakSet();
|
|
5185
|
+
function raisedBySource(err) {
|
|
5186
|
+
if (typeof err === "object" && err !== null) sourceErrors.add(err);
|
|
5187
|
+
return err;
|
|
5188
|
+
}
|
|
5189
|
+
function isSourceError(err) {
|
|
5190
|
+
return typeof err === "object" && err !== null && sourceErrors.has(err);
|
|
5191
|
+
}
|
|
5192
|
+
var SortedLoadInput = class {
|
|
5193
|
+
constructor(ids) {
|
|
5194
|
+
this.ids = ids;
|
|
5195
|
+
}
|
|
5196
|
+
ids;
|
|
5197
|
+
failure;
|
|
5198
|
+
/** Keep `error` as what the ids raised, unless an earlier one is kept. */
|
|
5199
|
+
fail(error) {
|
|
5200
|
+
this.failure ??= { error };
|
|
5201
|
+
}
|
|
5202
|
+
/** What the load throws for a write that failed with `err`: the error the ids raised, when they raised one. */
|
|
5203
|
+
thrown(err) {
|
|
5204
|
+
return this.failure === void 0 ? err : raisedBySource(this.failure.error);
|
|
5205
|
+
}
|
|
5206
|
+
};
|
|
5207
|
+
function kindOfOption(value) {
|
|
5208
|
+
if (value === null) return "null";
|
|
5209
|
+
if (Array.isArray(value)) return "an array";
|
|
5210
|
+
const kind = typeof value;
|
|
5211
|
+
return kind === "object" || kind === "undefined" ? `an ${kind}` : `a ${kind}`;
|
|
5212
|
+
}
|
|
5213
|
+
function wantsSorted(sorted) {
|
|
5214
|
+
if (sorted === void 0 || sorted === false) return false;
|
|
5215
|
+
if (sorted !== true) {
|
|
5216
|
+
throw new ValidationError(`sorted must be a boolean; got ${kindOfOption(sorted)}`);
|
|
5217
|
+
}
|
|
5218
|
+
return true;
|
|
5219
|
+
}
|
|
5220
|
+
function sortedLoadInput(prepared, sorted) {
|
|
5221
|
+
if (!wantsSorted(sorted)) return prepared;
|
|
5222
|
+
if (prepared instanceof DecodedLoadInput) {
|
|
5223
|
+
throw new ValidationError(
|
|
5224
|
+
"sorted: true applies to ids, and this load was given a bitmap ({ serialized }, { bitmap } or a bitmap itself), which is already held whole and is written from its own chunks: leave sorted off"
|
|
5225
|
+
);
|
|
5226
|
+
}
|
|
5227
|
+
return new SortedLoadInput(prepared);
|
|
5228
|
+
}
|
|
6123
5229
|
function tagOf(value) {
|
|
6124
5230
|
return Object.prototype.toString.call(value).slice(8, -1);
|
|
6125
5231
|
}
|
|
@@ -6204,6 +5310,98 @@ function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
|
6204
5310
|
}
|
|
6205
5311
|
}
|
|
6206
5312
|
|
|
5313
|
+
// src/core/sorted-chunks.ts
|
|
5314
|
+
var NOT_ASCENDING = "lower than the id before it; { sorted: true } takes ids in ascending order (repeats are fine): sort at the source, or leave sorted off";
|
|
5315
|
+
var SourceFailed = class extends Error {
|
|
5316
|
+
constructor() {
|
|
5317
|
+
super("the ids of a sorted load raised an error");
|
|
5318
|
+
}
|
|
5319
|
+
};
|
|
5320
|
+
function sortedChunks(input, codec, clock) {
|
|
5321
|
+
return chunksOf(input, codec, clock);
|
|
5322
|
+
}
|
|
5323
|
+
function chunkCutter(codec) {
|
|
5324
|
+
const remainders = new Uint32Array(1 << 16);
|
|
5325
|
+
let position = 0;
|
|
5326
|
+
let last = -1;
|
|
5327
|
+
let chunk = -1;
|
|
5328
|
+
let held = 0;
|
|
5329
|
+
const cut = () => ({
|
|
5330
|
+
chunkKey: chunk,
|
|
5331
|
+
bitmap: codec.fromValues(remainders.subarray(0, held))
|
|
5332
|
+
});
|
|
5333
|
+
return {
|
|
5334
|
+
accept(id) {
|
|
5335
|
+
if (!Number.isInteger(id) || id < 0 || id > U32_MAX) {
|
|
5336
|
+
const cause = refusalOfId(id);
|
|
5337
|
+
const positioned = new ValidationError(`ids[${position}]: ${cause.message}`);
|
|
5338
|
+
positioned.cause = cause;
|
|
5339
|
+
throw positioned;
|
|
5340
|
+
}
|
|
5341
|
+
if (id <= last) {
|
|
5342
|
+
if (id < last) throw new ValidationError(`ids[${position}]: ${NOT_ASCENDING}`);
|
|
5343
|
+
position++;
|
|
5344
|
+
return void 0;
|
|
5345
|
+
}
|
|
5346
|
+
last = id;
|
|
5347
|
+
position++;
|
|
5348
|
+
const key = id >>> 16;
|
|
5349
|
+
let closed;
|
|
5350
|
+
if (key !== chunk) {
|
|
5351
|
+
if (held > 0) closed = cut();
|
|
5352
|
+
chunk = key;
|
|
5353
|
+
held = 0;
|
|
5354
|
+
}
|
|
5355
|
+
remainders[held++] = id & 65535;
|
|
5356
|
+
return closed;
|
|
5357
|
+
},
|
|
5358
|
+
rest() {
|
|
5359
|
+
return held > 0 ? cut() : void 0;
|
|
5360
|
+
}
|
|
5361
|
+
};
|
|
5362
|
+
}
|
|
5363
|
+
async function* chunksOf(input, codec, clock) {
|
|
5364
|
+
const cutter = chunkCutter(codec);
|
|
5365
|
+
const tick = yieldEvery(clock, YIELD_EVERY_IDS);
|
|
5366
|
+
let handing = false;
|
|
5367
|
+
const { ids } = input;
|
|
5368
|
+
try {
|
|
5369
|
+
if (Symbol.asyncIterator in ids) {
|
|
5370
|
+
for await (const id of ids) {
|
|
5371
|
+
const closed = cutter.accept(id);
|
|
5372
|
+
if (closed !== void 0) {
|
|
5373
|
+
handing = true;
|
|
5374
|
+
yield closed;
|
|
5375
|
+
handing = false;
|
|
5376
|
+
}
|
|
5377
|
+
const pause = tick();
|
|
5378
|
+
if (pause !== null) await pause;
|
|
5379
|
+
}
|
|
5380
|
+
} else {
|
|
5381
|
+
for (const id of ids) {
|
|
5382
|
+
const closed = cutter.accept(id);
|
|
5383
|
+
if (closed !== void 0) {
|
|
5384
|
+
handing = true;
|
|
5385
|
+
yield closed;
|
|
5386
|
+
handing = false;
|
|
5387
|
+
}
|
|
5388
|
+
const pause = tick();
|
|
5389
|
+
if (pause !== null) await pause;
|
|
5390
|
+
}
|
|
5391
|
+
}
|
|
5392
|
+
const last = cutter.rest();
|
|
5393
|
+
if (last !== void 0) {
|
|
5394
|
+
handing = true;
|
|
5395
|
+
yield last;
|
|
5396
|
+
handing = false;
|
|
5397
|
+
}
|
|
5398
|
+
} catch (err) {
|
|
5399
|
+
if (handing) throw err;
|
|
5400
|
+
input.fail(err);
|
|
5401
|
+
throw new SourceFailed();
|
|
5402
|
+
}
|
|
5403
|
+
}
|
|
5404
|
+
|
|
6207
5405
|
// src/core/crbm/writer.ts
|
|
6208
5406
|
var CrbmWriter = class {
|
|
6209
5407
|
constructor(sink, options) {
|
|
@@ -6351,25 +5549,42 @@ var CrbmWriter = class {
|
|
|
6351
5549
|
block.set(EXT_MAGIC, sectionsLength + 8);
|
|
6352
5550
|
return block;
|
|
6353
5551
|
}
|
|
5552
|
+
/**
|
|
5553
|
+
* The index region: per entry, four varints (key delta, offset gap, length, cardinality) and a fixed 4-byte
|
|
5554
|
+
* little-endian crc32c. A first pass sizes the region exactly, so the bytes are written once into one buffer and no
|
|
5555
|
+
* intermediate array is built.
|
|
5556
|
+
*/
|
|
6354
5557
|
encodeIndex() {
|
|
6355
|
-
|
|
5558
|
+
let size = 0;
|
|
6356
5559
|
let prevKey = 0;
|
|
6357
5560
|
let prevEnd = PAYLOAD_START;
|
|
6358
5561
|
for (const e of this.entries) {
|
|
6359
|
-
|
|
6360
|
-
writeVarint(out, e.offset - prevEnd);
|
|
6361
|
-
writeVarint(out, e.length);
|
|
6362
|
-
writeVarint(out, e.cardinality);
|
|
6363
|
-
out.push(
|
|
6364
|
-
e.crc32c & 255,
|
|
6365
|
-
e.crc32c >>> 8 & 255,
|
|
6366
|
-
e.crc32c >>> 16 & 255,
|
|
6367
|
-
e.crc32c >>> 24 & 255
|
|
6368
|
-
);
|
|
5562
|
+
size += varintLength(e.chunkKey - prevKey) + varintLength(e.offset - prevEnd) + varintLength(e.length) + varintLength(e.cardinality) + 4;
|
|
6369
5563
|
prevKey = e.chunkKey;
|
|
6370
5564
|
prevEnd = e.offset + e.length;
|
|
6371
5565
|
}
|
|
6372
|
-
|
|
5566
|
+
const out = new Uint8Array(size);
|
|
5567
|
+
let pos = 0;
|
|
5568
|
+
prevKey = 0;
|
|
5569
|
+
prevEnd = PAYLOAD_START;
|
|
5570
|
+
for (const e of this.entries) {
|
|
5571
|
+
pos = writeVarintAt(out, pos, e.chunkKey - prevKey);
|
|
5572
|
+
pos = writeVarintAt(out, pos, e.offset - prevEnd);
|
|
5573
|
+
pos = writeVarintAt(out, pos, e.length);
|
|
5574
|
+
pos = writeVarintAt(out, pos, e.cardinality);
|
|
5575
|
+
out[pos++] = e.crc32c & 255;
|
|
5576
|
+
out[pos++] = e.crc32c >>> 8 & 255;
|
|
5577
|
+
out[pos++] = e.crc32c >>> 16 & 255;
|
|
5578
|
+
out[pos++] = e.crc32c >>> 24 & 255;
|
|
5579
|
+
prevKey = e.chunkKey;
|
|
5580
|
+
prevEnd = e.offset + e.length;
|
|
5581
|
+
}
|
|
5582
|
+
if (pos !== out.length) {
|
|
5583
|
+
throw new IntegrityError(
|
|
5584
|
+
`the chunk index was sized at ${out.length} bytes and encoded as ${pos}; the index was not written`
|
|
5585
|
+
);
|
|
5586
|
+
}
|
|
5587
|
+
return out;
|
|
6373
5588
|
}
|
|
6374
5589
|
buildFooter(indexOffset, indexRegion, indexNonce, indexTag) {
|
|
6375
5590
|
const encrypted = this.options.crypto !== void 0;
|
|
@@ -6451,13 +5666,15 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
6451
5666
|
let cardinality = 0;
|
|
6452
5667
|
let identity;
|
|
6453
5668
|
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
5669
|
+
chunkCount = 0;
|
|
5670
|
+
cardinality = 0;
|
|
6454
5671
|
const writer = new CrbmWriter(sink, {
|
|
6455
5672
|
generation: key.generation,
|
|
6456
5673
|
crypto: options.crypto,
|
|
6457
5674
|
metadata: options.metadata,
|
|
6458
5675
|
entropy: options.entropy
|
|
6459
5676
|
});
|
|
6460
|
-
for (const chunk of chunks) {
|
|
5677
|
+
for (const chunk of chunks()) {
|
|
6461
5678
|
await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
|
|
6462
5679
|
chunkCount++;
|
|
6463
5680
|
cardinality += chunk.cardinality;
|
|
@@ -6486,24 +5703,42 @@ async function writeCrbmGenerationStream(driver, key, chunks, options) {
|
|
|
6486
5703
|
let cardinality = 0;
|
|
6487
5704
|
const tick = yieldEvery(options.clock);
|
|
6488
5705
|
let identity;
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
if (
|
|
6498
|
-
|
|
6499
|
-
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
6500
|
-
chunkKeys.push(chunkKey);
|
|
6501
|
-
cardinality += bitmap.size;
|
|
6502
|
-
const pause = tick();
|
|
6503
|
-
if (pause !== null) await pause;
|
|
5706
|
+
let runs = 0;
|
|
5707
|
+
let rerun;
|
|
5708
|
+
let firstRun;
|
|
5709
|
+
const put = driver.putImmutable(key, async (sink) => {
|
|
5710
|
+
if (++runs > 1) {
|
|
5711
|
+
rerun = new IntegrityError(
|
|
5712
|
+
`the storage driver ran the write of ${key.segment}.${key.generation} again: its chunks are streamed and can be read once, so nothing was written`
|
|
5713
|
+
);
|
|
5714
|
+
if (firstRun !== void 0) rerun.cause = firstRun.error;
|
|
5715
|
+
throw rerun;
|
|
6504
5716
|
}
|
|
6505
|
-
|
|
6506
|
-
|
|
5717
|
+
try {
|
|
5718
|
+
const writer = new CrbmWriter(sink, {
|
|
5719
|
+
generation: key.generation,
|
|
5720
|
+
crypto: options.crypto,
|
|
5721
|
+
metadata: options.metadata,
|
|
5722
|
+
entropy: options.entropy
|
|
5723
|
+
});
|
|
5724
|
+
for await (const { chunkKey, bitmap } of chunks) {
|
|
5725
|
+
if (bitmap.isEmpty) continue;
|
|
5726
|
+
bitmap.optimize?.();
|
|
5727
|
+
await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
|
|
5728
|
+
chunkKeys.push(chunkKey);
|
|
5729
|
+
cardinality += bitmap.size;
|
|
5730
|
+
const pause = tick();
|
|
5731
|
+
if (pause !== null) await pause;
|
|
5732
|
+
}
|
|
5733
|
+
await writer.finish();
|
|
5734
|
+
identity = writer.identity;
|
|
5735
|
+
} catch (err) {
|
|
5736
|
+
firstRun = { error: err };
|
|
5737
|
+
throw err;
|
|
5738
|
+
}
|
|
5739
|
+
});
|
|
5740
|
+
const { size, sha256 } = await put.catch((err) => {
|
|
5741
|
+
throw rerun ?? err;
|
|
6507
5742
|
});
|
|
6508
5743
|
if (identity === void 0) {
|
|
6509
5744
|
throw new IntegrityError(
|
|
@@ -6567,17 +5802,18 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6567
5802
|
let throughChurn = false;
|
|
6568
5803
|
let waited;
|
|
6569
5804
|
let sawUnanswered = false;
|
|
5805
|
+
let readFault;
|
|
5806
|
+
const readAgain = readOf(registry, key, (e) => readFault = e);
|
|
5807
|
+
let tokenReported;
|
|
6570
5808
|
let lastRecord;
|
|
6571
5809
|
const sent = registry.reportsWriteTokens === true ? /* @__PURE__ */ new Set() : void 0;
|
|
6572
|
-
const report = sent === void 0 ? void 0 : (token) => void sent.add(token);
|
|
5810
|
+
const report = sent === void 0 ? void 0 : (token) => void (sent.add(token), tokenReported = true);
|
|
6573
5811
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
6574
5812
|
const reused = attempt === 0 && options.row !== void 0;
|
|
6575
|
-
const holdsRow = fresh === void 0;
|
|
5813
|
+
const holdsRow = fresh === void 0 || fresh === waited;
|
|
6576
5814
|
const record = fresh !== void 0 ? fresh : reused ? options.row ?? null : await registry.get(key);
|
|
6577
|
-
const write = holdsRow
|
|
6578
|
-
|
|
6579
|
-
...report !== void 0 ? { onToken: report } : {}
|
|
6580
|
-
} : void 0;
|
|
5815
|
+
const write = writeOptions(holdsRow ? { held: record } : {}, report);
|
|
5816
|
+
tokenReported = false;
|
|
6581
5817
|
fresh = void 0;
|
|
6582
5818
|
unanswered = void 0;
|
|
6583
5819
|
lastRecord = record;
|
|
@@ -6614,7 +5850,7 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6614
5850
|
if (record !== null && held != null && record === waited && onlyLeasesDiffer(held, record, options.expectRowIgnores)) {
|
|
6615
5851
|
throughChurn = true;
|
|
6616
5852
|
} else {
|
|
6617
|
-
const settled2 = record === waited ? void 0 : await churn.settle(held, record,
|
|
5853
|
+
const settled2 = record === waited ? void 0 : await churn.settle(held, record, readAgain, options.expectRowIgnores);
|
|
6618
5854
|
if (settled2 !== void 0) {
|
|
6619
5855
|
waited = settled2.row;
|
|
6620
5856
|
fresh = settled2.row;
|
|
@@ -6642,7 +5878,7 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6642
5878
|
)).token;
|
|
6643
5879
|
} else if (record.status === "destroyed") {
|
|
6644
5880
|
throw new ValidationError(
|
|
6645
|
-
`segment "${key.segment}" was destroyed
|
|
5881
|
+
`segment "${key.segment}" was destroyed while generation ${key.generation} was being written \u2014 refusing to publish it: no generation resolves through a destroyed row. Use a new segment.`
|
|
6646
5882
|
);
|
|
6647
5883
|
} else if (record.currentGen === null) {
|
|
6648
5884
|
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
@@ -6690,13 +5926,14 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6690
5926
|
};
|
|
6691
5927
|
} catch (err) {
|
|
6692
5928
|
if (isWriteConflictError(err)) {
|
|
5929
|
+
if (isUnsettledConflict(err)) sawUnanswered = noteUnsettled(options);
|
|
6693
5930
|
failedOn = record;
|
|
6694
5931
|
if (throughChurn) attempt -= 1;
|
|
6695
5932
|
continue;
|
|
6696
5933
|
}
|
|
6697
|
-
if (!isTransientError(err)) throw err;
|
|
6698
|
-
|
|
6699
|
-
sawUnanswered =
|
|
5934
|
+
if (!isTransientError(err) || err === readFault) throw err;
|
|
5935
|
+
if (sent !== void 0 && !tokenReported) throw err;
|
|
5936
|
+
sawUnanswered = noteUnsettled(options);
|
|
6700
5937
|
let now;
|
|
6701
5938
|
try {
|
|
6702
5939
|
now = await registry.get(key);
|
|
@@ -6740,6 +5977,23 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6740
5977
|
`publishGeneration: contention setting currentGen for ${key.segment}${CONTENTION_NEXT_STEP}`
|
|
6741
5978
|
);
|
|
6742
5979
|
}
|
|
5980
|
+
function writeOptions(held, onToken) {
|
|
5981
|
+
return held.held === void 0 && onToken === void 0 ? void 0 : { ...held, ...onToken === void 0 ? {} : { onToken } };
|
|
5982
|
+
}
|
|
5983
|
+
function readOf(registry, key, onFault) {
|
|
5984
|
+
return async () => {
|
|
5985
|
+
try {
|
|
5986
|
+
return await registry.get(key);
|
|
5987
|
+
} catch (err) {
|
|
5988
|
+
onFault(err);
|
|
5989
|
+
throw err;
|
|
5990
|
+
}
|
|
5991
|
+
};
|
|
5992
|
+
}
|
|
5993
|
+
function noteUnsettled(options) {
|
|
5994
|
+
options.onUnanswered?.();
|
|
5995
|
+
return true;
|
|
5996
|
+
}
|
|
6743
5997
|
var REFUSED = { published: false };
|
|
6744
5998
|
async function publishGeneration(registry, key, options) {
|
|
6745
5999
|
return (await publishGenerationKept(registry, key, options)).published;
|
|
@@ -6747,7 +6001,7 @@ async function publishGeneration(registry, key, options) {
|
|
|
6747
6001
|
async function landedHere(now, before, key, holdsOwnObject, sent) {
|
|
6748
6002
|
if (now === null || now.status === "destroyed" || now.currentGen !== key.generation) return false;
|
|
6749
6003
|
if (before !== null && !sameIncarnation(before, now)) return false;
|
|
6750
|
-
if (sent !== void 0) return
|
|
6004
|
+
if (sent !== void 0) return pointerSetBy(now, before, key.generation, sent);
|
|
6751
6005
|
if (holdsOwnObject === void 0) {
|
|
6752
6006
|
throw new TransientError(
|
|
6753
6007
|
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled, since the caller gave no way to tell its own object from another's. Its object is kept; re-run the write, which numbers past it.`
|
|
@@ -6768,18 +6022,21 @@ function outcomeUnknown(key, cause) {
|
|
|
6768
6022
|
|
|
6769
6023
|
// src/core/bulk-load.ts
|
|
6770
6024
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
6771
|
-
var YIELD_EVERY_IDS = 1 << 14;
|
|
6772
6025
|
async function encodedChunksOfInput(ids, codec, clock) {
|
|
6773
|
-
|
|
6026
|
+
const { bitmap } = ids instanceof DecodedLoadInput ? ids : { bitmap: void 0 };
|
|
6027
|
+
if (bitmap?.encodeChunks !== void 0) {
|
|
6774
6028
|
const pause = yieldEvery(clock, 1);
|
|
6775
6029
|
await pause();
|
|
6776
|
-
|
|
6030
|
+
let first = bitmap.encodeChunks();
|
|
6777
6031
|
await pause();
|
|
6778
|
-
return
|
|
6032
|
+
return () => {
|
|
6033
|
+
const chunks = first ?? bitmap.encodeChunks();
|
|
6034
|
+
first = void 0;
|
|
6035
|
+
return chunks;
|
|
6036
|
+
};
|
|
6779
6037
|
}
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
);
|
|
6038
|
+
const bucketed = await bucketIds(bitmap ?? ids, codec, clock);
|
|
6039
|
+
return () => encodeEach(bucketed);
|
|
6783
6040
|
}
|
|
6784
6041
|
async function bucketIds(ids, codec, clock) {
|
|
6785
6042
|
const byChunk = /* @__PURE__ */ new Map();
|
|
@@ -6852,47 +6109,22 @@ function refuseCleartextWhenEncryptionRequired(options, ref, currentGen) {
|
|
|
6852
6109
|
}
|
|
6853
6110
|
}
|
|
6854
6111
|
async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
6112
|
+
checkRequireEncryption(options.requireEncryption);
|
|
6855
6113
|
requireKeystoreWhenEncryptionRequired(options);
|
|
6856
6114
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
6857
6115
|
const entropy = requireEntropy(options.entropy, "a load");
|
|
6858
|
-
let
|
|
6859
|
-
if (ids instanceof
|
|
6860
|
-
|
|
6116
|
+
let write;
|
|
6117
|
+
if (ids instanceof SortedLoadInput) {
|
|
6118
|
+
write = (key2, writeOptions2) => writeSorted(driver, key2, ids, codec, writeOptions2);
|
|
6861
6119
|
} else {
|
|
6862
|
-
chunks = await
|
|
6863
|
-
|
|
6864
|
-
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6865
|
-
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
6866
|
-
const passed = options.row;
|
|
6867
|
-
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(ref) : null;
|
|
6868
|
-
if (existing?.status === "destroyed") {
|
|
6869
|
-
throw new ValidationError(
|
|
6870
|
-
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
6871
|
-
);
|
|
6872
|
-
}
|
|
6873
|
-
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
6874
|
-
throw new KeyUnavailableError(
|
|
6875
|
-
`segment "${ref.segment}" is encrypted but this load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
6876
|
-
);
|
|
6120
|
+
const chunks = await encodedBeforeTheRow(ids, codec, options.clock);
|
|
6121
|
+
write = (key2, writeOptions2) => writeEncodedChunks(driver, key2, chunks, writeOptions2);
|
|
6877
6122
|
}
|
|
6123
|
+
const existing = await rowForWrite(ref, options);
|
|
6878
6124
|
const generation = await ahead.generation;
|
|
6879
6125
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
if (options.keystore !== void 0) {
|
|
6883
|
-
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0) {
|
|
6884
|
-
const early = ahead.unwrapped;
|
|
6885
|
-
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
6886
|
-
crypto = { aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
6887
|
-
} else if (existing !== null && existing.currentGen !== null) {
|
|
6888
|
-
refuseCleartextWhenEncryptionRequired(options, ref, existing.currentGen);
|
|
6889
|
-
} else {
|
|
6890
|
-
const minted = await options.keystore.createDek();
|
|
6891
|
-
newWrapped = minted.wrapped;
|
|
6892
|
-
crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
6893
|
-
}
|
|
6894
|
-
}
|
|
6895
|
-
const { size, sha256, chunkCount, cardinality, fingerprint, tailBytes } = await writeEncodedChunks(driver, key, chunks, {
|
|
6126
|
+
const { crypto, newWrapped } = await keyForWrite(existing, key, ahead, options);
|
|
6127
|
+
const { size, sha256, chunkCount, cardinality, fingerprint, tailBytes } = await write(key, {
|
|
6896
6128
|
crypto,
|
|
6897
6129
|
clock: options.clock,
|
|
6898
6130
|
metadata: options.metadata,
|
|
@@ -6904,48 +6136,95 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6904
6136
|
{ cardinality, metadata: options.metadata, fingerprint, tailBytes },
|
|
6905
6137
|
crypto?.aead
|
|
6906
6138
|
);
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6139
|
+
const result = {
|
|
6140
|
+
size,
|
|
6141
|
+
sha256,
|
|
6142
|
+
chunkCount,
|
|
6143
|
+
cardinality,
|
|
6144
|
+
summary,
|
|
6145
|
+
wrappedDeks: newWrapped,
|
|
6146
|
+
encrypted: crypto !== void 0,
|
|
6147
|
+
fingerprint
|
|
6148
|
+
};
|
|
6149
|
+
if (options.registry === void 0 || options.publish === false) return result;
|
|
6150
|
+
const published = await publishGenerationKept(options.registry, key, {
|
|
6151
|
+
wrappedDeks: newWrapped,
|
|
6152
|
+
summary,
|
|
6153
|
+
footerCrc: footerCrcOf(fingerprint),
|
|
6154
|
+
cleartext: crypto === void 0,
|
|
6155
|
+
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
6156
|
+
clock: options.clock
|
|
6157
|
+
});
|
|
6158
|
+
const becameCurrent = published.published;
|
|
6159
|
+
if (becameCurrent) {
|
|
6160
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
6161
|
+
kind: "segment.publish",
|
|
6162
|
+
namespace: key.namespace,
|
|
6163
|
+
segment: key.segment,
|
|
6164
|
+
...incarnationField(published.token),
|
|
6165
|
+
generation: key.generation
|
|
6915
6166
|
});
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
});
|
|
6167
|
+
}
|
|
6168
|
+
return { ...result, becameCurrent };
|
|
6169
|
+
}
|
|
6170
|
+
async function encodedBeforeTheRow(ids, codec, clock) {
|
|
6171
|
+
try {
|
|
6172
|
+
if (ids instanceof ChunkLoadInput) {
|
|
6173
|
+
const chunks = await collectChunks(ids, codec, clock);
|
|
6174
|
+
return () => encodeEach(rangeCheckedAtWrite(chunks));
|
|
6925
6175
|
}
|
|
6926
|
-
return
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
6933
|
-
|
|
6934
|
-
|
|
6935
|
-
|
|
6936
|
-
|
|
6176
|
+
return await encodedChunksOfInput(ids, codec, clock);
|
|
6177
|
+
} catch (err) {
|
|
6178
|
+
throw raisedBySource(err);
|
|
6179
|
+
}
|
|
6180
|
+
}
|
|
6181
|
+
async function rowForWrite(ref, options) {
|
|
6182
|
+
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6183
|
+
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
6184
|
+
const passed = options.row;
|
|
6185
|
+
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(ref) : null;
|
|
6186
|
+
if (existing?.status === "destroyed") {
|
|
6187
|
+
throw new ValidationError(
|
|
6188
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to write: no generation resolves through a destroyed row; use a new segment`
|
|
6189
|
+
);
|
|
6190
|
+
}
|
|
6191
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0 && options.keystore === void 0) {
|
|
6192
|
+
throw new KeyUnavailableError(
|
|
6193
|
+
`segment "${ref.segment}" is encrypted but this load has no keystore \u2014 refusing to write a cleartext generation onto an encrypted segment. Pass the keystore holding its DEK.`
|
|
6194
|
+
);
|
|
6195
|
+
}
|
|
6196
|
+
return existing;
|
|
6197
|
+
}
|
|
6198
|
+
async function keyForWrite(existing, key, ahead, options) {
|
|
6199
|
+
if (options.keystore === void 0) return {};
|
|
6200
|
+
if (existing?.wrappedDeks !== void 0 && existing.wrappedDeks.length > 0) {
|
|
6201
|
+
const early = ahead.unwrapped;
|
|
6202
|
+
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
6203
|
+
return { crypto: { aead, aadFor: (scope) => aadFor(key, key.generation, scope) } };
|
|
6204
|
+
}
|
|
6205
|
+
if (existing !== null && existing.currentGen !== null) {
|
|
6206
|
+
refuseCleartextWhenEncryptionRequired(options, key, existing.currentGen);
|
|
6207
|
+
return {};
|
|
6937
6208
|
}
|
|
6209
|
+
const minted = await options.keystore.createDek();
|
|
6938
6210
|
return {
|
|
6939
|
-
|
|
6940
|
-
|
|
6941
|
-
chunkCount,
|
|
6942
|
-
cardinality,
|
|
6943
|
-
summary,
|
|
6944
|
-
wrappedDeks: newWrapped,
|
|
6945
|
-
encrypted: crypto !== void 0,
|
|
6946
|
-
fingerprint
|
|
6211
|
+
crypto: { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) },
|
|
6212
|
+
newWrapped: minted.wrapped
|
|
6947
6213
|
};
|
|
6948
6214
|
}
|
|
6215
|
+
async function writeSorted(driver, key, input, codec, options) {
|
|
6216
|
+
try {
|
|
6217
|
+
const { chunkKeys, ...written } = await writeCrbmGenerationStream(
|
|
6218
|
+
driver,
|
|
6219
|
+
key,
|
|
6220
|
+
sortedChunks(input, codec, options.clock),
|
|
6221
|
+
options
|
|
6222
|
+
);
|
|
6223
|
+
return { ...written, chunkCount: chunkKeys.length };
|
|
6224
|
+
} catch (err) {
|
|
6225
|
+
throw input.thrown(err);
|
|
6226
|
+
}
|
|
6227
|
+
}
|
|
6949
6228
|
function* rangeCheckedAtWrite(chunks) {
|
|
6950
6229
|
for (const chunk of chunks) {
|
|
6951
6230
|
const max = chunk.bitmap.maximum?.();
|
|
@@ -7039,7 +6318,84 @@ async function readSeed(storage, ref, row, generation, keep, listed2, leased, re
|
|
|
7039
6318
|
return read.filter((e) => e !== void 0);
|
|
7040
6319
|
}
|
|
7041
6320
|
|
|
6321
|
+
// src/core/settle.ts
|
|
6322
|
+
function writeTokens(registry, onSent) {
|
|
6323
|
+
const sent = registry.reportsWriteTokens === true ? /* @__PURE__ */ new Set() : void 0;
|
|
6324
|
+
return {
|
|
6325
|
+
sent,
|
|
6326
|
+
options: sent === void 0 ? void 0 : {
|
|
6327
|
+
onToken: (token) => {
|
|
6328
|
+
sent.add(token);
|
|
6329
|
+
onSent?.(token);
|
|
6330
|
+
}
|
|
6331
|
+
},
|
|
6332
|
+
wrote: (row) => row !== null && sent !== void 0 && (sent.has(row.token) || sent.has(row.pointerId))
|
|
6333
|
+
};
|
|
6334
|
+
}
|
|
6335
|
+
var READ_BACK_TRIES = 3;
|
|
6336
|
+
async function readBack(read, unanswered, waits = {}) {
|
|
6337
|
+
let last;
|
|
6338
|
+
for (let attempt = 0; attempt < READ_BACK_TRIES; attempt++) {
|
|
6339
|
+
try {
|
|
6340
|
+
return await read();
|
|
6341
|
+
} catch (err) {
|
|
6342
|
+
if (!isTransientError(err)) throw err;
|
|
6343
|
+
last = err;
|
|
6344
|
+
}
|
|
6345
|
+
if (attempt < READ_BACK_TRIES - 1) {
|
|
6346
|
+
await jitteredWait(waits.clock, waits.rng, UNANSWERED_BASE_MS * 2 ** attempt);
|
|
6347
|
+
}
|
|
6348
|
+
}
|
|
6349
|
+
if (unanswered.cause === void 0) unanswered.cause = last;
|
|
6350
|
+
throw unanswered;
|
|
6351
|
+
}
|
|
6352
|
+
async function deleteSettled(storage, key, options, read = (op) => op(), waits = {}) {
|
|
6353
|
+
try {
|
|
6354
|
+
await storage.delete(key, options);
|
|
6355
|
+
return;
|
|
6356
|
+
} catch (err) {
|
|
6357
|
+
if (!isTransientError(err)) throw err;
|
|
6358
|
+
const tail = await readBack(
|
|
6359
|
+
() => read(() => storage.getTail(key, 0)).then(
|
|
6360
|
+
(t) => t,
|
|
6361
|
+
(readErr) => {
|
|
6362
|
+
if (isNotFoundError(readErr)) return null;
|
|
6363
|
+
throw readErr;
|
|
6364
|
+
}
|
|
6365
|
+
),
|
|
6366
|
+
err,
|
|
6367
|
+
waits
|
|
6368
|
+
);
|
|
6369
|
+
if (tail === null) return;
|
|
6370
|
+
const version = tail.version;
|
|
6371
|
+
if (options?.ifVersion !== void 0 && version !== void 0 && version !== options.ifVersion) {
|
|
6372
|
+
throw new WriteConflictError(
|
|
6373
|
+
`generation s.${key.generation} is another object than the version given; not deleted`
|
|
6374
|
+
);
|
|
6375
|
+
}
|
|
6376
|
+
throw err;
|
|
6377
|
+
}
|
|
6378
|
+
}
|
|
6379
|
+
async function rowDeleteSettled(registry, ref, expected, waits = {}) {
|
|
6380
|
+
try {
|
|
6381
|
+
await registry.delete(ref, expected);
|
|
6382
|
+
return;
|
|
6383
|
+
} catch (err) {
|
|
6384
|
+
if (!isTransientError(err) && !isWriteConflictError(err)) throw err;
|
|
6385
|
+
const row = await readBack(() => registry.get(ref), err, waits);
|
|
6386
|
+
if (row === null) return;
|
|
6387
|
+
throw err;
|
|
6388
|
+
}
|
|
6389
|
+
}
|
|
6390
|
+
|
|
7042
6391
|
// src/core/generation-gc.ts
|
|
6392
|
+
function deleteGeneration(deps, ref, generation) {
|
|
6393
|
+
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
6394
|
+
return deleteSettled(deps.storage, key, void 0, void 0, {
|
|
6395
|
+
clock: deps.clock,
|
|
6396
|
+
rng: deps.rng
|
|
6397
|
+
});
|
|
6398
|
+
}
|
|
7043
6399
|
var DELETE_SETTLE_WAITS_MS = Object.freeze([250, 500, 1e3]);
|
|
7044
6400
|
function watchDeletes(deps, ref) {
|
|
7045
6401
|
const gone = /* @__PURE__ */ new Set();
|
|
@@ -7127,19 +6483,44 @@ async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
|
7127
6483
|
async function collectBelowPointer(ref, deps, options = {}) {
|
|
7128
6484
|
const { deleted, missing, knownSpared } = await listingPass(ref, deps, {
|
|
7129
6485
|
keep: options.keep,
|
|
7130
|
-
skipKnown: options.skipKnown
|
|
6486
|
+
skipKnown: options.skipKnown,
|
|
6487
|
+
incarnation: options.incarnation
|
|
7131
6488
|
});
|
|
7132
6489
|
return missing === void 0 ? { deleted, knownSpared } : { deleted, missing, knownSpared };
|
|
7133
6490
|
}
|
|
6491
|
+
function isAskedRow(asked, row) {
|
|
6492
|
+
return asked === void 0 || sameIncarnation(asked, row) && asked.status === "destroyed" === (row.status === "destroyed");
|
|
6493
|
+
}
|
|
6494
|
+
async function readHeldRow(deps, ref, asked) {
|
|
6495
|
+
const row = await deps.registry.get(ref);
|
|
6496
|
+
if (row === null || isAskedRow(asked, row)) return row;
|
|
6497
|
+
throw refusedOtherRow(ref);
|
|
6498
|
+
}
|
|
6499
|
+
function quietOf(options) {
|
|
6500
|
+
if (options.quiet === true && options.incarnation !== void 0) {
|
|
6501
|
+
throw new ValidationError("a quiet collection pass cannot be held to a row incarnation");
|
|
6502
|
+
}
|
|
6503
|
+
return options.quiet === true;
|
|
6504
|
+
}
|
|
6505
|
+
function changedWhileCollecting(ref) {
|
|
6506
|
+
return new WriteConflictError(
|
|
6507
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
6508
|
+
);
|
|
6509
|
+
}
|
|
6510
|
+
function refusedOtherRow(ref) {
|
|
6511
|
+
return new WriteConflictError(
|
|
6512
|
+
`segment ${ref.segment} is not the row this collection works from: the row was purged and created again, or is no longer of the same kind (a tombstone or an active row), since it was read`
|
|
6513
|
+
);
|
|
6514
|
+
}
|
|
7134
6515
|
async function listingPass(ref, deps, options) {
|
|
7135
6516
|
const keep = options.keep ?? 1;
|
|
7136
|
-
const quiet = options
|
|
6517
|
+
const quiet = quietOf(options);
|
|
7137
6518
|
requireInteger(keep, { min: 0, what: "keep" });
|
|
7138
|
-
const record = await deps.
|
|
6519
|
+
const record = await readHeldRow(deps, ref, options.incarnation);
|
|
7139
6520
|
if (record === null) return { deleted: [] };
|
|
7140
6521
|
const current = record.currentGen;
|
|
7141
6522
|
const gens = await generationsOf(deps.storage, ref);
|
|
7142
|
-
const after = await deps.
|
|
6523
|
+
const after = await readHeldRow(deps, ref, options.incarnation);
|
|
7143
6524
|
if (after === null) {
|
|
7144
6525
|
if (quiet) return { deleted: [] };
|
|
7145
6526
|
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
@@ -7167,11 +6548,10 @@ async function listingPass(ref, deps, options) {
|
|
|
7167
6548
|
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
7168
6549
|
const now = protect === void 0 ? void 0 : windowOf(still);
|
|
7169
6550
|
const unnamed = protect === void 0 || record.status === "destroyed" || now !== void 0 && !now.includes(generation);
|
|
7170
|
-
|
|
6551
|
+
const asked = isAskedRow(options.incarnation, still);
|
|
6552
|
+
if (!ok || !unnamed || !asked) {
|
|
7171
6553
|
if (quiet) return "stop";
|
|
7172
|
-
throw
|
|
7173
|
-
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
7174
|
-
);
|
|
6554
|
+
throw asked ? changedWhileCollecting(ref) : refusedOtherRow(ref);
|
|
7175
6555
|
}
|
|
7176
6556
|
if (guard !== void 0 && heldGenerations(still, guard.now()).has(generation)) return "skip";
|
|
7177
6557
|
if (options.skipKnown === true && record.status !== "destroyed" && usableKnownGens(still)?.some((e) => e.generation === generation) === true) {
|
|
@@ -7209,7 +6589,7 @@ async function listingPass(ref, deps, options) {
|
|
|
7209
6589
|
leasedSpared.add(generation);
|
|
7210
6590
|
continue;
|
|
7211
6591
|
}
|
|
7212
|
-
await deps
|
|
6592
|
+
await deleteGeneration(deps, ref, generation);
|
|
7213
6593
|
done.push(generation);
|
|
7214
6594
|
watch.deleted(generation);
|
|
7215
6595
|
}
|
|
@@ -7292,7 +6672,7 @@ async function deleteEvicted(ref, deps, options) {
|
|
|
7292
6672
|
continue;
|
|
7293
6673
|
}
|
|
7294
6674
|
held ??= still;
|
|
7295
|
-
await deps
|
|
6675
|
+
await deleteGeneration(deps, ref, generation);
|
|
7296
6676
|
deleted.push(generation);
|
|
7297
6677
|
watch.deleted(generation);
|
|
7298
6678
|
}
|
|
@@ -7351,17 +6731,18 @@ function guardRefusal(cardinality, before, guard, allowEmpty) {
|
|
|
7351
6731
|
return void 0;
|
|
7352
6732
|
}
|
|
7353
6733
|
async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
7354
|
-
validateUserRef(ref);
|
|
6734
|
+
validateUserRef(ref, "judgeLoad");
|
|
7355
6735
|
requireInteger(cardinality, { min: 0, what: "judgeLoad: cardinality" });
|
|
7356
6736
|
const guard = checkedGuard(options.guard, "");
|
|
7357
6737
|
assertRegistryCanWrite(deps.registry, "load");
|
|
6738
|
+
checkRequireEncryption(deps.requireEncryption);
|
|
7358
6739
|
if (deps.requireEncryption === true && deps.keystore === void 0) {
|
|
7359
6740
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
7360
6741
|
}
|
|
7361
6742
|
const row = await deps.registry.get(ref);
|
|
7362
6743
|
if (row?.status === "destroyed") {
|
|
7363
6744
|
throw new ValidationError(
|
|
7364
|
-
`segment "${ref.segment}" is destroyed
|
|
6745
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to write: no generation resolves through a destroyed row; use a new segment`
|
|
7365
6746
|
);
|
|
7366
6747
|
}
|
|
7367
6748
|
const cleartext = row?.wrappedDeks === void 0 || row.wrappedDeks.length === 0;
|
|
@@ -7396,6 +6777,9 @@ async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
|
7396
6777
|
function sizeNeeded(guard, allowEmpty) {
|
|
7397
6778
|
return guard?.minRetained !== void 0 || guard?.maxGrowth !== void 0 || allowEmpty !== true;
|
|
7398
6779
|
}
|
|
6780
|
+
function lostNumberRace(err) {
|
|
6781
|
+
return isWriteConflictError(err) && !isSourceError(err);
|
|
6782
|
+
}
|
|
7399
6783
|
function settled(promise) {
|
|
7400
6784
|
promise.catch(() => void 0);
|
|
7401
6785
|
return promise;
|
|
@@ -7433,7 +6817,12 @@ async function currentCardinality(ref, deps, record, unwrapped) {
|
|
|
7433
6817
|
return { cardinality: reader.count(), fromSummary: false };
|
|
7434
6818
|
}
|
|
7435
6819
|
function loadSegment(ref, input, deps, options = {}) {
|
|
7436
|
-
return runLoad(
|
|
6820
|
+
return runLoad(
|
|
6821
|
+
ref,
|
|
6822
|
+
(codec) => sortedLoadInput(prepareLoadInput(input, codec), options.sorted),
|
|
6823
|
+
deps,
|
|
6824
|
+
options
|
|
6825
|
+
);
|
|
7437
6826
|
}
|
|
7438
6827
|
function startUnwrap(row, keystore) {
|
|
7439
6828
|
const wrappedKeys = row?.wrappedDeks;
|
|
@@ -7448,6 +6837,11 @@ function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
|
7448
6837
|
"loadSegmentChunks takes an async iterable of { chunkKey, bitmap }"
|
|
7449
6838
|
);
|
|
7450
6839
|
}
|
|
6840
|
+
if (wantsSorted(options.sorted)) {
|
|
6841
|
+
throw new ValidationError(
|
|
6842
|
+
"loadSegmentChunks takes no sorted: true: its chunks ascend by key already, and are written as they are"
|
|
6843
|
+
);
|
|
6844
|
+
}
|
|
7451
6845
|
return new ChunkLoadInput(chunks);
|
|
7452
6846
|
},
|
|
7453
6847
|
deps,
|
|
@@ -7455,11 +6849,11 @@ function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
|
7455
6849
|
);
|
|
7456
6850
|
}
|
|
7457
6851
|
async function runLoad(ref, prepare, deps, options) {
|
|
7458
|
-
validateUserRef(ref);
|
|
6852
|
+
validateUserRef(ref, "load");
|
|
7459
6853
|
checkedAuditSink(options.audit, "load");
|
|
7460
6854
|
const codec = requireCodec(deps.codec, "loadSegment");
|
|
7461
6855
|
const entropy = requireEntropy(deps.entropy, "a load");
|
|
7462
|
-
const keep = options.keep
|
|
6856
|
+
const keep = options.keep === void 0 ? 1 : options.keep;
|
|
7463
6857
|
requireInteger(keep, { min: 0, what: "keep" });
|
|
7464
6858
|
const guard = checkedGuard(options.guard, "");
|
|
7465
6859
|
const metadata = copiedMetadata(options.metadata, (message) => {
|
|
@@ -7496,7 +6890,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7496
6890
|
}
|
|
7497
6891
|
);
|
|
7498
6892
|
} catch (err) {
|
|
7499
|
-
if (!
|
|
6893
|
+
if (!lostNumberRace(err)) throw err;
|
|
7500
6894
|
const { generation: generation2 } = await numbering;
|
|
7501
6895
|
audit.onEvent({
|
|
7502
6896
|
kind: "segment.load-refused",
|
|
@@ -8405,9 +7799,13 @@ function compileCombineMany(request) {
|
|
|
8405
7799
|
max: MAX_BATCH_PARALLELISM,
|
|
8406
7800
|
what: "publishConcurrency"
|
|
8407
7801
|
});
|
|
8408
|
-
requireInteger(request.concurrency, {
|
|
8409
|
-
|
|
8410
|
-
|
|
7802
|
+
requireInteger(request.concurrency, {
|
|
7803
|
+
min: 1,
|
|
7804
|
+
max: MAX_BATCH_PARALLELISM,
|
|
7805
|
+
what: "materializeMany: concurrency"
|
|
7806
|
+
});
|
|
7807
|
+
assertIdBound("after", request.after, "materializeMany");
|
|
7808
|
+
assertIdBound("through", request.through, "materializeMany");
|
|
8411
7809
|
const index = /* @__PURE__ */ new Map();
|
|
8412
7810
|
operands.forEach((op, i) => index.set(op.name, i));
|
|
8413
7811
|
const feed = request.feed === void 0 ? void 0 : checkFeed(request.feed, new Set(index.keys()));
|
|
@@ -8494,7 +7892,7 @@ var Gate = class {
|
|
|
8494
7892
|
return this.peak;
|
|
8495
7893
|
}
|
|
8496
7894
|
async run(task) {
|
|
8497
|
-
if (this.active >= this.limit) await new Promise((
|
|
7895
|
+
if (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
|
|
8498
7896
|
else this.active++;
|
|
8499
7897
|
if (this.active > this.peak) this.peak = this.active;
|
|
8500
7898
|
try {
|
|
@@ -8679,7 +8077,7 @@ var Run = class {
|
|
|
8679
8077
|
running = [];
|
|
8680
8078
|
bounds;
|
|
8681
8079
|
async execute() {
|
|
8682
|
-
this.window = windowOf2(this.req);
|
|
8080
|
+
this.window = windowOf2(this.req, "materializeMany");
|
|
8683
8081
|
if (this.checkedFeed !== void 0) this.feed = this.openFeed(this.checkedFeed);
|
|
8684
8082
|
await this.readIndexes();
|
|
8685
8083
|
await this.refuseAbsentOperands();
|
|
@@ -8867,9 +8265,9 @@ var Run = class {
|
|
|
8867
8265
|
}
|
|
8868
8266
|
const bounds = this.boundsList();
|
|
8869
8267
|
for (const o of this.outputs) {
|
|
8870
|
-
const
|
|
8871
|
-
if (
|
|
8872
|
-
this.fail(o,
|
|
8268
|
+
const failed2 = o.compiled.operands.map((i) => this.operands[i]).find((op) => op.error);
|
|
8269
|
+
if (failed2?.error !== void 0) {
|
|
8270
|
+
this.fail(o, failed2.error.error);
|
|
8873
8271
|
continue;
|
|
8874
8272
|
}
|
|
8875
8273
|
if (this.window === "empty") {
|
|
@@ -9403,7 +8801,7 @@ var Run = class {
|
|
|
9403
8801
|
const wake = () => {
|
|
9404
8802
|
const now = waiters;
|
|
9405
8803
|
waiters = [];
|
|
9406
|
-
for (const
|
|
8804
|
+
for (const resolve of now) resolve();
|
|
9407
8805
|
};
|
|
9408
8806
|
const worker = async () => {
|
|
9409
8807
|
for (; ; ) {
|
|
@@ -9422,7 +8820,7 @@ var Run = class {
|
|
|
9422
8820
|
const transient = o.serialized;
|
|
9423
8821
|
let admitted = this.ledger.tryCharge(transient);
|
|
9424
8822
|
while (!admitted && inFlight > 0) {
|
|
9425
|
-
await new Promise((
|
|
8823
|
+
await new Promise((resolve) => waiters.push(resolve));
|
|
9426
8824
|
admitted = this.ledger.tryCharge(transient);
|
|
9427
8825
|
}
|
|
9428
8826
|
if (!admitted) {
|
|
@@ -9692,9 +9090,9 @@ function evaluate(node, kc) {
|
|
|
9692
9090
|
|
|
9693
9091
|
// src/core/rollback.ts
|
|
9694
9092
|
async function listGenerations(ref, deps) {
|
|
9695
|
-
validateUserRef(ref);
|
|
9093
|
+
validateUserRef(ref, "generations");
|
|
9696
9094
|
const record = await deps.registry.get(ref);
|
|
9697
|
-
const current = record?.currentGen ?? null;
|
|
9095
|
+
const current = record?.status === "destroyed" ? null : record?.currentGen ?? null;
|
|
9698
9096
|
const seen = await generationsOf(deps.storage, ref);
|
|
9699
9097
|
const described = record === null ? void 0 : await currentDescription(ref, record, deps);
|
|
9700
9098
|
const known = new Set((usableKnownGens(record) ?? []).map((e) => e.generation));
|
|
@@ -9718,7 +9116,7 @@ async function currentDescription(ref, record, deps) {
|
|
|
9718
9116
|
return metadata === void 0 || Object.keys(metadata).length === 0 ? { cardinality: usable.cardinality } : { cardinality: usable.cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
9719
9117
|
}
|
|
9720
9118
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
9721
|
-
validateUserRef(ref);
|
|
9119
|
+
validateUserRef(ref, "rollback");
|
|
9722
9120
|
checkedAuditSink(options.audit, "rollback");
|
|
9723
9121
|
requireInteger(toGeneration, { min: 0, what: "rollback: generation" });
|
|
9724
9122
|
const record = await deps.registry.get(ref);
|
|
@@ -9727,7 +9125,7 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9727
9125
|
}
|
|
9728
9126
|
if (record.status === "destroyed") {
|
|
9729
9127
|
throw new ValidationError(
|
|
9730
|
-
`rollback: segment "${ref.segment}" is destroyed
|
|
9128
|
+
`rollback: segment "${ref.segment}" is destroyed \u2014 no generation resolves through a destroyed row`
|
|
9731
9129
|
);
|
|
9732
9130
|
}
|
|
9733
9131
|
const known = usableKnownGens(record);
|
|
@@ -9800,43 +9198,13 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9800
9198
|
footerCrc: footer.footerCrc
|
|
9801
9199
|
});
|
|
9802
9200
|
const churn = leaseChurn(deps);
|
|
9803
|
-
|
|
9804
|
-
|
|
9805
|
-
|
|
9806
|
-
|
|
9807
|
-
|
|
9808
|
-
|
|
9809
|
-
|
|
9810
|
-
...knownAfter === void 0 ? {} : { knownGens: knownAfter }
|
|
9811
|
-
});
|
|
9812
|
-
swapped = { pointerId: token, token };
|
|
9813
|
-
break;
|
|
9814
|
-
} catch (err) {
|
|
9815
|
-
if (isTransientError(err)) {
|
|
9816
|
-
const landed = await swapLanded(
|
|
9817
|
-
ref,
|
|
9818
|
-
record,
|
|
9819
|
-
{ toGeneration, summary, known: knownAfter },
|
|
9820
|
-
deps.registry,
|
|
9821
|
-
err
|
|
9822
|
-
);
|
|
9823
|
-
swapped = { pointerId: landed.pointerId, token: landed.token };
|
|
9824
|
-
break;
|
|
9825
|
-
}
|
|
9826
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9827
|
-
const seen = await deps.registry.get(ref);
|
|
9828
|
-
const settled2 = await churn.settle(record, seen, () => deps.registry.get(ref));
|
|
9829
|
-
const now = settled2?.row ?? null;
|
|
9830
|
-
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now)) {
|
|
9831
|
-
const lost = new WriteConflictError(
|
|
9832
|
-
"rollback: the segment's row changed while the rollback ran and the pointer was not moved; check store.generations(ref) and run it again"
|
|
9833
|
-
);
|
|
9834
|
-
lost.cause = err;
|
|
9835
|
-
throw lost;
|
|
9836
|
-
}
|
|
9837
|
-
against = now.token;
|
|
9838
|
-
}
|
|
9839
|
-
}
|
|
9201
|
+
const swapped = await swapPointer(
|
|
9202
|
+
ref,
|
|
9203
|
+
record,
|
|
9204
|
+
{ toGeneration, summary, known: knownAfter },
|
|
9205
|
+
deps,
|
|
9206
|
+
churn
|
|
9207
|
+
);
|
|
9840
9208
|
const emitMove = () => {
|
|
9841
9209
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
9842
9210
|
kind: "segment.rollback",
|
|
@@ -9862,43 +9230,9 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9862
9230
|
}
|
|
9863
9231
|
}
|
|
9864
9232
|
if (!stillThere || replaced) {
|
|
9865
|
-
|
|
9866
|
-
let undoAgainst = swapped.token;
|
|
9867
|
-
for (; ; ) {
|
|
9868
|
-
try {
|
|
9869
|
-
const restored = describingKnown(record, toGeneration)?.filter(
|
|
9870
|
-
(e) => e.generation !== toGeneration || e.generation === record.currentGen
|
|
9871
|
-
);
|
|
9872
|
-
await deps.registry.compareAndSwap(ref, undoAgainst, {
|
|
9873
|
-
currentGen: record.currentGen,
|
|
9874
|
-
summary: describingSummary(record),
|
|
9875
|
-
...restored === void 0 ? {} : { knownGens: restored }
|
|
9876
|
-
});
|
|
9877
|
-
undone = true;
|
|
9878
|
-
break;
|
|
9879
|
-
} catch (err) {
|
|
9880
|
-
if (!isWriteConflictError(err)) break;
|
|
9881
|
-
let now;
|
|
9882
|
-
try {
|
|
9883
|
-
const seen = await deps.registry.get(ref);
|
|
9884
|
-
if (seen === null || seen.currentGen !== toGeneration) break;
|
|
9885
|
-
const settled2 = await churn.settle(
|
|
9886
|
-
record,
|
|
9887
|
-
seen,
|
|
9888
|
-
() => deps.registry.get(ref),
|
|
9889
|
-
SWAP_MOVES
|
|
9890
|
-
);
|
|
9891
|
-
now = settled2?.row ?? null;
|
|
9892
|
-
} catch {
|
|
9893
|
-
break;
|
|
9894
|
-
}
|
|
9895
|
-
if (now === null || now.currentGen !== toGeneration || now.pointerId !== swapped.pointerId || !onlyLeasesDiffer(record, now, SWAP_MOVES))
|
|
9896
|
-
break;
|
|
9897
|
-
undoAgainst = now.token;
|
|
9898
|
-
}
|
|
9899
|
-
}
|
|
9233
|
+
const undone = await moveBack(ref, record, toGeneration, swapped, deps.registry, churn, deps);
|
|
9900
9234
|
throw new NotFoundError(
|
|
9901
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (
|
|
9235
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (the row could not show whether the move back landed). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
9902
9236
|
);
|
|
9903
9237
|
}
|
|
9904
9238
|
emitMove();
|
|
@@ -9951,13 +9285,113 @@ function keyNotHeld(err) {
|
|
|
9951
9285
|
function keyServiceUnanswered(err) {
|
|
9952
9286
|
return isTransientError(err) || err instanceof Error && err.name === "TimeoutError";
|
|
9953
9287
|
}
|
|
9954
|
-
async function
|
|
9955
|
-
let
|
|
9956
|
-
|
|
9957
|
-
|
|
9958
|
-
|
|
9959
|
-
if (
|
|
9960
|
-
|
|
9288
|
+
async function moveBack(ref, record, toGeneration, swapped, registry, churn, waits) {
|
|
9289
|
+
let undone = false;
|
|
9290
|
+
let undoAgainst = swapped.token;
|
|
9291
|
+
const undo = writeTokens(registry);
|
|
9292
|
+
const undoLanded = (row) => {
|
|
9293
|
+
if (row === null || undo.sent === void 0) return false;
|
|
9294
|
+
if (record.currentGen !== null) return pointerSetBy(row, record, record.currentGen, undo.sent);
|
|
9295
|
+
return row.currentGen === null && row.status !== "destroyed" && sameIncarnation(record, row) && undo.sent.has(row.pointerId);
|
|
9296
|
+
};
|
|
9297
|
+
for (; ; ) {
|
|
9298
|
+
try {
|
|
9299
|
+
const restored = describingKnown(record, toGeneration)?.filter(
|
|
9300
|
+
(e) => e.generation !== toGeneration || e.generation === record.currentGen
|
|
9301
|
+
);
|
|
9302
|
+
await registry.compareAndSwap(
|
|
9303
|
+
ref,
|
|
9304
|
+
undoAgainst,
|
|
9305
|
+
{
|
|
9306
|
+
currentGen: record.currentGen,
|
|
9307
|
+
summary: describingSummary(record),
|
|
9308
|
+
...restored === void 0 ? {} : { knownGens: restored }
|
|
9309
|
+
},
|
|
9310
|
+
undo.options
|
|
9311
|
+
);
|
|
9312
|
+
undone = true;
|
|
9313
|
+
break;
|
|
9314
|
+
} catch (err) {
|
|
9315
|
+
if (isTransientError(err)) {
|
|
9316
|
+
try {
|
|
9317
|
+
const row = await readBack(() => registry.get(ref), err, waits);
|
|
9318
|
+
if (undoLanded(row)) undone = true;
|
|
9319
|
+
} catch {
|
|
9320
|
+
}
|
|
9321
|
+
break;
|
|
9322
|
+
}
|
|
9323
|
+
if (!isWriteConflictError(err)) break;
|
|
9324
|
+
let now;
|
|
9325
|
+
try {
|
|
9326
|
+
const seen = await readBack(() => registry.get(ref), err, waits);
|
|
9327
|
+
if (undoLanded(seen)) {
|
|
9328
|
+
undone = true;
|
|
9329
|
+
break;
|
|
9330
|
+
}
|
|
9331
|
+
if (seen === null || seen.currentGen !== toGeneration) break;
|
|
9332
|
+
const settled2 = await churn.settle(record, seen, () => registry.get(ref), SWAP_MOVES);
|
|
9333
|
+
now = settled2?.row ?? null;
|
|
9334
|
+
} catch {
|
|
9335
|
+
break;
|
|
9336
|
+
}
|
|
9337
|
+
if (now === null || now.currentGen !== toGeneration || now.pointerId !== swapped.pointerId || !onlyLeasesDiffer(record, now, SWAP_MOVES))
|
|
9338
|
+
break;
|
|
9339
|
+
undoAgainst = now.token;
|
|
9340
|
+
}
|
|
9341
|
+
}
|
|
9342
|
+
return undone;
|
|
9343
|
+
}
|
|
9344
|
+
function unconfirmedSwap(err, cause = err) {
|
|
9345
|
+
const lost = new WriteConflictError(
|
|
9346
|
+
"rollback: the segment's row changed while the rollback ran, and this rollback could not confirm that its swap is the one the row shows; check store.generations(ref) and run it again"
|
|
9347
|
+
);
|
|
9348
|
+
lost.cause = cause;
|
|
9349
|
+
return lost;
|
|
9350
|
+
}
|
|
9351
|
+
async function swapPointer(ref, record, swap, deps, churn) {
|
|
9352
|
+
const { registry } = deps;
|
|
9353
|
+
let against = record.token;
|
|
9354
|
+
const { sent, options: writeOptions2 } = writeTokens(registry);
|
|
9355
|
+
for (; ; ) {
|
|
9356
|
+
try {
|
|
9357
|
+
const { token } = await registry.compareAndSwap(
|
|
9358
|
+
ref,
|
|
9359
|
+
against,
|
|
9360
|
+
{
|
|
9361
|
+
currentGen: swap.toGeneration,
|
|
9362
|
+
summary: swap.summary,
|
|
9363
|
+
...swap.known === void 0 ? {} : { knownGens: swap.known }
|
|
9364
|
+
},
|
|
9365
|
+
writeOptions2
|
|
9366
|
+
);
|
|
9367
|
+
return { pointerId: token, token };
|
|
9368
|
+
} catch (err) {
|
|
9369
|
+
if (isTransientError(err)) return swapLanded(ref, record, swap, registry, err, sent, deps);
|
|
9370
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9371
|
+
let seen;
|
|
9372
|
+
try {
|
|
9373
|
+
seen = await readBack(() => registry.get(ref), err, deps);
|
|
9374
|
+
} catch (readErr) {
|
|
9375
|
+
if (readErr === err) throw unconfirmedSwap(err, err.cause);
|
|
9376
|
+
throw readErr;
|
|
9377
|
+
}
|
|
9378
|
+
if (seen !== null && sent !== void 0 && pointerSetBy(seen, record, swap.toGeneration, sent)) {
|
|
9379
|
+
return seen;
|
|
9380
|
+
}
|
|
9381
|
+
const settled2 = await churn.settle(record, seen, () => registry.get(ref));
|
|
9382
|
+
const now = settled2?.row ?? null;
|
|
9383
|
+
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now)) {
|
|
9384
|
+
throw unconfirmedSwap(err);
|
|
9385
|
+
}
|
|
9386
|
+
against = now.token;
|
|
9387
|
+
}
|
|
9388
|
+
}
|
|
9389
|
+
}
|
|
9390
|
+
async function swapLanded(ref, record, swap, registry, unanswered, sent, waits) {
|
|
9391
|
+
const now = await readBack(() => registry.get(ref), unanswered, waits);
|
|
9392
|
+
if (sent !== void 0) {
|
|
9393
|
+
if (now === null || !pointerSetBy(now, record, swap.toGeneration, sent)) throw unanswered;
|
|
9394
|
+
return now;
|
|
9961
9395
|
}
|
|
9962
9396
|
const left = {
|
|
9963
9397
|
...record,
|
|
@@ -10006,7 +9440,7 @@ function recordCollect(ref, result, pass, options) {
|
|
|
10006
9440
|
return result;
|
|
10007
9441
|
}
|
|
10008
9442
|
async function eraseOnce(ref, id, deps, options, seen) {
|
|
10009
|
-
validateUserRef(ref);
|
|
9443
|
+
validateUserRef(ref, "eraseIdFromSegment");
|
|
10010
9444
|
checkedAuditSink(options.audit, "eraseIdFromSegment");
|
|
10011
9445
|
const { chunkKey, remainder } = splitId(id);
|
|
10012
9446
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
@@ -10044,6 +9478,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
10044
9478
|
read,
|
|
10045
9479
|
sealed: /* @__PURE__ */ new Set(),
|
|
10046
9480
|
versions: /* @__PURE__ */ new Map(),
|
|
9481
|
+
fromFingerprint: void 0,
|
|
10047
9482
|
premise: record,
|
|
10048
9483
|
premiseToken: fromToken,
|
|
10049
9484
|
fenceLost: 0,
|
|
@@ -10120,6 +9555,7 @@ async function holds(c, generation) {
|
|
|
10120
9555
|
reader = await read(() => openGenerationReader(deps.storage, key, void 0));
|
|
10121
9556
|
}
|
|
10122
9557
|
versions.set(generation, objectVersionOf(reader));
|
|
9558
|
+
sealed.delete(generation);
|
|
10123
9559
|
let bytes;
|
|
10124
9560
|
try {
|
|
10125
9561
|
bytes = await read(() => reader.getChunk(chunkKey));
|
|
@@ -10163,11 +9599,32 @@ async function holdsEach(c, generations, { stopAtHolder }) {
|
|
|
10163
9599
|
}
|
|
10164
9600
|
async function holderLeft(c, clean) {
|
|
10165
9601
|
const { ref, deps } = c;
|
|
10166
|
-
const
|
|
9602
|
+
const listed2 = (await generationsOf(deps.storage, ref)).reverse();
|
|
9603
|
+
const own = (generation) => clean.has(generation) ? ownFingerprint(c, generation) : void 0;
|
|
9604
|
+
const known = listed2.some((generation) => own(generation) !== void 0) ? await ledgerAfterListing(c) : void 0;
|
|
9605
|
+
const vouched = (generation) => {
|
|
9606
|
+
const fingerprint = own(generation);
|
|
9607
|
+
if (fingerprint === void 0 || known === void 0) return false;
|
|
9608
|
+
const footerCrc = footerCrcOf(fingerprint);
|
|
9609
|
+
return known.some((e) => e.generation === generation && e.footerCrc === footerCrc);
|
|
9610
|
+
};
|
|
9611
|
+
const toRead = listed2.filter((generation) => !vouched(generation));
|
|
10167
9612
|
return (await holdsEach(c, toRead, { stopAtHolder: true })).find(
|
|
10168
9613
|
(outcome) => outcome.held === true
|
|
10169
9614
|
)?.generation;
|
|
10170
9615
|
}
|
|
9616
|
+
function ownFingerprint(c, generation) {
|
|
9617
|
+
if (generation === c.from) return c.fromFingerprint;
|
|
9618
|
+
if (generation === c.rewrote?.generation) return c.rewrote.fingerprint;
|
|
9619
|
+
return void 0;
|
|
9620
|
+
}
|
|
9621
|
+
async function ledgerAfterListing(c) {
|
|
9622
|
+
try {
|
|
9623
|
+
return usableKnownGens(await c.deps.registry.get(c.ref));
|
|
9624
|
+
} catch {
|
|
9625
|
+
return void 0;
|
|
9626
|
+
}
|
|
9627
|
+
}
|
|
10171
9628
|
function cannotRemove(c, generation) {
|
|
10172
9629
|
const { ref, sealed } = c;
|
|
10173
9630
|
return new WriteConflictError(
|
|
@@ -10301,7 +9758,13 @@ async function deleteFenced(c, holders, fenced) {
|
|
|
10301
9758
|
}
|
|
10302
9759
|
if (moved !== null) break;
|
|
10303
9760
|
try {
|
|
10304
|
-
await
|
|
9761
|
+
await deleteSettled(
|
|
9762
|
+
deps.storage,
|
|
9763
|
+
{ ...base, generation },
|
|
9764
|
+
{ ifVersion: versions.get(generation) },
|
|
9765
|
+
c.read,
|
|
9766
|
+
{ clock: deps.clock, rng: deps.rng }
|
|
9767
|
+
);
|
|
10305
9768
|
} catch (err) {
|
|
10306
9769
|
if (!isWriteConflictError(err)) throw err;
|
|
10307
9770
|
moved = rowVerdict(c, await deps.registry.get(ref)) ?? "superseded";
|
|
@@ -10365,14 +9828,35 @@ async function unpublished(c) {
|
|
|
10365
9828
|
}
|
|
10366
9829
|
throw cannotRemove(c, left);
|
|
10367
9830
|
}
|
|
9831
|
+
async function holderUnderTombstone(c) {
|
|
9832
|
+
const newestFirst = (await c.read(() => generationsOf(c.deps.storage, c.ref))).reverse();
|
|
9833
|
+
return (await holdsEach(c, newestFirst, { stopAtHolder: true })).find((h) => h.held === true);
|
|
9834
|
+
}
|
|
9835
|
+
async function findIdUnderTombstone(ref, id, deps) {
|
|
9836
|
+
validateUserRef(ref, "findIdUnderTombstone");
|
|
9837
|
+
const { chunkKey, remainder } = splitId(id);
|
|
9838
|
+
const codec = requireCodec(deps.codec, "findIdUnderTombstone");
|
|
9839
|
+
const c = {
|
|
9840
|
+
ref,
|
|
9841
|
+
deps,
|
|
9842
|
+
chunkKey,
|
|
9843
|
+
remainder,
|
|
9844
|
+
codec,
|
|
9845
|
+
maxBytes: deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES,
|
|
9846
|
+
base: { segment: ref.segment, namespace: ref.namespace },
|
|
9847
|
+
tombstoned: true,
|
|
9848
|
+
aead: void 0,
|
|
9849
|
+
read: (op) => retryRead(op, deps.readRetry),
|
|
9850
|
+
sealed: /* @__PURE__ */ new Set(),
|
|
9851
|
+
versions: /* @__PURE__ */ new Map()
|
|
9852
|
+
};
|
|
9853
|
+
return (await holderUnderTombstone(c))?.generation ?? null;
|
|
9854
|
+
}
|
|
10368
9855
|
async function underTombstone(c) {
|
|
10369
9856
|
const { ref, deps, base } = c;
|
|
10370
|
-
const
|
|
10371
|
-
const holder = (await holdsEach(c, newestFirst, { stopAtHolder: true })).find(
|
|
10372
|
-
(h) => h.held === true
|
|
10373
|
-
);
|
|
9857
|
+
const holder = await holderUnderTombstone(c);
|
|
10374
9858
|
if (holder === void 0) return { ...base, erased: false, reason: "destroyed", collected: [] };
|
|
10375
|
-
const collected = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
9859
|
+
const collected = await gcOrphanGenerations(ref, deps, { keep: 0, incarnation: c.record });
|
|
10376
9860
|
const left = await holderLeft(c, /* @__PURE__ */ new Set());
|
|
10377
9861
|
if (left !== void 0) throw cannotRemove(c, left);
|
|
10378
9862
|
return collectedAll(c, holder.generation, collected);
|
|
@@ -10488,6 +9972,7 @@ async function stage(c) {
|
|
|
10488
9972
|
const reader = await read(
|
|
10489
9973
|
() => openGenerationReader(deps.storage, fromKey, cryptoAt(c, from), {}, described?.fingerprint)
|
|
10490
9974
|
);
|
|
9975
|
+
c.fromFingerprint = reader.fingerprint;
|
|
10491
9976
|
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
10492
9977
|
if (bytes === null) return await notInCurrent(c);
|
|
10493
9978
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
@@ -10498,17 +9983,29 @@ async function stage(c) {
|
|
|
10498
9983
|
const generation = await nextGeneration(ref, deps);
|
|
10499
9984
|
const key = { ...base, generation };
|
|
10500
9985
|
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
10501
|
-
|
|
10502
|
-
|
|
10503
|
-
|
|
10504
|
-
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
9986
|
+
let tally;
|
|
9987
|
+
try {
|
|
9988
|
+
tally = await writeCrbmGenerationStream(
|
|
9989
|
+
deps.storage,
|
|
9990
|
+
key,
|
|
9991
|
+
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
9992
|
+
{
|
|
9993
|
+
crypto: cryptoAt(c, generation),
|
|
9994
|
+
clock: deps.clock,
|
|
9995
|
+
metadata,
|
|
9996
|
+
entropy: deps.entropy
|
|
9997
|
+
}
|
|
9998
|
+
);
|
|
9999
|
+
} catch (err) {
|
|
10000
|
+
if (!isWriteConflictError(err)) throw err;
|
|
10001
|
+
let now;
|
|
10002
|
+
try {
|
|
10003
|
+
now = await deps.registry.get(ref);
|
|
10004
|
+
} catch {
|
|
10005
|
+
throw err;
|
|
10510
10006
|
}
|
|
10511
|
-
|
|
10007
|
+
return refusedResult(c, rowVerdict(c, now) ?? "superseded");
|
|
10008
|
+
}
|
|
10512
10009
|
written = generation;
|
|
10513
10010
|
c.rewrote = { generation, fingerprint: tally.fingerprint };
|
|
10514
10011
|
const beforeVerify = await deps.registry.get(ref);
|
|
@@ -10550,7 +10047,10 @@ async function discardRefused(c, own) {
|
|
|
10550
10047
|
let row = await deps.registry.get(ref);
|
|
10551
10048
|
for (let attempts = 1; ; attempts++) {
|
|
10552
10049
|
if (row !== null && row.status === "destroyed") {
|
|
10553
|
-
await deps.storage.
|
|
10050
|
+
await deleteSettled(deps.storage, key, void 0, c.read, {
|
|
10051
|
+
clock: deps.clock,
|
|
10052
|
+
rng: deps.rng
|
|
10053
|
+
});
|
|
10554
10054
|
return row;
|
|
10555
10055
|
}
|
|
10556
10056
|
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
@@ -10560,7 +10060,10 @@ async function discardRefused(c, own) {
|
|
|
10560
10060
|
const renewed = await renewal.renew(row);
|
|
10561
10061
|
if ("row" in renewed) {
|
|
10562
10062
|
try {
|
|
10563
|
-
await deps.storage
|
|
10063
|
+
await deleteSettled(deps.storage, key, { ifVersion: proven.version }, c.read, {
|
|
10064
|
+
clock: deps.clock,
|
|
10065
|
+
rng: deps.rng
|
|
10066
|
+
});
|
|
10564
10067
|
} catch (err) {
|
|
10565
10068
|
if (!isWriteConflictError(err)) throw err;
|
|
10566
10069
|
}
|
|
@@ -10640,17 +10143,20 @@ async function finishRewrite(c, staged) {
|
|
|
10640
10143
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
10641
10144
|
}
|
|
10642
10145
|
async function resolveDek(ref, deps, record, row) {
|
|
10146
|
+
checkRequireEncryption(deps.requireEncryption);
|
|
10643
10147
|
if (row.tombstoned) return void 0;
|
|
10644
10148
|
if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
10645
10149
|
if (deps.keystore === void 0) {
|
|
10646
10150
|
throw new KeyUnavailableError(
|
|
10647
|
-
`segment "${ref.segment}" is encrypted
|
|
10151
|
+
`segment "${ref.segment}" is encrypted, and the erasure has no keystore to open its key (encryption.keystore on the store)`
|
|
10648
10152
|
);
|
|
10649
10153
|
}
|
|
10650
10154
|
return await deps.keystore.openDek(record.wrappedDeks);
|
|
10651
10155
|
}
|
|
10652
10156
|
if (!row.pointerless && deps.requireEncryption === true) {
|
|
10653
|
-
throw new
|
|
10157
|
+
throw new KeyUnavailableError(
|
|
10158
|
+
`requireEncryption: segment "${ref.segment}" is cleartext but encryption is required`
|
|
10159
|
+
);
|
|
10654
10160
|
}
|
|
10655
10161
|
return void 0;
|
|
10656
10162
|
}
|
|
@@ -10749,116 +10255,6 @@ function assertRemaindersInRange(bitmap, chunkKey) {
|
|
|
10749
10255
|
}
|
|
10750
10256
|
}
|
|
10751
10257
|
|
|
10752
|
-
// src/drivers/crypto.ts
|
|
10753
|
-
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
10754
|
-
var UTF82 = new TextEncoder();
|
|
10755
|
-
var KEY_BYTES = 32;
|
|
10756
|
-
var NONCE_BYTES = 12;
|
|
10757
|
-
var TAG_BYTES = 16;
|
|
10758
|
-
var isBytes = (v) => Object.prototype.toString.call(v) === "[object Uint8Array]";
|
|
10759
|
-
function ownKey(key) {
|
|
10760
|
-
const own = Buffer.alloc(key.length);
|
|
10761
|
-
own.set(key);
|
|
10762
|
-
return own;
|
|
10763
|
-
}
|
|
10764
|
-
var NodeAead = class {
|
|
10765
|
-
key;
|
|
10766
|
-
constructor(key) {
|
|
10767
|
-
if (!isBytes(key))
|
|
10768
|
-
throw new ValidationError("AEAD key must be a Uint8Array of key bytes, not text");
|
|
10769
|
-
if (key.length !== KEY_BYTES)
|
|
10770
|
-
throw new ValidationError(`AEAD key must be ${KEY_BYTES} bytes, got ${key.length}`);
|
|
10771
|
-
this.key = ownKey(key);
|
|
10772
|
-
}
|
|
10773
|
-
seal(plaintext, aad) {
|
|
10774
|
-
const nonce = randomBytes(NONCE_BYTES);
|
|
10775
|
-
const cipher = createCipheriv("aes-256-gcm", this.key, nonce);
|
|
10776
|
-
cipher.setAAD(aad);
|
|
10777
|
-
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
10778
|
-
return { nonce, ciphertext, tag: cipher.getAuthTag() };
|
|
10779
|
-
}
|
|
10780
|
-
open(sealed, aad) {
|
|
10781
|
-
if (sealed.nonce.length !== NONCE_BYTES || sealed.tag.length !== TAG_BYTES)
|
|
10782
|
-
throw new IntegrityError("AEAD nonce/tag have the wrong length");
|
|
10783
|
-
const decipher = createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
|
|
10784
|
-
decipher.setAAD(aad);
|
|
10785
|
-
decipher.setAuthTag(Buffer.from(sealed.tag));
|
|
10786
|
-
try {
|
|
10787
|
-
return Buffer.concat([decipher.update(sealed.ciphertext), decipher.final()]);
|
|
10788
|
-
} catch {
|
|
10789
|
-
throw new IntegrityError(
|
|
10790
|
-
"AEAD authentication failed (wrong key, tampered data, or wrong context)"
|
|
10791
|
-
);
|
|
10792
|
-
}
|
|
10793
|
-
}
|
|
10794
|
-
};
|
|
10795
|
-
function wrapKey(dek, kek, keyId) {
|
|
10796
|
-
const sealed = new NodeAead(kek).seal(dek, UTF82.encode(keyId));
|
|
10797
|
-
return Buffer.concat([sealed.nonce, sealed.ciphertext, sealed.tag]).toString("base64");
|
|
10798
|
-
}
|
|
10799
|
-
function unwrapKey(wrapped, kek, keyId) {
|
|
10800
|
-
const blob = Buffer.from(wrapped, "base64");
|
|
10801
|
-
if (blob.length < NONCE_BYTES + TAG_BYTES)
|
|
10802
|
-
throw new IntegrityError("wrapped DEK is too short to be valid");
|
|
10803
|
-
const nonce = blob.subarray(0, NONCE_BYTES);
|
|
10804
|
-
const tag = blob.subarray(blob.length - TAG_BYTES);
|
|
10805
|
-
const ciphertext = blob.subarray(NONCE_BYTES, blob.length - TAG_BYTES);
|
|
10806
|
-
return new NodeAead(kek).open({ nonce, ciphertext, tag }, UTF82.encode(keyId));
|
|
10807
|
-
}
|
|
10808
|
-
var InProcessKeystore = class {
|
|
10809
|
-
keys;
|
|
10810
|
-
activeKeyId;
|
|
10811
|
-
recoveryKeyId;
|
|
10812
|
-
constructor(options) {
|
|
10813
|
-
const entries = Object.entries(options.keys);
|
|
10814
|
-
if (entries.length === 0) throw new ValidationError("keystore needs at least one KEK");
|
|
10815
|
-
for (const [id, k] of entries) {
|
|
10816
|
-
if (!isBytes(k))
|
|
10817
|
-
throw new ValidationError(`KEK "${id}" must be a Uint8Array of key bytes, not text`);
|
|
10818
|
-
if (k.length !== KEY_BYTES)
|
|
10819
|
-
throw new ValidationError(`KEK "${id}" must be ${KEY_BYTES} bytes, got ${k.length}`);
|
|
10820
|
-
}
|
|
10821
|
-
this.keys = new Map(entries.map(([id, k]) => [id, ownKey(k)]));
|
|
10822
|
-
if (!this.keys.has(options.activeKeyId))
|
|
10823
|
-
throw new ValidationError(`activeKeyId "${String(options.activeKeyId)}" is not in keys`);
|
|
10824
|
-
if (options.recoveryKeyId !== void 0 && !this.keys.has(options.recoveryKeyId))
|
|
10825
|
-
throw new ValidationError(`recoveryKeyId "${String(options.recoveryKeyId)}" is not in keys`);
|
|
10826
|
-
this.activeKeyId = options.activeKeyId;
|
|
10827
|
-
this.recoveryKeyId = options.recoveryKeyId === options.activeKeyId ? void 0 : options.recoveryKeyId;
|
|
10828
|
-
}
|
|
10829
|
-
// async to satisfy the IKeystore seam (a KMS adapter awaits a network call here); the in-process path is
|
|
10830
|
-
// synchronous, but staying async keeps unwrap failures as rejections, matching a remote keystore.
|
|
10831
|
-
async createDek() {
|
|
10832
|
-
const dek = randomBytes(KEY_BYTES);
|
|
10833
|
-
const ids = this.recoveryKeyId ? [this.activeKeyId, this.recoveryKeyId] : [this.activeKeyId];
|
|
10834
|
-
const wrapped = ids.map((keyId) => ({
|
|
10835
|
-
keyId,
|
|
10836
|
-
wrapped: wrapKey(dek, this.keys.get(keyId), keyId)
|
|
10837
|
-
}));
|
|
10838
|
-
return { wrapped, aead: new NodeAead(dek) };
|
|
10839
|
-
}
|
|
10840
|
-
// async for the same reason as createDek — a synchronous unwrap failure surfaces as a promise rejection.
|
|
10841
|
-
async openDek(wrapped) {
|
|
10842
|
-
let anyKekHeld = false;
|
|
10843
|
-
let lastUnwrapError;
|
|
10844
|
-
for (const w of wrapped) {
|
|
10845
|
-
const kek = this.keys.get(w.keyId);
|
|
10846
|
-
if (kek === void 0) continue;
|
|
10847
|
-
anyKekHeld = true;
|
|
10848
|
-
try {
|
|
10849
|
-
return new NodeAead(unwrapKey(w.wrapped, kek, w.keyId));
|
|
10850
|
-
} catch (err) {
|
|
10851
|
-
lastUnwrapError = err;
|
|
10852
|
-
}
|
|
10853
|
-
}
|
|
10854
|
-
if (anyKekHeld) throw lastUnwrapError;
|
|
10855
|
-
const need = wrapped.map((w) => w.keyId).join(", ") || "(none)";
|
|
10856
|
-
throw new KeyUnavailableError(
|
|
10857
|
-
`no held KEK can unwrap this DEK \u2014 the wrappings reference [${need}]. Restore the missing KEK (or its recovery KEK), or rebuild the segment from source.`
|
|
10858
|
-
);
|
|
10859
|
-
}
|
|
10860
|
-
};
|
|
10861
|
-
|
|
10862
10258
|
// src/core/cas-retry.ts
|
|
10863
10259
|
async function casRetry(deps, ref, plan, step) {
|
|
10864
10260
|
const churn = leaseChurn(deps);
|
|
@@ -10904,6 +10300,7 @@ function validateRetentionPolicy(policy) {
|
|
|
10904
10300
|
`retention policy must be an object such as { expiresAt }; got ${policy === null ? "null" : Array.isArray(policy) ? "an array" : `a ${typeof policy}`}`
|
|
10905
10301
|
);
|
|
10906
10302
|
}
|
|
10303
|
+
refuseUnknownOptions("setRetention policy", policy, ["expiresAt"]);
|
|
10907
10304
|
const { expiresAt } = policy;
|
|
10908
10305
|
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
10909
10306
|
throw new ValidationError(
|
|
@@ -10931,7 +10328,7 @@ function readRetentionPolicy(meta) {
|
|
|
10931
10328
|
return { expiresAt: raw };
|
|
10932
10329
|
}
|
|
10933
10330
|
async function getSegmentRetention(ref, deps) {
|
|
10934
|
-
validateUserRef(ref);
|
|
10331
|
+
validateUserRef(ref, "getRetention");
|
|
10935
10332
|
const record = await deps.registry.get(ref);
|
|
10936
10333
|
if (record === null || record.status === "destroyed") return null;
|
|
10937
10334
|
return readRetentionPolicy(record.retention);
|
|
@@ -10940,20 +10337,25 @@ function readExpiresAt(record) {
|
|
|
10940
10337
|
const parsed = record === null ? null : readRetentionPolicy(record.retention);
|
|
10941
10338
|
return parsed === null || parsed === "invalid" ? void 0 : parsed.expiresAt;
|
|
10942
10339
|
}
|
|
10943
|
-
async function reindex(ref, deps, expiresAt,
|
|
10340
|
+
async function reindex(ref, deps, expiresAt, previous) {
|
|
10944
10341
|
if (!canIndex(ref)) return false;
|
|
10945
10342
|
const nextBucket = expiresAt === void 0 ? void 0 : dueBucket(expiresAt);
|
|
10946
|
-
const oldBucket = previousExpiresAt === void 0 ? void 0 : dueBucket(previousExpiresAt);
|
|
10947
10343
|
let indexed = expiresAt === void 0;
|
|
10948
10344
|
if (nextBucket !== void 0) {
|
|
10345
|
+
const pointer = dueIndexRef(nextBucket, ref);
|
|
10949
10346
|
try {
|
|
10950
|
-
await deps.registry.create(
|
|
10347
|
+
await deps.registry.create(pointer, { currentGen: null });
|
|
10951
10348
|
indexed = true;
|
|
10952
10349
|
} catch (err) {
|
|
10953
|
-
if (isWriteConflictError(err))
|
|
10350
|
+
if (isWriteConflictError(err)) {
|
|
10351
|
+
indexed = true;
|
|
10352
|
+
} else {
|
|
10353
|
+
indexed = await deps.registry.get(pointer).catch(() => null) !== null;
|
|
10354
|
+
}
|
|
10954
10355
|
}
|
|
10955
10356
|
}
|
|
10956
|
-
|
|
10357
|
+
for (const oldBucket of new Set([...previous].map(dueBucket))) {
|
|
10358
|
+
if (oldBucket === nextBucket) continue;
|
|
10957
10359
|
try {
|
|
10958
10360
|
await deps.registry.delete(dueIndexRef(oldBucket, ref));
|
|
10959
10361
|
} catch {
|
|
@@ -10961,10 +10363,38 @@ async function reindex(ref, deps, expiresAt, previousExpiresAt) {
|
|
|
10961
10363
|
}
|
|
10962
10364
|
return indexed;
|
|
10963
10365
|
}
|
|
10366
|
+
async function settledWrite(ref, deps, tokens, before, send) {
|
|
10367
|
+
try {
|
|
10368
|
+
await send();
|
|
10369
|
+
return null;
|
|
10370
|
+
} catch (err) {
|
|
10371
|
+
if (!isTransientError(err) || tokens.sent === void 0) throw err;
|
|
10372
|
+
const now = await readBack(() => deps.registry.get(ref), err, deps);
|
|
10373
|
+
if (tokens.wrote(now)) return now;
|
|
10374
|
+
if (now === null ? before === null : now.token === before?.token) throw err;
|
|
10375
|
+
const moved = new WriteConflictError(
|
|
10376
|
+
`the row of segment "${ref.segment}" changed while its retention write got no answer`
|
|
10377
|
+
);
|
|
10378
|
+
moved.cause = err;
|
|
10379
|
+
throw moved;
|
|
10380
|
+
}
|
|
10381
|
+
}
|
|
10964
10382
|
async function setSegmentRetention(ref, deps, policy) {
|
|
10965
|
-
validateUserRef(ref);
|
|
10383
|
+
validateUserRef(ref, "setRetention");
|
|
10966
10384
|
validateRetentionPolicy(policy);
|
|
10967
10385
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
10386
|
+
const created = /* @__PURE__ */ new Set();
|
|
10387
|
+
let creating = false;
|
|
10388
|
+
const tokens = writeTokens(deps.registry, (token) => {
|
|
10389
|
+
if (creating) created.add(token);
|
|
10390
|
+
});
|
|
10391
|
+
const previous = /* @__PURE__ */ new Set();
|
|
10392
|
+
const done = async (createdRow, held = policy.expiresAt) => ({
|
|
10393
|
+
...base,
|
|
10394
|
+
expiresAt: policy.expiresAt,
|
|
10395
|
+
createdRow,
|
|
10396
|
+
indexed: await reindex(ref, deps, held, previous)
|
|
10397
|
+
});
|
|
10968
10398
|
return casRetry(
|
|
10969
10399
|
deps,
|
|
10970
10400
|
ref,
|
|
@@ -10975,34 +10405,53 @@ async function setSegmentRetention(ref, deps, policy) {
|
|
|
10975
10405
|
)
|
|
10976
10406
|
},
|
|
10977
10407
|
async (record) => {
|
|
10978
|
-
|
|
10408
|
+
if (record !== null && tokens.wrote(record)) {
|
|
10409
|
+
return done(created.has(record.pointerId), readExpiresAt(record));
|
|
10410
|
+
}
|
|
10979
10411
|
if (record === null) {
|
|
10980
|
-
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
|
|
10984
|
-
|
|
10985
|
-
|
|
10986
|
-
|
|
10987
|
-
|
|
10988
|
-
|
|
10989
|
-
|
|
10412
|
+
creating = true;
|
|
10413
|
+
let landed2;
|
|
10414
|
+
try {
|
|
10415
|
+
landed2 = await settledWrite(
|
|
10416
|
+
ref,
|
|
10417
|
+
deps,
|
|
10418
|
+
tokens,
|
|
10419
|
+
null,
|
|
10420
|
+
() => deps.registry.create(
|
|
10421
|
+
ref,
|
|
10422
|
+
{ currentGen: null, retention: { [EXPIRES_AT]: policy.expiresAt } },
|
|
10423
|
+
tokens.options
|
|
10424
|
+
)
|
|
10425
|
+
);
|
|
10426
|
+
} finally {
|
|
10427
|
+
creating = false;
|
|
10428
|
+
}
|
|
10429
|
+
return done(true, landed2 === null ? policy.expiresAt : readExpiresAt(landed2));
|
|
10990
10430
|
}
|
|
10991
10431
|
assertNotDestroyed(record, ref);
|
|
10992
|
-
|
|
10993
|
-
|
|
10994
|
-
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
10998
|
-
|
|
10999
|
-
|
|
11000
|
-
|
|
10432
|
+
const was = readExpiresAt(record);
|
|
10433
|
+
if (was !== void 0) previous.add(was);
|
|
10434
|
+
const landed = await settledWrite(
|
|
10435
|
+
ref,
|
|
10436
|
+
deps,
|
|
10437
|
+
tokens,
|
|
10438
|
+
record,
|
|
10439
|
+
() => deps.registry.compareAndSwap(
|
|
10440
|
+
ref,
|
|
10441
|
+
record.token,
|
|
10442
|
+
{ retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt } },
|
|
10443
|
+
tokens.options
|
|
10444
|
+
)
|
|
10445
|
+
);
|
|
10446
|
+
return done(false, landed === null ? policy.expiresAt : readExpiresAt(landed));
|
|
11001
10447
|
}
|
|
11002
10448
|
);
|
|
11003
10449
|
}
|
|
11004
10450
|
async function clearSegmentRetention(ref, deps) {
|
|
11005
|
-
validateUserRef(ref);
|
|
10451
|
+
validateUserRef(ref, "clearRetention");
|
|
10452
|
+
const tokens = writeTokens(deps.registry);
|
|
10453
|
+
const previous = /* @__PURE__ */ new Set();
|
|
10454
|
+
let unanswered = false;
|
|
11006
10455
|
return casRetry(
|
|
11007
10456
|
deps,
|
|
11008
10457
|
ref,
|
|
@@ -11013,15 +10462,40 @@ async function clearSegmentRetention(ref, deps) {
|
|
|
11013
10462
|
)
|
|
11014
10463
|
},
|
|
11015
10464
|
async (record) => {
|
|
10465
|
+
if (tokens.wrote(record)) {
|
|
10466
|
+
await reindex(ref, deps, void 0, previous);
|
|
10467
|
+
return true;
|
|
10468
|
+
}
|
|
11016
10469
|
if (record === null) return false;
|
|
11017
10470
|
if (record.status === "destroyed") return false;
|
|
11018
|
-
|
|
10471
|
+
const expiresAt = readExpiresAt(record);
|
|
10472
|
+
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) {
|
|
10473
|
+
if (unanswered) await reindex(ref, deps, void 0, previous);
|
|
10474
|
+
return false;
|
|
10475
|
+
}
|
|
10476
|
+
if (expiresAt !== void 0) previous.add(expiresAt);
|
|
11019
10477
|
const rest = { ...record.retention };
|
|
11020
10478
|
delete rest[EXPIRES_AT];
|
|
11021
|
-
|
|
11022
|
-
|
|
11023
|
-
|
|
11024
|
-
|
|
10479
|
+
try {
|
|
10480
|
+
await settledWrite(
|
|
10481
|
+
ref,
|
|
10482
|
+
deps,
|
|
10483
|
+
tokens,
|
|
10484
|
+
record,
|
|
10485
|
+
() => deps.registry.compareAndSwap(
|
|
10486
|
+
ref,
|
|
10487
|
+
record.token,
|
|
10488
|
+
{ retention: Object.keys(rest).length === 0 ? void 0 : rest },
|
|
10489
|
+
tokens.options
|
|
10490
|
+
)
|
|
10491
|
+
);
|
|
10492
|
+
} catch (err) {
|
|
10493
|
+
if (isTransientError(err) || isUnsettledConflict(err) || isWriteConflictError(err) && isTransientError(err.cause)) {
|
|
10494
|
+
unanswered = true;
|
|
10495
|
+
}
|
|
10496
|
+
throw err;
|
|
10497
|
+
}
|
|
10498
|
+
await reindex(ref, deps, void 0, previous);
|
|
11025
10499
|
return true;
|
|
11026
10500
|
}
|
|
11027
10501
|
);
|
|
@@ -11029,7 +10503,7 @@ async function clearSegmentRetention(ref, deps) {
|
|
|
11029
10503
|
function assertNotDestroyed(record, ref) {
|
|
11030
10504
|
if (record.status === "destroyed") {
|
|
11031
10505
|
throw new ValidationError(
|
|
11032
|
-
`segment "${ref.segment}" is destroyed
|
|
10506
|
+
`segment "${ref.segment}" is destroyed \u2014 refusing to set a retention policy on a tombstone. Use a new segment; to delete objects a destroy left in the bucket, run dropSegment.`
|
|
11033
10507
|
);
|
|
11034
10508
|
}
|
|
11035
10509
|
}
|
|
@@ -11038,8 +10512,31 @@ function assertNotDestroyed(record, ref) {
|
|
|
11038
10512
|
var MAX_CAS_ATTEMPTS = 8;
|
|
11039
10513
|
var ERASE_CONCURRENCY = 8;
|
|
11040
10514
|
var MAX_STORAGE_SWEEPS = 3;
|
|
10515
|
+
function checkEraseDeps(name, deps) {
|
|
10516
|
+
const registry = deps === null || typeof deps !== "object" ? void 0 : deps.registry;
|
|
10517
|
+
if (registry === void 0 || registry === null || typeof registry.get !== "function") {
|
|
10518
|
+
throw new ValidationError(
|
|
10519
|
+
`${name}: \`deps.registry\` must be a registry driver, such as \`backend.registry\``
|
|
10520
|
+
);
|
|
10521
|
+
}
|
|
10522
|
+
}
|
|
10523
|
+
function checkDropOptions(options) {
|
|
10524
|
+
refuseUnknownOptions("dropSegment", options, ["confirmSegment", "dryRun", "audit"]);
|
|
10525
|
+
requireBool("dropSegment: `dryRun`", options.dryRun);
|
|
10526
|
+
checkedAuditSink(options.audit, "dropSegment");
|
|
10527
|
+
}
|
|
10528
|
+
var DESTROY_SEGMENT_KEYS = ["confirmSegment", "allowCleartext", "audit"];
|
|
10529
|
+
var ERASE_NAMESPACE_KEYS = [
|
|
10530
|
+
"confirmNamespace",
|
|
10531
|
+
"allowCleartext",
|
|
10532
|
+
"audit",
|
|
10533
|
+
"maxScanSegments"
|
|
10534
|
+
];
|
|
11041
10535
|
async function destroySegment(ref, deps, options) {
|
|
11042
|
-
validateUserRef(ref);
|
|
10536
|
+
validateUserRef(ref, "destroySegment");
|
|
10537
|
+
checkEraseDeps("destroySegment", deps);
|
|
10538
|
+
refuseUnknownOptions("destroySegment", options, DESTROY_SEGMENT_KEYS);
|
|
10539
|
+
requireBool("destroySegment: `allowCleartext`", options.allowCleartext);
|
|
11043
10540
|
checkedAuditSink(options.audit, "destroySegment");
|
|
11044
10541
|
if (options.confirmSegment !== ref.segment) {
|
|
11045
10542
|
throw new ValidationError(
|
|
@@ -11056,14 +10553,31 @@ async function destroySegment(ref, deps, options) {
|
|
|
11056
10553
|
segment: ref.segment,
|
|
11057
10554
|
...incarnationField(token)
|
|
11058
10555
|
});
|
|
10556
|
+
} else if (isCleartextTombstone(result)) {
|
|
10557
|
+
safeAudit(options.audit ?? NOOP_AUDIT).onEvent(tombstoneEvent(ref, token));
|
|
11059
10558
|
}
|
|
11060
10559
|
return result;
|
|
11061
10560
|
}
|
|
10561
|
+
function isCleartextTombstone(result) {
|
|
10562
|
+
return result.destroyed && !result.cryptoShredded && result.reason === void 0;
|
|
10563
|
+
}
|
|
10564
|
+
function tombstoneEvent(ref, token) {
|
|
10565
|
+
return {
|
|
10566
|
+
kind: "segment.tombstone",
|
|
10567
|
+
namespace: ref.namespace,
|
|
10568
|
+
segment: ref.segment,
|
|
10569
|
+
...incarnationField(token),
|
|
10570
|
+
objectsReadable: true
|
|
10571
|
+
};
|
|
10572
|
+
}
|
|
11062
10573
|
async function eraseNamespace(namespace, deps, options) {
|
|
11063
10574
|
if (typeof namespace !== "string" || namespace.length === 0) {
|
|
11064
10575
|
throw new ValidationError("eraseNamespace: namespace must be a non-empty string");
|
|
11065
10576
|
}
|
|
11066
10577
|
validateUserNamespace(namespace);
|
|
10578
|
+
checkEraseDeps("eraseNamespace", deps);
|
|
10579
|
+
refuseUnknownOptions("eraseNamespace", options, ERASE_NAMESPACE_KEYS);
|
|
10580
|
+
requireBool("eraseNamespace: `allowCleartext`", options.allowCleartext);
|
|
11067
10581
|
checkedAuditSink(options.audit, "eraseNamespace");
|
|
11068
10582
|
if (options.confirmNamespace !== namespace) {
|
|
11069
10583
|
throw new ValidationError(
|
|
@@ -11108,6 +10622,8 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
11108
10622
|
segment: ref.segment,
|
|
11109
10623
|
...incarnationField(token)
|
|
11110
10624
|
});
|
|
10625
|
+
} else if (isCleartextTombstone(result)) {
|
|
10626
|
+
audit.onEvent(tombstoneEvent(ref, token));
|
|
11111
10627
|
}
|
|
11112
10628
|
return result;
|
|
11113
10629
|
});
|
|
@@ -11118,8 +10634,8 @@ async function dropSegment(ref, deps, options) {
|
|
|
11118
10634
|
return await dropSegmentFor(ref, deps, options, void 0);
|
|
11119
10635
|
}
|
|
11120
10636
|
async function dropSegmentFor(ref, deps, options, sweep) {
|
|
11121
|
-
validateUserRef(ref);
|
|
11122
|
-
|
|
10637
|
+
validateUserRef(ref, "dropSegment");
|
|
10638
|
+
checkDropOptions(options);
|
|
11123
10639
|
if (options.confirmSegment !== ref.segment) {
|
|
11124
10640
|
throw new ValidationError(
|
|
11125
10641
|
`dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
|
|
@@ -11155,18 +10671,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11155
10671
|
reason: "absent"
|
|
11156
10672
|
};
|
|
11157
10673
|
}
|
|
11158
|
-
|
|
11159
|
-
const tombstone = await deps.registry.create(ref, {
|
|
11160
|
-
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
11161
|
-
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
11162
|
-
currentGen: orphans[orphans.length - 1],
|
|
11163
|
-
status: "destroyed"
|
|
11164
|
-
});
|
|
11165
|
-
shred = { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
11166
|
-
} catch (err) {
|
|
11167
|
-
if (!isWriteConflictError(err)) throw err;
|
|
11168
|
-
shred = await shredSegment(ref, deps, { allowCleartext: true, op: "dropSegment" });
|
|
11169
|
-
}
|
|
10674
|
+
shred = await claimIdentity(ref, deps, orphans[orphans.length - 1], shred);
|
|
11170
10675
|
}
|
|
11171
10676
|
if (shred.cryptoShredded) {
|
|
11172
10677
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -11182,7 +10687,7 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11182
10687
|
if (present.length === 0) break;
|
|
11183
10688
|
const outcomes = await mapWithConcurrency(present, ERASE_CONCURRENCY, async (generation) => {
|
|
11184
10689
|
try {
|
|
11185
|
-
await deps.storage
|
|
10690
|
+
await deleteSettled(deps.storage, { ...ref, generation }, void 0, void 0, deps);
|
|
11186
10691
|
return generation;
|
|
11187
10692
|
} catch {
|
|
11188
10693
|
return null;
|
|
@@ -11210,16 +10715,49 @@ async function dropSegmentFor(ref, deps, options, sweep) {
|
|
|
11210
10715
|
return {
|
|
11211
10716
|
...base,
|
|
11212
10717
|
dropped: shred.destroyed,
|
|
11213
|
-
generationsDeleted,
|
|
10718
|
+
generationsDeleted: generationsDeleted.sort((a, b) => a - b),
|
|
11214
10719
|
generationsRemaining,
|
|
11215
10720
|
cryptoShredded: shred.cryptoShredded,
|
|
11216
10721
|
reason: shred.reason
|
|
11217
10722
|
};
|
|
11218
10723
|
}
|
|
10724
|
+
async function claimIdentity(ref, deps, newestGeneration, shred) {
|
|
10725
|
+
const createTokens = writeTokens(deps.registry);
|
|
10726
|
+
try {
|
|
10727
|
+
const tombstone = await deps.registry.create(
|
|
10728
|
+
ref,
|
|
10729
|
+
{
|
|
10730
|
+
// The max listed generation, so the row is consistent with what is on disk if anything reads it before
|
|
10731
|
+
// the sweep finishes. It is a tombstone, so no reader resolves through it either way.
|
|
10732
|
+
currentGen: newestGeneration,
|
|
10733
|
+
status: "destroyed"
|
|
10734
|
+
},
|
|
10735
|
+
createTokens.options
|
|
10736
|
+
);
|
|
10737
|
+
return { ...shred, destroyed: true, reason: void 0, token: tombstone.token };
|
|
10738
|
+
} catch (err) {
|
|
10739
|
+
if (!isWriteConflictError(err) && !isTransientError(err)) throw err;
|
|
10740
|
+
if (createTokens.sent === void 0 && isTransientError(err)) throw err;
|
|
10741
|
+
const now = isTransientError(err) ? await readBack(() => deps.registry.get(ref), err, deps) : await deps.registry.get(ref);
|
|
10742
|
+
if (now?.status === "destroyed" && createTokens.wrote(now)) {
|
|
10743
|
+
return { ...shred, destroyed: true, reason: void 0, token: now.token };
|
|
10744
|
+
}
|
|
10745
|
+
if (isTransientError(err) && now === null) throw err;
|
|
10746
|
+
return shredSegment(ref, deps, { allowCleartext: true, op: "dropSegment" });
|
|
10747
|
+
}
|
|
10748
|
+
}
|
|
11219
10749
|
async function shredSegment(ref, deps, options) {
|
|
11220
10750
|
const { allowCleartext, op = "destroySegment", sweep } = options;
|
|
11221
10751
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
11222
|
-
|
|
10752
|
+
const encryptedBy = /* @__PURE__ */ new Map();
|
|
10753
|
+
let sentEncrypted = false;
|
|
10754
|
+
const tokens = writeTokens(deps.registry, (token) => void encryptedBy.set(token, sentEncrypted));
|
|
10755
|
+
const ownShred = (row) => {
|
|
10756
|
+
if (row === null || row.status !== "destroyed") return null;
|
|
10757
|
+
const encrypted = encryptedBy.get(row.pointerId);
|
|
10758
|
+
return encrypted === void 0 ? null : { ...base, destroyed: true, cryptoShredded: encrypted, token: row.token };
|
|
10759
|
+
};
|
|
10760
|
+
const shred = casRetry(
|
|
11223
10761
|
deps,
|
|
11224
10762
|
ref,
|
|
11225
10763
|
{
|
|
@@ -11227,6 +10765,8 @@ async function shredSegment(ref, deps, options) {
|
|
|
11227
10765
|
contention: () => new WriteConflictError(`${op}: contention shredding "${ref.segment}" \u2014 retry`)
|
|
11228
10766
|
},
|
|
11229
10767
|
async (record) => {
|
|
10768
|
+
const own = ownShred(record);
|
|
10769
|
+
if (own !== null) return own;
|
|
11230
10770
|
if (sweep !== void 0 && (record === null || record.status === "destroyed" || !sweep.wanted(record))) {
|
|
11231
10771
|
return null;
|
|
11232
10772
|
}
|
|
@@ -11247,32 +10787,67 @@ async function shredSegment(ref, deps, options) {
|
|
|
11247
10787
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "cleartext" };
|
|
11248
10788
|
}
|
|
11249
10789
|
const unmarked = sweep === void 0 ? withoutSweepMark(record.retention) : null;
|
|
11250
|
-
|
|
11251
|
-
|
|
11252
|
-
|
|
11253
|
-
|
|
11254
|
-
|
|
11255
|
-
|
|
11256
|
-
|
|
11257
|
-
|
|
11258
|
-
|
|
11259
|
-
|
|
11260
|
-
|
|
11261
|
-
|
|
11262
|
-
|
|
11263
|
-
|
|
11264
|
-
|
|
11265
|
-
|
|
10790
|
+
sentEncrypted = encrypted;
|
|
10791
|
+
let tombstone;
|
|
10792
|
+
try {
|
|
10793
|
+
tombstone = await deps.registry.compareAndSwap(
|
|
10794
|
+
ref,
|
|
10795
|
+
record.token,
|
|
10796
|
+
tombstonePatch(record, sweep, unmarked),
|
|
10797
|
+
tokens.options
|
|
10798
|
+
);
|
|
10799
|
+
} catch (err) {
|
|
10800
|
+
if (!isTransientError(err) || tokens.sent === void 0) throw err;
|
|
10801
|
+
const now = await readBack(() => deps.registry.get(ref), err, deps);
|
|
10802
|
+
const landed = ownShred(now);
|
|
10803
|
+
if (landed !== null) return landed;
|
|
10804
|
+
if (now !== null && now.token === record.token) throw err;
|
|
10805
|
+
const moved = new WriteConflictError(
|
|
10806
|
+
`${op}: the row of "${ref.segment}" changed while its shred got no answer`
|
|
10807
|
+
);
|
|
10808
|
+
moved.cause = err;
|
|
10809
|
+
throw moved;
|
|
10810
|
+
}
|
|
11266
10811
|
return { ...base, destroyed: true, cryptoShredded: encrypted, token: tombstone.token };
|
|
11267
10812
|
}
|
|
11268
10813
|
);
|
|
10814
|
+
try {
|
|
10815
|
+
return await shred;
|
|
10816
|
+
} catch (err) {
|
|
10817
|
+
if (isWriteConflictError(err)) {
|
|
10818
|
+
const own = ownShred(await deps.registry.get(ref).catch(() => null));
|
|
10819
|
+
if (own !== null) return own;
|
|
10820
|
+
}
|
|
10821
|
+
throw err;
|
|
10822
|
+
}
|
|
10823
|
+
}
|
|
10824
|
+
function tombstonePatch(record, sweep, unmarked) {
|
|
10825
|
+
return {
|
|
10826
|
+
status: "destroyed",
|
|
10827
|
+
// ← the crypto-shred: the current row's DEK wrappings are gone (earlier versions and backups of the row keep
|
|
10828
|
+
// theirs)
|
|
10829
|
+
wrappedDeks: void 0,
|
|
10830
|
+
keyId: void 0,
|
|
10831
|
+
// The current generation's cached count and metadata go with it: sealed, they cannot be opened without
|
|
10832
|
+
// the wrappings; clear, they would outlive the segment on its tombstone.
|
|
10833
|
+
summary: void 0,
|
|
10834
|
+
// A tombstone holds nothing: a lease names a generation of a segment that now resolves none, and the ledger
|
|
10835
|
+
// vouches for objects of a segment no read resolves, which the pointer staying put would otherwise keep.
|
|
10836
|
+
knownGens: void 0,
|
|
10837
|
+
leases: void 0,
|
|
10838
|
+
// A sweep's mark, in this same write, so no other tombstone ever carries it: any other tombstone write removes
|
|
10839
|
+
// one a restored or hand-edited row carried.
|
|
10840
|
+
...sweep !== void 0 ? { retention: sweep.stamp(record.retention) } : unmarked === null ? {} : { retention: unmarked }
|
|
10841
|
+
};
|
|
11269
10842
|
}
|
|
11270
10843
|
|
|
11271
10844
|
// src/core/retention-sweep.ts
|
|
11272
10845
|
var DEFAULT_RETIRE_LIMIT = 100;
|
|
11273
10846
|
var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
11274
10847
|
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
10848
|
+
var MIN_TOMBSTONE_GRACE_MS = 36e5;
|
|
11275
10849
|
var DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
10850
|
+
var MAX_LOOKBACK_BUCKETS = 3660;
|
|
11276
10851
|
async function rowsFromDueIndex(registry, options) {
|
|
11277
10852
|
const rows = [];
|
|
11278
10853
|
const pointers = /* @__PURE__ */ new Map();
|
|
@@ -11348,10 +10923,10 @@ async function rowsFromFleet(registry, options) {
|
|
|
11348
10923
|
}
|
|
11349
10924
|
return { rows, pointers, litter };
|
|
11350
10925
|
}
|
|
11351
|
-
async function forgetDuePointer(
|
|
10926
|
+
async function forgetDuePointer(deps, ref, expiresAt, onFault) {
|
|
11352
10927
|
if (!canIndex(ref)) return;
|
|
11353
10928
|
try {
|
|
11354
|
-
await registry
|
|
10929
|
+
await rowDeleteSettled(deps.registry, dueIndexRef(dueBucket(expiresAt), ref), void 0, deps);
|
|
11355
10930
|
} catch (err) {
|
|
11356
10931
|
onFault(err);
|
|
11357
10932
|
}
|
|
@@ -11363,22 +10938,22 @@ async function filePurgePointer(registry, ref, purgeAt) {
|
|
|
11363
10938
|
} catch {
|
|
11364
10939
|
}
|
|
11365
10940
|
}
|
|
11366
|
-
async function forgetFoundPointers(
|
|
10941
|
+
async function forgetFoundPointers(deps, found2, onFault) {
|
|
11367
10942
|
for (const pointer of found2) {
|
|
11368
10943
|
try {
|
|
11369
|
-
await registry
|
|
10944
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11370
10945
|
} catch (err) {
|
|
11371
10946
|
onFault(err);
|
|
11372
10947
|
}
|
|
11373
10948
|
}
|
|
11374
10949
|
}
|
|
11375
|
-
async function forgetPurgePointers(
|
|
10950
|
+
async function forgetPurgePointers(deps, ref, purgeAt, found2, onFault) {
|
|
11376
10951
|
if (!canIndex(ref)) return;
|
|
11377
10952
|
const filed = dueIndexRef(dueBucket(purgeAt), ref);
|
|
11378
10953
|
const all = found2.some((p) => p.ref.namespace === filed.namespace) ? found2 : [...found2, { ref: filed, target: ref }];
|
|
11379
10954
|
for (const pointer of all) {
|
|
11380
10955
|
try {
|
|
11381
|
-
await registry
|
|
10956
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11382
10957
|
} catch (err) {
|
|
11383
10958
|
onFault(err);
|
|
11384
10959
|
}
|
|
@@ -11394,10 +10969,38 @@ function resolveRetireOptions(options) {
|
|
|
11394
10969
|
if (!Array.isArray(options.shards)) {
|
|
11395
10970
|
throw new ValidationError("retireExpired: shards must be an array of shard numbers");
|
|
11396
10971
|
}
|
|
10972
|
+
if (options.shards.length === 0) {
|
|
10973
|
+
throw new ValidationError(
|
|
10974
|
+
"retireExpired: shards is empty, so this sweep would own no segment; pass at least one shard number, or leave shards out to sweep every one"
|
|
10975
|
+
);
|
|
10976
|
+
}
|
|
10977
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11397
10978
|
for (const shard of options.shards) {
|
|
11398
|
-
requireInteger(shard, { min: 0, max: total - 1, what: "retireExpired: a shard" });
|
|
10979
|
+
const n = requireInteger(shard, { min: 0, max: total - 1, what: "retireExpired: a shard" });
|
|
10980
|
+
if (seen.has(n)) {
|
|
10981
|
+
throw new ValidationError(
|
|
10982
|
+
`retireExpired: shard ${n} is listed twice; shards is a set of shard numbers`
|
|
10983
|
+
);
|
|
10984
|
+
}
|
|
10985
|
+
seen.add(n);
|
|
11399
10986
|
}
|
|
11400
10987
|
}
|
|
10988
|
+
const scan = options.scan === void 0 ? "fleet" : options.scan;
|
|
10989
|
+
if (scan !== "fleet" && scan !== "index") {
|
|
10990
|
+
throw new ValidationError(
|
|
10991
|
+
`retireExpired: \`scan\` must be 'fleet' or 'index'; got ${typeof scan === "string" ? JSON.stringify(scan) : scan === null ? "null" : typeof scan}`
|
|
10992
|
+
);
|
|
10993
|
+
}
|
|
10994
|
+
const lookbackBuckets = options.lookbackBuckets === void 0 ? DEFAULT_LOOKBACK_BUCKETS : options.lookbackBuckets;
|
|
10995
|
+
requireInteger(lookbackBuckets, {
|
|
10996
|
+
min: 0,
|
|
10997
|
+
max: MAX_LOOKBACK_BUCKETS,
|
|
10998
|
+
what: "retireExpired: `lookbackBuckets`",
|
|
10999
|
+
note: "days; older rows are the fleet scan's to find"
|
|
11000
|
+
});
|
|
11001
|
+
if (options.shards === void 0 && options.totalShards !== void 0) {
|
|
11002
|
+
requirePositiveInteger(options.totalShards, "retireExpired: `totalShards`");
|
|
11003
|
+
}
|
|
11401
11004
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
11402
11005
|
const now = options.now;
|
|
11403
11006
|
requireFiniteNumber(now, { what: "retireExpired: `now`", note: "epoch milliseconds" });
|
|
@@ -11406,11 +11009,15 @@ function resolveRetireOptions(options) {
|
|
|
11406
11009
|
`retireExpired: \`now\` (${now}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds. A sweep with a wrong-units clock treats every policy as expired.`
|
|
11407
11010
|
);
|
|
11408
11011
|
}
|
|
11409
|
-
const limit = options.limit
|
|
11012
|
+
const limit = options.limit === void 0 ? DEFAULT_RETIRE_LIMIT : options.limit;
|
|
11410
11013
|
requirePositiveInteger(limit, "retireExpired: `limit`");
|
|
11411
11014
|
const purgeTombstones = options.purgeTombstones ?? true;
|
|
11412
|
-
const grace = options.tombstoneGraceMs
|
|
11413
|
-
requireFiniteNumber(grace, {
|
|
11015
|
+
const grace = options.tombstoneGraceMs === void 0 ? DEFAULT_TOMBSTONE_GRACE_MS : options.tombstoneGraceMs;
|
|
11016
|
+
requireFiniteNumber(grace, {
|
|
11017
|
+
min: MIN_TOMBSTONE_GRACE_MS,
|
|
11018
|
+
what: "retireExpired: `tombstoneGraceMs`",
|
|
11019
|
+
note: "one hour: the tombstone must outlast a create sent again after it got no answer"
|
|
11020
|
+
});
|
|
11414
11021
|
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11415
11022
|
requirePositiveInteger(maxScanSegments, "retireExpired: `maxScanSegments`");
|
|
11416
11023
|
return {
|
|
@@ -11422,8 +11029,8 @@ function resolveRetireOptions(options) {
|
|
|
11422
11029
|
grace,
|
|
11423
11030
|
maxScanSegments,
|
|
11424
11031
|
dryRun: options.dryRun === true,
|
|
11425
|
-
scan
|
|
11426
|
-
lookbackBuckets
|
|
11032
|
+
scan,
|
|
11033
|
+
lookbackBuckets,
|
|
11427
11034
|
shards: options.shards,
|
|
11428
11035
|
totalShards: options.totalShards ?? 0
|
|
11429
11036
|
};
|
|
@@ -11480,7 +11087,7 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11480
11087
|
try {
|
|
11481
11088
|
const live = await deps.registry.get(ref);
|
|
11482
11089
|
if (live?.status === "destroyed" && retirementStamp(live.retention) === null) {
|
|
11483
|
-
await forgetFoundPointers(deps
|
|
11090
|
+
await forgetFoundPointers(deps, found2, tally.noteFault);
|
|
11484
11091
|
}
|
|
11485
11092
|
} catch (err) {
|
|
11486
11093
|
tally.noteFault(err);
|
|
@@ -11500,7 +11107,7 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11500
11107
|
if (!await isFullyReclaimed(deps, ref)) {
|
|
11501
11108
|
let collectFault;
|
|
11502
11109
|
if (!dryRun) {
|
|
11503
|
-
await gcOrphanGenerations(ref, deps).catch((error) => {
|
|
11110
|
+
await gcOrphanGenerations(ref, deps, { incarnation: rec }).catch((error) => {
|
|
11504
11111
|
collectFault = { error };
|
|
11505
11112
|
});
|
|
11506
11113
|
}
|
|
@@ -11518,12 +11125,12 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11518
11125
|
}
|
|
11519
11126
|
}
|
|
11520
11127
|
if (!dryRun) {
|
|
11521
|
-
await deps.registry
|
|
11128
|
+
await rowDeleteSettled(deps.registry, ref, rec.token, deps);
|
|
11522
11129
|
tally.tombstonesPurged += 1;
|
|
11523
11130
|
tally.purgeRun = 0;
|
|
11524
11131
|
if (removesRows) {
|
|
11525
11132
|
await forgetPurgePointers(
|
|
11526
|
-
deps
|
|
11133
|
+
deps,
|
|
11527
11134
|
ref,
|
|
11528
11135
|
retiredAt + grace,
|
|
11529
11136
|
pointers.get(segmentKey(ref)) ?? [],
|
|
@@ -11554,8 +11161,9 @@ async function retireLive(rec, ctx) {
|
|
|
11554
11161
|
tally.limited = true;
|
|
11555
11162
|
return null;
|
|
11556
11163
|
}
|
|
11164
|
+
let live = null;
|
|
11557
11165
|
try {
|
|
11558
|
-
|
|
11166
|
+
live = await deps.registry.get(ref);
|
|
11559
11167
|
const livePolicy = live === null ? null : readRetentionPolicy(live.retention);
|
|
11560
11168
|
if (live === null || live.status === "destroyed" || livePolicy === null || livePolicy === "invalid" || livePolicy.expiresAt > now) {
|
|
11561
11169
|
return { ...base, action: "skipped", reason: "policy-changed" };
|
|
@@ -11586,11 +11194,11 @@ async function retireLive(rec, ctx) {
|
|
|
11586
11194
|
}
|
|
11587
11195
|
tally.retired += 1;
|
|
11588
11196
|
tally.add({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
11589
|
-
await forgetDuePointer(deps
|
|
11197
|
+
await forgetDuePointer(deps, ref, livePolicy.expiresAt, tally.noteFault);
|
|
11590
11198
|
if (opts.purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
11591
11199
|
const tombstone = await deps.registry.get(ref).catch(() => null);
|
|
11592
11200
|
if (tombstone?.status === "destroyed") {
|
|
11593
|
-
const deleted = await deps.registry
|
|
11201
|
+
const deleted = await rowDeleteSettled(deps.registry, ref, tombstone.token, deps).then(
|
|
11594
11202
|
() => true,
|
|
11595
11203
|
(err) => {
|
|
11596
11204
|
tally.noteFault(err);
|
|
@@ -11603,28 +11211,58 @@ async function retireLive(rec, ctx) {
|
|
|
11603
11211
|
if (removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
11604
11212
|
return null;
|
|
11605
11213
|
} catch (err) {
|
|
11606
|
-
const after = await deps
|
|
11607
|
-
if (after?.status === "destroyed" && after.retention?.[SWEEP_MARK] === now) {
|
|
11608
|
-
|
|
11609
|
-
|
|
11214
|
+
const after = await rowAfterFault(deps, ref, err);
|
|
11215
|
+
if (after?.status === "destroyed" && after.retention?.[SWEEP_MARK] === now && live !== null && sameIncarnation(live, after)) {
|
|
11216
|
+
return retiredAfterFault(live, policy.expiresAt, err, ctx);
|
|
11217
|
+
}
|
|
11218
|
+
if (after === void 0) {
|
|
11610
11219
|
return {
|
|
11611
11220
|
...base,
|
|
11612
|
-
action: "
|
|
11613
|
-
|
|
11614
|
-
result: {
|
|
11615
|
-
...base,
|
|
11616
|
-
dropped: true,
|
|
11617
|
-
generationsDeleted: [],
|
|
11618
|
-
generationsRemaining: [],
|
|
11619
|
-
cryptoShredded: false,
|
|
11620
|
-
reason: void 0
|
|
11621
|
-
},
|
|
11622
|
-
fault: failureReason(err)
|
|
11221
|
+
action: "skipped",
|
|
11222
|
+
reason: `${failureReason(err)} (the row could not be read afterwards: the tombstone may have landed, and the segment may be shredded)`
|
|
11623
11223
|
};
|
|
11624
11224
|
}
|
|
11625
11225
|
return { ...base, action: "skipped", reason: failureReason(err) };
|
|
11626
11226
|
}
|
|
11627
11227
|
}
|
|
11228
|
+
async function rowAfterFault(deps, ref, err) {
|
|
11229
|
+
try {
|
|
11230
|
+
return await readBack(
|
|
11231
|
+
() => deps.registry.get(ref),
|
|
11232
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
11233
|
+
deps
|
|
11234
|
+
);
|
|
11235
|
+
} catch {
|
|
11236
|
+
return void 0;
|
|
11237
|
+
}
|
|
11238
|
+
}
|
|
11239
|
+
async function retiredAfterFault(live, expiresAt, err, ctx) {
|
|
11240
|
+
const { deps, opts, removesRows, tally } = ctx;
|
|
11241
|
+
const ref = { namespace: live.namespace, segment: live.segment };
|
|
11242
|
+
const base = { segment: live.segment, namespace: live.namespace };
|
|
11243
|
+
if (removesRows) await filePurgePointer(deps.registry, ref, opts.now + opts.grace);
|
|
11244
|
+
tally.retired += 1;
|
|
11245
|
+
const remaining = await generationsOf(deps.storage, ref).catch(() => namedBy(live));
|
|
11246
|
+
return {
|
|
11247
|
+
...base,
|
|
11248
|
+
action: "retired",
|
|
11249
|
+
expiresAt,
|
|
11250
|
+
result: {
|
|
11251
|
+
...base,
|
|
11252
|
+
dropped: true,
|
|
11253
|
+
generationsDeleted: [],
|
|
11254
|
+
generationsRemaining: remaining,
|
|
11255
|
+
cryptoShredded: live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0,
|
|
11256
|
+
reason: void 0
|
|
11257
|
+
},
|
|
11258
|
+
fault: failureReason(err)
|
|
11259
|
+
};
|
|
11260
|
+
}
|
|
11261
|
+
function namedBy(row) {
|
|
11262
|
+
const names = new Set((row.knownGens ?? []).map((k) => k.generation));
|
|
11263
|
+
if (row.currentGen !== null) names.add(row.currentGen);
|
|
11264
|
+
return [...names].sort((a, b) => a - b);
|
|
11265
|
+
}
|
|
11628
11266
|
async function retireExpired(deps, options) {
|
|
11629
11267
|
const opts = resolveRetireOptions(options);
|
|
11630
11268
|
const { now, limit, dryRun, shards, totalShards } = opts;
|
|
@@ -11656,7 +11294,7 @@ async function retireExpired(deps, options) {
|
|
|
11656
11294
|
attemptedLitter += 1;
|
|
11657
11295
|
try {
|
|
11658
11296
|
if (await deps.registry.get(pointer.target) !== null) continue;
|
|
11659
|
-
await deps.registry
|
|
11297
|
+
await rowDeleteSettled(deps.registry, pointer.ref, pointer.token, deps);
|
|
11660
11298
|
} catch (err) {
|
|
11661
11299
|
tally.noteFault(err);
|
|
11662
11300
|
}
|
|
@@ -11694,6 +11332,411 @@ function failureReason(err) {
|
|
|
11694
11332
|
return `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
11695
11333
|
}
|
|
11696
11334
|
|
|
11335
|
+
// src/core/preflight.ts
|
|
11336
|
+
var PREFLIGHT_NAMESPACE = `${RESERVED_NAMESPACE_PREFIX}preflight`;
|
|
11337
|
+
var PREFLIGHT_SEGMENT = "probe";
|
|
11338
|
+
var LIFECYCLE_TIMEOUT_MS = 3e4;
|
|
11339
|
+
var LIFECYCLE_TICK_MS = 250;
|
|
11340
|
+
var MARKER_A = ascii("cloudbitmaps preflight probe A");
|
|
11341
|
+
var MARKER_B = ascii("cloudbitmaps preflight probe B");
|
|
11342
|
+
var TAIL_BYTES = 64;
|
|
11343
|
+
var MAX_MESSAGE_CHARS = 300;
|
|
11344
|
+
var MAX_LEFTOVERS_NAMED = 3;
|
|
11345
|
+
var MAX_LIFECYCLE_DETAIL_CHARS = 1e3;
|
|
11346
|
+
function ascii(text) {
|
|
11347
|
+
const bytes = new Uint8Array(text.length);
|
|
11348
|
+
for (let i = 0; i < text.length; i += 1) bytes[i] = text.charCodeAt(i);
|
|
11349
|
+
return bytes;
|
|
11350
|
+
}
|
|
11351
|
+
function sameBytes2(a, b) {
|
|
11352
|
+
if (a.length !== b.length) return false;
|
|
11353
|
+
for (let i = 0; i < a.length; i += 1) if (a[i] !== b[i]) return false;
|
|
11354
|
+
return true;
|
|
11355
|
+
}
|
|
11356
|
+
function probeNumber(entropy) {
|
|
11357
|
+
const bytes = entropy(6);
|
|
11358
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < 6) {
|
|
11359
|
+
throw new TypeError("preflight needs an entropy source that returns the bytes it is asked for");
|
|
11360
|
+
}
|
|
11361
|
+
let n = 0;
|
|
11362
|
+
for (let i = 0; i < 6; i += 1) n = n * 256 + (bytes[i] ?? 0);
|
|
11363
|
+
return n;
|
|
11364
|
+
}
|
|
11365
|
+
function describeError(err) {
|
|
11366
|
+
let name = "Error";
|
|
11367
|
+
let message = "";
|
|
11368
|
+
try {
|
|
11369
|
+
if (typeof err === "object" && err !== null) {
|
|
11370
|
+
const n = err.name;
|
|
11371
|
+
const m = err.message;
|
|
11372
|
+
if (typeof n === "string" && n !== "") name = n;
|
|
11373
|
+
if (typeof m === "string") message = m;
|
|
11374
|
+
} else {
|
|
11375
|
+
message = String(err);
|
|
11376
|
+
}
|
|
11377
|
+
} catch {
|
|
11378
|
+
}
|
|
11379
|
+
const cut = message.length > MAX_MESSAGE_CHARS ? `${message.slice(0, MAX_MESSAGE_CHARS)}...` : message;
|
|
11380
|
+
return cut === "" ? name : `${name}: ${cut}`;
|
|
11381
|
+
}
|
|
11382
|
+
function failed(doing, err) {
|
|
11383
|
+
let transient = false;
|
|
11384
|
+
try {
|
|
11385
|
+
transient = isTransientError(err);
|
|
11386
|
+
} catch {
|
|
11387
|
+
}
|
|
11388
|
+
return `${doing} failed with ${describeError(err)}.${transient ? " This is a transient fault: run preflight() again." : ""}`;
|
|
11389
|
+
}
|
|
11390
|
+
function isConflict(err) {
|
|
11391
|
+
try {
|
|
11392
|
+
return isWriteConflictError(err);
|
|
11393
|
+
} catch {
|
|
11394
|
+
return false;
|
|
11395
|
+
}
|
|
11396
|
+
}
|
|
11397
|
+
var REFUSAL_NAMES = /* @__PURE__ */ new Set([
|
|
11398
|
+
"AccessDenied",
|
|
11399
|
+
"Forbidden",
|
|
11400
|
+
"AuthorizationFailure",
|
|
11401
|
+
"AuthorizationPermissionMismatch"
|
|
11402
|
+
]);
|
|
11403
|
+
function provesRefused(err) {
|
|
11404
|
+
try {
|
|
11405
|
+
for (let e = err, depth = 0; typeof e === "object" && e !== null && depth < 2; depth += 1) {
|
|
11406
|
+
if (isValidationError(e)) return true;
|
|
11407
|
+
const shape = e;
|
|
11408
|
+
if (typeof shape.name === "string" && REFUSAL_NAMES.has(shape.name)) return true;
|
|
11409
|
+
const status = shape.$metadata?.httpStatusCode ?? shape.statusCode ?? shape.status ?? shape.code;
|
|
11410
|
+
if (status === 401 || status === 403) return true;
|
|
11411
|
+
e = shape.cause;
|
|
11412
|
+
}
|
|
11413
|
+
} catch {
|
|
11414
|
+
}
|
|
11415
|
+
return false;
|
|
11416
|
+
}
|
|
11417
|
+
var ORDER = [
|
|
11418
|
+
"registry",
|
|
11419
|
+
"write",
|
|
11420
|
+
"write-once",
|
|
11421
|
+
"read",
|
|
11422
|
+
"list",
|
|
11423
|
+
"delete",
|
|
11424
|
+
"lifecycle"
|
|
11425
|
+
];
|
|
11426
|
+
async function lifecycleStep(deps) {
|
|
11427
|
+
const check = deps.checkLifecycle;
|
|
11428
|
+
if (check === void 0) {
|
|
11429
|
+
return {
|
|
11430
|
+
status: "unchecked",
|
|
11431
|
+
detail: "This backend does not offer the lifecycle check. Look at the bucket lifecycle rules by hand: none may delete or archive the store objects or the registry rows."
|
|
11432
|
+
};
|
|
11433
|
+
}
|
|
11434
|
+
let settled2 = false;
|
|
11435
|
+
const run = (async () => {
|
|
11436
|
+
try {
|
|
11437
|
+
return { kind: "answered", verdict: await check() };
|
|
11438
|
+
} catch (error) {
|
|
11439
|
+
return { kind: "threw", error };
|
|
11440
|
+
} finally {
|
|
11441
|
+
settled2 = true;
|
|
11442
|
+
}
|
|
11443
|
+
})();
|
|
11444
|
+
const bound = (async () => {
|
|
11445
|
+
try {
|
|
11446
|
+
for (let waited = 0; waited < LIFECYCLE_TIMEOUT_MS && !settled2; waited += LIFECYCLE_TICK_MS) {
|
|
11447
|
+
await deps.clock.sleep(LIFECYCLE_TICK_MS);
|
|
11448
|
+
}
|
|
11449
|
+
} catch (error) {
|
|
11450
|
+
return { kind: "clock", error };
|
|
11451
|
+
}
|
|
11452
|
+
return { kind: "timeout" };
|
|
11453
|
+
})();
|
|
11454
|
+
let outcome = await Promise.race([run, bound]);
|
|
11455
|
+
if (outcome.kind === "timeout" && settled2) outcome = await run;
|
|
11456
|
+
if (outcome.kind === "timeout") {
|
|
11457
|
+
return {
|
|
11458
|
+
status: "unchecked",
|
|
11459
|
+
detail: `The lifecycle check did not answer within ${LIFECYCLE_TIMEOUT_MS / 1e3} seconds. Run preflight() again, or look at the bucket lifecycle rules by hand.`
|
|
11460
|
+
};
|
|
11461
|
+
}
|
|
11462
|
+
if (outcome.kind === "clock") {
|
|
11463
|
+
return {
|
|
11464
|
+
status: "unchecked",
|
|
11465
|
+
detail: `The store's clock failed while the lifecycle check was timed: ${describeError(outcome.error)}. Run preflight() again, or look at the bucket lifecycle rules by hand.`
|
|
11466
|
+
};
|
|
11467
|
+
}
|
|
11468
|
+
if (outcome.kind === "threw") {
|
|
11469
|
+
return {
|
|
11470
|
+
status: "unchecked",
|
|
11471
|
+
detail: `${failed("The lifecycle check", outcome.error)} Look at the bucket lifecycle rules by hand.`
|
|
11472
|
+
};
|
|
11473
|
+
}
|
|
11474
|
+
return readVerdict(outcome.verdict);
|
|
11475
|
+
}
|
|
11476
|
+
function readVerdict(answer) {
|
|
11477
|
+
const unreadable = {
|
|
11478
|
+
status: "unchecked",
|
|
11479
|
+
detail: "The lifecycle check gave an answer preflight could not read. Look at the bucket lifecycle rules by hand."
|
|
11480
|
+
};
|
|
11481
|
+
try {
|
|
11482
|
+
const verdict = answer;
|
|
11483
|
+
const status = verdict?.status;
|
|
11484
|
+
if (status !== "ok" && status !== "problem" && status !== "unchecked") return unreadable;
|
|
11485
|
+
const text = verdict?.detail;
|
|
11486
|
+
const detail = typeof text === "string" ? text : "";
|
|
11487
|
+
return {
|
|
11488
|
+
status,
|
|
11489
|
+
detail: detail.length > MAX_LIFECYCLE_DETAIL_CHARS ? `${detail.slice(0, MAX_LIFECYCLE_DETAIL_CHARS)}...` : detail
|
|
11490
|
+
};
|
|
11491
|
+
} catch {
|
|
11492
|
+
return unreadable;
|
|
11493
|
+
}
|
|
11494
|
+
}
|
|
11495
|
+
async function registryStep(registry) {
|
|
11496
|
+
const row = await registry.get({ namespace: PREFLIGHT_NAMESPACE, segment: PREFLIGHT_SEGMENT });
|
|
11497
|
+
if (row !== null) {
|
|
11498
|
+
return {
|
|
11499
|
+
status: "problem",
|
|
11500
|
+
detail: `A registry row exists at ${PREFLIGHT_NAMESPACE}/${PREFLIGHT_SEGMENT}, an address only the library uses and never writes a row at. The registry is not the one this store expects, or something else writes there.`
|
|
11501
|
+
};
|
|
11502
|
+
}
|
|
11503
|
+
if (registry.capabilities().canWrite === false) {
|
|
11504
|
+
return {
|
|
11505
|
+
status: "problem",
|
|
11506
|
+
detail: "The registry reads but cannot write in this runtime (no Web Crypto to draw its tokens from), so a load would be refused."
|
|
11507
|
+
};
|
|
11508
|
+
}
|
|
11509
|
+
return { status: "ok", detail: "The registry answered a read, and can write." };
|
|
11510
|
+
}
|
|
11511
|
+
async function guarded(run, onThrow) {
|
|
11512
|
+
try {
|
|
11513
|
+
return await run();
|
|
11514
|
+
} catch (err) {
|
|
11515
|
+
return onThrow(err);
|
|
11516
|
+
}
|
|
11517
|
+
}
|
|
11518
|
+
var skipped = (because) => ({
|
|
11519
|
+
status: "unchecked",
|
|
11520
|
+
detail: `Not run: ${because}.`
|
|
11521
|
+
});
|
|
11522
|
+
async function writeStep(probe) {
|
|
11523
|
+
try {
|
|
11524
|
+
await probe.storage.putImmutable(probe.key, async (sink) => sink.write(MARKER_A));
|
|
11525
|
+
probe.found.set("write", { status: "ok", detail: "A new object was written." });
|
|
11526
|
+
return { wrote: true, notOurs: false };
|
|
11527
|
+
} catch (err) {
|
|
11528
|
+
const notOurs = isConflict(err);
|
|
11529
|
+
probe.refused = !notOurs && provesRefused(err);
|
|
11530
|
+
probe.found.set("write", {
|
|
11531
|
+
status: "problem",
|
|
11532
|
+
detail: notOurs ? "The probe object already existed: another run drew the same number. Run preflight() again." : failed("Writing a new object", err)
|
|
11533
|
+
});
|
|
11534
|
+
return { wrote: false, notOurs };
|
|
11535
|
+
}
|
|
11536
|
+
}
|
|
11537
|
+
async function writeOnceStep(probe) {
|
|
11538
|
+
try {
|
|
11539
|
+
await probe.storage.putImmutable(probe.key, async (sink) => sink.write(MARKER_B));
|
|
11540
|
+
probe.accepted = MARKER_B;
|
|
11541
|
+
probe.found.set("write-once", {
|
|
11542
|
+
status: "problem",
|
|
11543
|
+
detail: "A second write to the same key succeeded. The storage does not refuse an overwrite, so a generation could be replaced under a reader."
|
|
11544
|
+
});
|
|
11545
|
+
} catch (err) {
|
|
11546
|
+
probe.found.set(
|
|
11547
|
+
"write-once",
|
|
11548
|
+
isConflict(err) ? { status: "ok", detail: "A second write to the same key was refused." } : {
|
|
11549
|
+
status: "problem",
|
|
11550
|
+
detail: `${failed("The second write to the same key", err)} Whether it was refused is not known.`
|
|
11551
|
+
}
|
|
11552
|
+
);
|
|
11553
|
+
}
|
|
11554
|
+
}
|
|
11555
|
+
async function readStep(probe) {
|
|
11556
|
+
try {
|
|
11557
|
+
const tail = await probe.storage.getTail(probe.key, TAIL_BYTES);
|
|
11558
|
+
probe.version = tail.version;
|
|
11559
|
+
const same = sameBytes2(tail.bytes, probe.accepted) && tail.size === probe.accepted.length;
|
|
11560
|
+
probe.found.set(
|
|
11561
|
+
"read",
|
|
11562
|
+
same ? { status: "ok", detail: "The object just written read back as written." } : {
|
|
11563
|
+
status: "problem",
|
|
11564
|
+
detail: `The object just written did not read back as written: the bytes or the size differ.${probe.found.get("write-once")?.status === "problem" ? " It was compared against the bytes the storage accepted last, so look at the write-once check first." : ""}`
|
|
11565
|
+
}
|
|
11566
|
+
);
|
|
11567
|
+
} catch (err) {
|
|
11568
|
+
probe.found.set("read", {
|
|
11569
|
+
status: "problem",
|
|
11570
|
+
detail: failed("Reading the object just written", err)
|
|
11571
|
+
});
|
|
11572
|
+
}
|
|
11573
|
+
}
|
|
11574
|
+
function leftoversSentence(others, named) {
|
|
11575
|
+
if (others === 0) return "";
|
|
11576
|
+
const more = others > named.length ? ", ..." : "";
|
|
11577
|
+
const one = others === 1;
|
|
11578
|
+
return ` ${others} other probe object${one ? "" : "s"} (generation ${named.join(", ")}${more}) ${one ? "was" : "were"} left alone: ${one ? "it may be" : "they may be"} a run in progress, or one that was cut off.`;
|
|
11579
|
+
}
|
|
11580
|
+
async function listStep(probe) {
|
|
11581
|
+
try {
|
|
11582
|
+
let seen = false;
|
|
11583
|
+
let others = 0;
|
|
11584
|
+
const named = [];
|
|
11585
|
+
for await (const listed2 of probe.storage.list({
|
|
11586
|
+
namespace: PREFLIGHT_NAMESPACE,
|
|
11587
|
+
segment: PREFLIGHT_SEGMENT
|
|
11588
|
+
})) {
|
|
11589
|
+
if (listed2.generation === probe.key.generation) {
|
|
11590
|
+
seen = true;
|
|
11591
|
+
} else {
|
|
11592
|
+
others += 1;
|
|
11593
|
+
if (named.length < MAX_LEFTOVERS_NAMED) named.push(listed2.generation);
|
|
11594
|
+
}
|
|
11595
|
+
}
|
|
11596
|
+
const leftovers = leftoversSentence(others, named);
|
|
11597
|
+
probe.found.set(
|
|
11598
|
+
"list",
|
|
11599
|
+
seen ? { status: "ok", detail: `The listing named the object just written.${leftovers}` } : {
|
|
11600
|
+
status: "problem",
|
|
11601
|
+
detail: `The listing did not name the object just written, so a listing can lag a write.${leftovers}`
|
|
11602
|
+
}
|
|
11603
|
+
);
|
|
11604
|
+
} catch (err) {
|
|
11605
|
+
probe.found.set("list", {
|
|
11606
|
+
status: "problem",
|
|
11607
|
+
detail: failed("Listing the probe segment", err)
|
|
11608
|
+
});
|
|
11609
|
+
}
|
|
11610
|
+
}
|
|
11611
|
+
async function deleteStep(probe, wrote) {
|
|
11612
|
+
try {
|
|
11613
|
+
await probe.storage.delete(
|
|
11614
|
+
probe.key,
|
|
11615
|
+
probe.version === void 0 ? void 0 : { ifVersion: probe.version }
|
|
11616
|
+
);
|
|
11617
|
+
probe.found.set(
|
|
11618
|
+
"delete",
|
|
11619
|
+
wrote ? { status: "ok", detail: "The object was deleted." } : {
|
|
11620
|
+
status: "unchecked",
|
|
11621
|
+
detail: "The write failed, so this check was not made. The run deleted its key in case the write had landed; that delete succeeded, which it also does when no object was there."
|
|
11622
|
+
}
|
|
11623
|
+
);
|
|
11624
|
+
} catch (err) {
|
|
11625
|
+
if (wrote) {
|
|
11626
|
+
probe.found.set("delete", {
|
|
11627
|
+
status: "problem",
|
|
11628
|
+
detail: `${failed("Deleting the object just written", err)} The object ${probe.object} was left behind: delete it by hand.`
|
|
11629
|
+
});
|
|
11630
|
+
return;
|
|
11631
|
+
}
|
|
11632
|
+
const write = probe.found.get("write");
|
|
11633
|
+
if (probe.refused) {
|
|
11634
|
+
probe.found.set("delete", {
|
|
11635
|
+
status: "unchecked",
|
|
11636
|
+
detail: `The write failed, so this check was not made. The run tried to delete its key in case the write had landed, and that failed with ${describeError(err)}. The write was most likely refused and stored nothing; a later run's \`list\` names any probe it finds.`
|
|
11637
|
+
});
|
|
11638
|
+
return;
|
|
11639
|
+
}
|
|
11640
|
+
probe.found.set("delete", {
|
|
11641
|
+
status: "unchecked",
|
|
11642
|
+
detail: `The write failed, so this check was not made. The run tried to delete its key in case the write had landed, and that failed with ${describeError(err)}.`
|
|
11643
|
+
});
|
|
11644
|
+
if (write !== void 0) {
|
|
11645
|
+
probe.found.set("write", {
|
|
11646
|
+
status: write.status,
|
|
11647
|
+
detail: `${write.detail} The write may have landed, and the delete of ${probe.object} failed, so it may have been left behind.`
|
|
11648
|
+
});
|
|
11649
|
+
}
|
|
11650
|
+
}
|
|
11651
|
+
}
|
|
11652
|
+
async function objectSteps(probe) {
|
|
11653
|
+
const { wrote, notOurs } = await writeStep(probe);
|
|
11654
|
+
try {
|
|
11655
|
+
if (wrote) {
|
|
11656
|
+
await writeOnceStep(probe);
|
|
11657
|
+
await readStep(probe);
|
|
11658
|
+
await listStep(probe);
|
|
11659
|
+
} else {
|
|
11660
|
+
const why = skipped("no object was written");
|
|
11661
|
+
for (const name of ["write-once", "read", "list"]) probe.found.set(name, why);
|
|
11662
|
+
}
|
|
11663
|
+
} finally {
|
|
11664
|
+
if (notOurs) {
|
|
11665
|
+
probe.found.set(
|
|
11666
|
+
"delete",
|
|
11667
|
+
skipped("the object under the key was not this run's, so it was left alone")
|
|
11668
|
+
);
|
|
11669
|
+
} else {
|
|
11670
|
+
await deleteStep(probe, wrote);
|
|
11671
|
+
}
|
|
11672
|
+
}
|
|
11673
|
+
}
|
|
11674
|
+
function drawKey(entropy) {
|
|
11675
|
+
try {
|
|
11676
|
+
return {
|
|
11677
|
+
namespace: PREFLIGHT_NAMESPACE,
|
|
11678
|
+
segment: PREFLIGHT_SEGMENT,
|
|
11679
|
+
generation: probeNumber(entropy)
|
|
11680
|
+
};
|
|
11681
|
+
} catch (error) {
|
|
11682
|
+
return { error };
|
|
11683
|
+
}
|
|
11684
|
+
}
|
|
11685
|
+
function noNumber(error) {
|
|
11686
|
+
const found2 = /* @__PURE__ */ new Map();
|
|
11687
|
+
found2.set("write", {
|
|
11688
|
+
status: "problem",
|
|
11689
|
+
detail: `No number could be drawn for the probe's name, so nothing was written: ${describeError(error)}. Give the store an entropy source (the runtime's Web Crypto, or the \`entropy\` seam).`
|
|
11690
|
+
});
|
|
11691
|
+
const why = skipped("no object was written");
|
|
11692
|
+
for (const name of ["write-once", "read", "list", "delete"]) found2.set(name, why);
|
|
11693
|
+
return found2;
|
|
11694
|
+
}
|
|
11695
|
+
async function objectFindings(deps, key) {
|
|
11696
|
+
const probe = {
|
|
11697
|
+
storage: deps.storage,
|
|
11698
|
+
key,
|
|
11699
|
+
object: `${PREFLIGHT_NAMESPACE}/segments/${PREFLIGHT_SEGMENT}.${key.generation}.crbm (under the store's prefix)`,
|
|
11700
|
+
found: /* @__PURE__ */ new Map(),
|
|
11701
|
+
accepted: MARKER_A,
|
|
11702
|
+
refused: false
|
|
11703
|
+
};
|
|
11704
|
+
try {
|
|
11705
|
+
await objectSteps(probe);
|
|
11706
|
+
} catch (err) {
|
|
11707
|
+
for (const name of ["write", "write-once", "read", "list", "delete"]) {
|
|
11708
|
+
if (!probe.found.has(name)) {
|
|
11709
|
+
probe.found.set(name, { status: "problem", detail: failed("The check", err) });
|
|
11710
|
+
}
|
|
11711
|
+
}
|
|
11712
|
+
}
|
|
11713
|
+
return probe.found;
|
|
11714
|
+
}
|
|
11715
|
+
async function runPreflight(deps) {
|
|
11716
|
+
const registry = await guarded(
|
|
11717
|
+
() => registryStep(deps.registry),
|
|
11718
|
+
(err) => ({ status: "problem", detail: failed("Reading the registry", err) })
|
|
11719
|
+
);
|
|
11720
|
+
const key = drawKey(deps.entropy);
|
|
11721
|
+
const found2 = "error" in key ? noNumber(key.error) : await objectFindings(deps, key);
|
|
11722
|
+
found2.set("registry", registry);
|
|
11723
|
+
found2.set(
|
|
11724
|
+
"lifecycle",
|
|
11725
|
+
await guarded(
|
|
11726
|
+
() => lifecycleStep(deps),
|
|
11727
|
+
(err) => ({
|
|
11728
|
+
status: "unchecked",
|
|
11729
|
+
detail: `${failed("The lifecycle check", err)} Look at the bucket lifecycle rules by hand.`
|
|
11730
|
+
})
|
|
11731
|
+
)
|
|
11732
|
+
);
|
|
11733
|
+
const checks = ORDER.map((name) => ({
|
|
11734
|
+
name,
|
|
11735
|
+
...found2.get(name) ?? skipped("the check did not run")
|
|
11736
|
+
}));
|
|
11737
|
+
return { ok: !checks.some((c) => c.status === "problem"), checks };
|
|
11738
|
+
}
|
|
11739
|
+
|
|
11697
11740
|
// src/core/consistency.ts
|
|
11698
11741
|
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
11699
11742
|
var NOTHING = { issues: [] };
|
|
@@ -11838,7 +11881,7 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
11838
11881
|
if (options.generations !== void 0 && typeof options.generations !== "boolean") {
|
|
11839
11882
|
throw new ValidationError(`generations must be a boolean; got ${String(options.generations)}`);
|
|
11840
11883
|
}
|
|
11841
|
-
const concurrency = options.concurrency
|
|
11884
|
+
const concurrency = options.concurrency === void 0 ? DEFAULT_CHECK_CONCURRENCY : options.concurrency;
|
|
11842
11885
|
requirePositiveInteger(concurrency, "concurrency");
|
|
11843
11886
|
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11844
11887
|
const recs = await drainRegistry(deps.registry, {
|
|
@@ -11997,19 +12040,15 @@ export {
|
|
|
11997
12040
|
CrbmStorageChunkSource,
|
|
11998
12041
|
DEFAULT_BUDGET,
|
|
11999
12042
|
DEFAULT_RETRY_POLICY,
|
|
12000
|
-
InProcessKeystore,
|
|
12001
12043
|
IntegrityError,
|
|
12002
12044
|
KeyUnavailableError,
|
|
12003
12045
|
LEASE_SKEW_MS,
|
|
12004
12046
|
LeaseExpiredError,
|
|
12005
12047
|
LeaseLimitError,
|
|
12006
|
-
LocalFsStorage,
|
|
12007
12048
|
MAX_LEASES_PER_SEGMENT,
|
|
12008
12049
|
MAX_LEASE_MS,
|
|
12009
12050
|
MIN_EXPIRES_AT_MS,
|
|
12010
|
-
MemoryStorage,
|
|
12011
12051
|
NOOP_METRICS,
|
|
12012
|
-
NodeAead,
|
|
12013
12052
|
NotFoundError,
|
|
12014
12053
|
PinnedStorageChunkSource,
|
|
12015
12054
|
RecordingAuditSink,
|
|
@@ -12032,6 +12071,7 @@ export {
|
|
|
12032
12071
|
eraseNamespace,
|
|
12033
12072
|
errorKind,
|
|
12034
12073
|
excludingReservedRows,
|
|
12074
|
+
findIdUnderTombstone,
|
|
12035
12075
|
getSegmentRetention,
|
|
12036
12076
|
isStorageBackend,
|
|
12037
12077
|
judgeLoad,
|
|
@@ -12042,6 +12082,7 @@ export {
|
|
|
12042
12082
|
mapWithConcurrency,
|
|
12043
12083
|
readRetentionPolicy,
|
|
12044
12084
|
rebindCombineMany,
|
|
12085
|
+
refuseUnknownRefKeys,
|
|
12045
12086
|
releaseLease,
|
|
12046
12087
|
resolveBudget,
|
|
12047
12088
|
resolvePerOpBudget,
|
|
@@ -12050,6 +12091,7 @@ export {
|
|
|
12050
12091
|
runCombineMany,
|
|
12051
12092
|
runConsistencyCheck,
|
|
12052
12093
|
runExport,
|
|
12094
|
+
runPreflight,
|
|
12053
12095
|
safeMetrics,
|
|
12054
12096
|
segmentExists,
|
|
12055
12097
|
segmentKey,
|