@cloudbitmaps/core 0.18.3 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-ML6AT53O.js → chunk-RSLVTRJY.js} +50 -9
- package/dist/chunk-RSLVTRJY.js.map +7 -0
- package/dist/core/audit.d.ts +13 -3
- package/dist/core/bit-route.d.ts +5 -5
- package/dist/core/blob.d.ts +4 -1
- package/dist/core/budget.d.ts +7 -2
- package/dist/core/combine-feed.d.ts +3 -7
- package/dist/core/combine-many.d.ts +28 -29
- package/dist/core/consistency.d.ts +0 -1
- package/dist/core/cost.d.ts +13 -2
- package/dist/core/crbm/reader.d.ts +9 -3
- package/dist/core/crbm-storage-source.d.ts +115 -29
- package/dist/core/discover.d.ts +3 -5
- package/dist/core/engine.d.ts +20 -0
- package/dist/core/erase-id.d.ts +8 -5
- package/dist/core/erasure.d.ts +22 -2
- package/dist/core/errors.d.ts +10 -2
- package/dist/core/generation-gc.d.ts +2 -2
- package/dist/core/keys.d.ts +28 -0
- package/dist/core/load-input.d.ts +1 -2
- package/dist/core/load.d.ts +54 -7
- package/dist/core/metrics.d.ts +19 -7
- package/dist/core/ports.d.ts +25 -62
- package/dist/core/retention-sweep.d.ts +9 -4
- package/dist/core/retention.d.ts +17 -0
- package/dist/core/retry.d.ts +6 -0
- package/dist/driver-kit.d.ts +1 -1
- package/dist/driver-kit.js +8 -101
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/object-registry.d.ts +1 -19
- package/dist/drivers/_shared/registry.d.ts +5 -2
- package/dist/drivers/localfs/fs-util.d.ts +7 -0
- package/dist/drivers/localfs/registry.d.ts +1 -0
- package/dist/index.d.ts +5 -9
- package/dist/index.js +975 -631
- package/dist/index.js.map +4 -4
- package/package.json +1 -1
- package/dist/chunk-ML6AT53O.js.map +0 -7
- package/dist/core/combine-held.d.ts +0 -43
- package/dist/core/registry-reap.d.ts +0 -22
package/dist/index.js
CHANGED
|
@@ -34,8 +34,11 @@ import {
|
|
|
34
34
|
entropyIsAvailable,
|
|
35
35
|
heldGenerations,
|
|
36
36
|
incarnationOf,
|
|
37
|
+
isBudgetExceededError,
|
|
38
|
+
isCapabilityError,
|
|
37
39
|
isCloudRoaringError,
|
|
38
40
|
isIntegrityError,
|
|
41
|
+
isKeyUnavailableError,
|
|
39
42
|
isLeaseExpiredError,
|
|
40
43
|
isLeaseLimitError,
|
|
41
44
|
isLive,
|
|
@@ -44,6 +47,7 @@ import {
|
|
|
44
47
|
isStaleOperandError,
|
|
45
48
|
isStorageBackend,
|
|
46
49
|
isTransientError,
|
|
50
|
+
isUnsupportedError,
|
|
47
51
|
isValidationError,
|
|
48
52
|
isWriteConflictError,
|
|
49
53
|
leaseChurn,
|
|
@@ -70,7 +74,7 @@ import {
|
|
|
70
74
|
validateUserNamespace,
|
|
71
75
|
validateUserRef,
|
|
72
76
|
webCryptoEntropy
|
|
73
|
-
} from "./chunk-
|
|
77
|
+
} from "./chunk-RSLVTRJY.js";
|
|
74
78
|
|
|
75
79
|
// src/core/bit-route.ts
|
|
76
80
|
var U32_MAX = 4294967295;
|
|
@@ -91,6 +95,20 @@ function joinId(chunkKey, remainder) {
|
|
|
91
95
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
92
96
|
function resolveBudget(opt, fallback) {
|
|
93
97
|
if (opt === false) return null;
|
|
98
|
+
if (opt !== void 0 && opt !== null) {
|
|
99
|
+
if (typeof opt !== "object" || Array.isArray(opt)) {
|
|
100
|
+
const got = Array.isArray(opt) ? "an array" : `a ${typeof opt}`;
|
|
101
|
+
throw new ValidationError(`budget must be { maxRequests } or false; got ${got}`);
|
|
102
|
+
}
|
|
103
|
+
const unknown = Object.keys(opt).filter(
|
|
104
|
+
(k) => k !== "maxRequests" && opt[k] !== void 0
|
|
105
|
+
);
|
|
106
|
+
if (unknown.length > 0) {
|
|
107
|
+
throw new ValidationError(
|
|
108
|
+
`budget: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { maxRequests } or false`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
94
112
|
const maxRequests = opt?.maxRequests ?? fallback.maxRequests;
|
|
95
113
|
if (!Number.isInteger(maxRequests) || maxRequests < 1) {
|
|
96
114
|
throw new ValidationError(`budget.maxRequests must be a positive integer; got ${maxRequests}`);
|
|
@@ -98,7 +116,7 @@ function resolveBudget(opt, fallback) {
|
|
|
98
116
|
return { maxRequests };
|
|
99
117
|
}
|
|
100
118
|
function resolvePerOpBudget(opt, storeBudget) {
|
|
101
|
-
if (opt === void 0) return storeBudget;
|
|
119
|
+
if (opt === void 0 || opt === null) return storeBudget;
|
|
102
120
|
return resolveBudget(opt, storeBudget ?? DEFAULT_BUDGET);
|
|
103
121
|
}
|
|
104
122
|
function checkBudget(budget, projectedRequests, op) {
|
|
@@ -311,11 +329,42 @@ function segmentKey(ref) {
|
|
|
311
329
|
function segmentPrefix(ref) {
|
|
312
330
|
return `${segmentKey(ref)}${FIELD}`;
|
|
313
331
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
332
|
+
var KEPT_SEGMENT_KEYS = 1024;
|
|
333
|
+
var KeptSegmentKeys = class {
|
|
334
|
+
constructor(encode, max = KEPT_SEGMENT_KEYS) {
|
|
335
|
+
this.encode = encode;
|
|
336
|
+
this.max = max;
|
|
337
|
+
}
|
|
338
|
+
encode;
|
|
339
|
+
max;
|
|
340
|
+
byNamespace = /* @__PURE__ */ new Map();
|
|
341
|
+
kept = 0;
|
|
342
|
+
/** The segment's encoded key. */
|
|
343
|
+
of(ref) {
|
|
344
|
+
let names = this.byNamespace.get(ref.namespace);
|
|
345
|
+
const found = names?.get(ref.segment);
|
|
346
|
+
if (found !== void 0) return found;
|
|
347
|
+
const encoded = this.encode(ref);
|
|
348
|
+
if (this.kept >= this.max) {
|
|
349
|
+
this.byNamespace.clear();
|
|
350
|
+
this.kept = 0;
|
|
351
|
+
names = void 0;
|
|
352
|
+
}
|
|
353
|
+
if (names === void 0) {
|
|
354
|
+
names = /* @__PURE__ */ new Map();
|
|
355
|
+
this.byNamespace.set(ref.namespace, names);
|
|
356
|
+
}
|
|
357
|
+
names.set(ref.segment, encoded);
|
|
358
|
+
this.kept += 1;
|
|
359
|
+
return encoded;
|
|
360
|
+
}
|
|
361
|
+
/** How many encoded keys are kept. */
|
|
362
|
+
get size() {
|
|
363
|
+
return this.kept;
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
function chunkKeyUnder(prefix, chunkKey, version) {
|
|
367
|
+
return version === void 0 ? `${prefix}${chunkKey}` : `${prefix}${chunkKey}${FIELD}${version}`;
|
|
319
368
|
}
|
|
320
369
|
function shardOf(key, totalShards) {
|
|
321
370
|
let h = 2166136261;
|
|
@@ -326,6 +375,50 @@ function shardOf(key, totalShards) {
|
|
|
326
375
|
return (h >>> 0) % totalShards;
|
|
327
376
|
}
|
|
328
377
|
|
|
378
|
+
// src/core/audit.ts
|
|
379
|
+
var NOOP_AUDIT = {
|
|
380
|
+
onEvent() {
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
function safeAudit(sink) {
|
|
384
|
+
if (sink === NOOP_AUDIT) return sink;
|
|
385
|
+
return {
|
|
386
|
+
onEvent(event) {
|
|
387
|
+
try {
|
|
388
|
+
ignoreRejection(sink.onEvent(event));
|
|
389
|
+
} catch {
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
function checkedAuditSink(sink, op) {
|
|
395
|
+
if (sink === void 0 || sink === null) return;
|
|
396
|
+
if (typeof sink.onEvent !== "function") {
|
|
397
|
+
throw new ValidationError(
|
|
398
|
+
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
function ignoreRejection(returned) {
|
|
403
|
+
if (typeof returned?.then === "function") {
|
|
404
|
+
returned.then(void 0, () => {
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
var RecordingAuditSink = class {
|
|
409
|
+
recorded = [];
|
|
410
|
+
onEvent(event) {
|
|
411
|
+
this.recorded.push(event);
|
|
412
|
+
}
|
|
413
|
+
/** An independent copy of the recorded events, in emission order. */
|
|
414
|
+
snapshot() {
|
|
415
|
+
return this.recorded.map((e) => ({ ...e }));
|
|
416
|
+
}
|
|
417
|
+
reset() {
|
|
418
|
+
this.recorded.length = 0;
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
|
|
329
422
|
// src/core/metrics.ts
|
|
330
423
|
var NOOP_METRICS = {
|
|
331
424
|
onEvent() {
|
|
@@ -333,10 +426,15 @@ var NOOP_METRICS = {
|
|
|
333
426
|
};
|
|
334
427
|
function safeMetrics(sink) {
|
|
335
428
|
if (sink === NOOP_METRICS) return sink;
|
|
429
|
+
if (typeof sink?.onEvent !== "function") {
|
|
430
|
+
throw new ValidationError(
|
|
431
|
+
"metrics must be a sink with an onEvent(event) method, such as a CountingMetricsSink"
|
|
432
|
+
);
|
|
433
|
+
}
|
|
336
434
|
return {
|
|
337
435
|
onEvent(event) {
|
|
338
436
|
try {
|
|
339
|
-
sink.onEvent(event);
|
|
437
|
+
ignoreRejection(sink.onEvent(event));
|
|
340
438
|
} catch {
|
|
341
439
|
}
|
|
342
440
|
}
|
|
@@ -347,7 +445,8 @@ var OP_NAMES = [
|
|
|
347
445
|
"count",
|
|
348
446
|
"intersectInto",
|
|
349
447
|
"unionInto",
|
|
350
|
-
"andNotInto"
|
|
448
|
+
"andNotInto",
|
|
449
|
+
"materializeMany"
|
|
351
450
|
];
|
|
352
451
|
var CountingMetricsSink = class _CountingMetricsSink {
|
|
353
452
|
storage = { gets: 0, bytes: 0, totalMs: 0 };
|
|
@@ -551,6 +650,11 @@ var SegmentEngine = class {
|
|
|
551
650
|
openStreams = /* @__PURE__ */ new Set();
|
|
552
651
|
/** How many times {@link invalidate} has been called: a read compares it with the count it began under. */
|
|
553
652
|
invalidations = 0;
|
|
653
|
+
/**
|
|
654
|
+
* The cache-key prefixes ({@link segmentPrefix}) of the segments this engine reads, kept rather than encoded for each
|
|
655
|
+
* chunk. A kept prefix is the one {@link invalidate} finds the segment's chunks by.
|
|
656
|
+
*/
|
|
657
|
+
prefixes = new KeptSegmentKeys(segmentPrefix);
|
|
554
658
|
constructor(deps) {
|
|
555
659
|
this.storage = deps.storage;
|
|
556
660
|
this.cache = deps.cache;
|
|
@@ -586,6 +690,7 @@ var SegmentEngine = class {
|
|
|
586
690
|
let total2 = 0;
|
|
587
691
|
for (const [k, n] of cardinalities) {
|
|
588
692
|
assertChunkKeyInRange(k);
|
|
693
|
+
if (n !== 0) assertChunkCardinalityInRange(n);
|
|
589
694
|
total2 += n;
|
|
590
695
|
}
|
|
591
696
|
return total2;
|
|
@@ -625,7 +730,7 @@ var SegmentEngine = class {
|
|
|
625
730
|
chunkWindow(seg, chunkKeys, gen, ramp) {
|
|
626
731
|
return new ChunkWindow(
|
|
627
732
|
chunkKeys,
|
|
628
|
-
(chunkKey) => this.storageChunk({ ...seg, chunkKey }, gen),
|
|
733
|
+
(chunkKey) => this.storageChunk({ ...seg, chunkKey }, gen, true),
|
|
629
734
|
DEFAULT_INTERSECT_CONCURRENCY,
|
|
630
735
|
ramp
|
|
631
736
|
);
|
|
@@ -1092,7 +1197,9 @@ var SegmentEngine = class {
|
|
|
1092
1197
|
}
|
|
1093
1198
|
if (acc.isEmpty) return null;
|
|
1094
1199
|
if (relevant.length > 0) {
|
|
1095
|
-
const cuts = together ? fetched.slice(present.length) : await Promise.all(
|
|
1200
|
+
const cuts = together ? fetched.slice(present.length) : await Promise.all(
|
|
1201
|
+
relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen, true))
|
|
1202
|
+
);
|
|
1096
1203
|
for (const cut of cuts) {
|
|
1097
1204
|
if (cut === null) continue;
|
|
1098
1205
|
acc.andNotInPlace(cut);
|
|
@@ -1106,7 +1213,7 @@ var SegmentEngine = class {
|
|
|
1106
1213
|
* so a stream's take is made before anything is awaited: concurrent takes of one stream must be made in key order.
|
|
1107
1214
|
*/
|
|
1108
1215
|
operandChunk(op, chunkKey) {
|
|
1109
|
-
return op.streamed === void 0 ? this.storageChunk({ ...op.seg, chunkKey }, op.gen) : this.streamedChunk(op.streamed, chunkKey);
|
|
1216
|
+
return op.streamed === void 0 ? this.storageChunk({ ...op.seg, chunkKey }, op.gen, true) : this.streamedChunk(op.streamed, chunkKey);
|
|
1110
1217
|
}
|
|
1111
1218
|
/**
|
|
1112
1219
|
* `keys` of `seg` as a stream of coalesced ranges, or `undefined` when the read is per key: a source with no
|
|
@@ -1180,15 +1287,16 @@ var SegmentEngine = class {
|
|
|
1180
1287
|
if (!streamed.opened) {
|
|
1181
1288
|
const cache = this.cache;
|
|
1182
1289
|
if (cache) {
|
|
1183
|
-
const
|
|
1184
|
-
|
|
1185
|
-
if (hit) return
|
|
1290
|
+
const ref = { ...streamed.seg, chunkKey };
|
|
1291
|
+
const hit = cache.get(this.chunkCacheKey(ref, streamed.gen));
|
|
1292
|
+
if (hit) return this.cachedIfCurrent(ref, streamed.gen, hit, true, streamed);
|
|
1293
|
+
if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
1186
1294
|
}
|
|
1187
1295
|
this.startStream(streamed, chunkKey);
|
|
1188
1296
|
} else if (streamed.inStream !== void 0 && !streamed.inStream.has(chunkKey)) {
|
|
1189
1297
|
const ref = { ...streamed.seg, chunkKey };
|
|
1190
1298
|
const hit = this.cache?.get(this.chunkCacheKey(ref, streamed.gen));
|
|
1191
|
-
return hit ?
|
|
1299
|
+
return hit ? this.cachedIfCurrent(ref, streamed.gen, hit, false, streamed) : this.storageChunk(ref, streamed.gen, true, false);
|
|
1192
1300
|
}
|
|
1193
1301
|
return streamed.stream.take(chunkKey).then((read) => {
|
|
1194
1302
|
if (read.bytes === null) return null;
|
|
@@ -1210,9 +1318,31 @@ var SegmentEngine = class {
|
|
|
1210
1318
|
streamedVersion(planned, read) {
|
|
1211
1319
|
return planned === void 0 || this.storage.currentVersion === void 0 ? planned : read;
|
|
1212
1320
|
}
|
|
1321
|
+
/**
|
|
1322
|
+
* `hit`, a chunk cached under the version a multi-chunk read planned under, if that is still the segment's version;
|
|
1323
|
+
* else the chunk as it is now. A read resolves the segment again before it serves each chunk, which the source's
|
|
1324
|
+
* stream does for the chunks it delivers; a chunk served from the cache is checked here, or a read whose chunks are
|
|
1325
|
+
* all cached would go on serving the generation it planned under, an erased id included, for as long as it is pulled.
|
|
1326
|
+
* Within `cache.genTtlMs` the source answers from its snapshot, so the check is a lookup, not a request. `report`:
|
|
1327
|
+
* emit the `cache` event for this lookup, which a key counted when its stream opened does not. `streamed`: the read
|
|
1328
|
+
* this chunk belongs to, which a move carries to the version now current, so the rest of it looks there: a read whose
|
|
1329
|
+
* stream is not yet open reads the rest as one stream, not a chunk at a time.
|
|
1330
|
+
*/
|
|
1331
|
+
async cachedIfCurrent(ref, planned, hit, report, streamed) {
|
|
1332
|
+
const now = planned === void 0 ? void 0 : await this.cacheVersion(ref);
|
|
1333
|
+
if (now === planned) {
|
|
1334
|
+
if (report && this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: true });
|
|
1335
|
+
return hit;
|
|
1336
|
+
}
|
|
1337
|
+
if (streamed !== void 0 && streamed.gen === planned && now !== null) {
|
|
1338
|
+
streamed.gen = now;
|
|
1339
|
+
if (streamed.opened && this.storage.currentVersion === void 0) streamed.invalidated = true;
|
|
1340
|
+
}
|
|
1341
|
+
return this.storageChunk(ref, now, false, report);
|
|
1342
|
+
}
|
|
1213
1343
|
/** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
|
|
1214
1344
|
chunkCacheKey(ref, version) {
|
|
1215
|
-
return
|
|
1345
|
+
return chunkKeyUnder(this.prefixes.of(ref), ref.chunkKey, version);
|
|
1216
1346
|
}
|
|
1217
1347
|
/** The segment's chunk keys, ascending, checked as untrusted tier data. */
|
|
1218
1348
|
async chunkKeys(seg) {
|
|
@@ -1249,7 +1379,7 @@ var SegmentEngine = class {
|
|
|
1249
1379
|
if (absent.length === 0) return;
|
|
1250
1380
|
const named = absent.map((s) => s.namespace === void 0 ? `"${s.segment}"` : `"${s.namespace}/${s.segment}"`).join(", ");
|
|
1251
1381
|
throw new ValidationError(
|
|
1252
|
-
`${op}: operand ${named} names a segment that does not exist, so it would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
1382
|
+
`${op}: operand ${named} names a segment that does not exist or was dropped, so it would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
1253
1383
|
);
|
|
1254
1384
|
}
|
|
1255
1385
|
/**
|
|
@@ -1307,17 +1437,22 @@ var SegmentEngine = class {
|
|
|
1307
1437
|
*
|
|
1308
1438
|
* The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
|
|
1309
1439
|
* one): callers read it or clone it, never mutate it.
|
|
1440
|
+
*
|
|
1441
|
+
* `recheck`: `gen` was planned for a read of many chunks, so a cached chunk is served only while it is still the
|
|
1442
|
+
* segment's version ({@link cachedIfCurrent}). A point read resolves `gen` just before it asks, and passes none.
|
|
1443
|
+
* `report`: emit the `cache` event for this lookup; a key already counted for this read does not.
|
|
1310
1444
|
*/
|
|
1311
|
-
async storageChunk(ref, gen) {
|
|
1445
|
+
async storageChunk(ref, gen, recheck = false, report = true) {
|
|
1312
1446
|
if (gen === null) return null;
|
|
1313
1447
|
const cacheKey = this.chunkCacheKey(ref, gen);
|
|
1314
1448
|
if (this.cache) {
|
|
1315
1449
|
const cached = this.cache.get(cacheKey);
|
|
1316
1450
|
if (cached) {
|
|
1317
|
-
if (
|
|
1451
|
+
if (recheck) return this.cachedIfCurrent(ref, gen, cached, report);
|
|
1452
|
+
if (report && this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: true });
|
|
1318
1453
|
return cached;
|
|
1319
1454
|
}
|
|
1320
|
-
if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
1455
|
+
if (report && this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
1321
1456
|
}
|
|
1322
1457
|
const open4 = this.openReads.get(cacheKey);
|
|
1323
1458
|
if (open4) return open4.read;
|
|
@@ -1657,7 +1792,7 @@ async function reconcile(ref, deps, options) {
|
|
|
1657
1792
|
try {
|
|
1658
1793
|
await deps.registry.compareAndSwap(ref, token, { keptGens: present });
|
|
1659
1794
|
} catch (err) {
|
|
1660
|
-
if (!(err
|
|
1795
|
+
if (!isWriteConflictError(err)) throw err;
|
|
1661
1796
|
}
|
|
1662
1797
|
}
|
|
1663
1798
|
return deleted;
|
|
@@ -1741,11 +1876,11 @@ var STORE_LOAD_POINTER_READS = 2 + 2 / LIST_COLLECTION_CADENCE;
|
|
|
1741
1876
|
var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
|
|
1742
1877
|
var RETENTION_SWEEP_REQUESTS = deepFreeze({
|
|
1743
1878
|
conditionalDelete: {
|
|
1744
|
-
retirement: { reads:
|
|
1879
|
+
retirement: { reads: 7, writes: 2, deletes: 1 },
|
|
1745
1880
|
purge: { reads: 4, writes: 0, deletes: 2 }
|
|
1746
1881
|
},
|
|
1747
1882
|
tombstoning: {
|
|
1748
|
-
retirement: { reads:
|
|
1883
|
+
retirement: { reads: 6, writes: 2, deletes: 0 },
|
|
1749
1884
|
purge: { reads: 3, writes: 1, deletes: 0 }
|
|
1750
1885
|
}
|
|
1751
1886
|
});
|
|
@@ -1926,6 +2061,11 @@ function buildReport(input) {
|
|
|
1926
2061
|
const storedGiB = storageBytes / GIB;
|
|
1927
2062
|
const baseline = resolveBaseline(redis, storedGiB);
|
|
1928
2063
|
const baselineUSD = baseline.monthlyUSD;
|
|
2064
|
+
if ((input.workload.cacheHitRate ?? 0) > 1) {
|
|
2065
|
+
throw new ValidationError(
|
|
2066
|
+
`cacheHitRate must be a fraction from 0 to 1; got ${String(input.workload.cacheHitRate)}`
|
|
2067
|
+
);
|
|
2068
|
+
}
|
|
1929
2069
|
const cacheHitRate = clamp01(
|
|
1930
2070
|
requireFiniteNonNeg(input.workload.cacheHitRate ?? 0, "cacheHitRate")
|
|
1931
2071
|
);
|
|
@@ -2060,7 +2200,31 @@ function buildReport(input) {
|
|
|
2060
2200
|
assumptions: { cacheHitRate, pricingName: pricing.name, grounded, notes }
|
|
2061
2201
|
};
|
|
2062
2202
|
}
|
|
2203
|
+
function checkModelInputs(op, pricing, workload) {
|
|
2204
|
+
const storage = pricing !== null && typeof pricing === "object" ? pricing.storage : void 0;
|
|
2205
|
+
if (pricing !== void 0 && (storage === null || typeof storage !== "object")) {
|
|
2206
|
+
throw new ValidationError(`${op}: pricing must be a profile with a storage price list`);
|
|
2207
|
+
}
|
|
2208
|
+
if (workload !== void 0 && workload !== null && (typeof workload !== "object" || Array.isArray(workload))) {
|
|
2209
|
+
throw new ValidationError(`${op}: workload must be an object`);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2063
2212
|
function estimateCost(input) {
|
|
2213
|
+
if (input === null || typeof input !== "object") {
|
|
2214
|
+
throw new ValidationError("estimateCost: input must be an object such as { segments }");
|
|
2215
|
+
}
|
|
2216
|
+
if (!Array.isArray(input.segments)) {
|
|
2217
|
+
throw new ValidationError(
|
|
2218
|
+
"estimateCost: segments must be an array of { sizeBytes } or { cardinality }"
|
|
2219
|
+
);
|
|
2220
|
+
}
|
|
2221
|
+
for (let i = 0; i < input.segments.length; i++) {
|
|
2222
|
+
const spec = input.segments[i];
|
|
2223
|
+
if (spec === null || typeof spec !== "object") {
|
|
2224
|
+
throw new ValidationError(`estimateCost: segments[${i}] must be an object`);
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
checkModelInputs("estimateCost", input.pricing, input.workload);
|
|
2064
2228
|
const pricing = input.pricing ?? AWS_US_EAST_1_ONDEMAND;
|
|
2065
2229
|
const workload = input.workload ?? {};
|
|
2066
2230
|
let storageBytes = 0;
|
|
@@ -2075,6 +2239,7 @@ function estimateCost(input) {
|
|
|
2075
2239
|
return buildReport({ storageBytes, workload, pricing, grounded: false, sizesFromCardinality });
|
|
2076
2240
|
}
|
|
2077
2241
|
function groundedReport(input) {
|
|
2242
|
+
checkModelInputs("costReport", input.pricing, input.workload);
|
|
2078
2243
|
return buildReport({
|
|
2079
2244
|
storageBytes: input.storageBytes,
|
|
2080
2245
|
workload: input.workload ?? {},
|
|
@@ -2185,7 +2350,22 @@ function requireCodec(codec, api) {
|
|
|
2185
2350
|
var DEFAULT_NDJSON_BATCH_BYTES = 64 * 1024;
|
|
2186
2351
|
var errMessage = (err) => err instanceof Error ? err.message : String(err);
|
|
2187
2352
|
async function runExport(reader, registry, sink, options = {}) {
|
|
2353
|
+
if (sink === null || typeof sink !== "object" || typeof sink.open !== "function") {
|
|
2354
|
+
throw new ValidationError(
|
|
2355
|
+
"runExport: the sink must be an object with an open(ref, ext) method"
|
|
2356
|
+
);
|
|
2357
|
+
}
|
|
2358
|
+
options ??= {};
|
|
2188
2359
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
2360
|
+
if (options.format !== void 0 && options.format !== "roaring" && options.format !== "ndjson") {
|
|
2361
|
+
throw new ValidationError(`runExport: format must be 'roaring' or 'ndjson'`);
|
|
2362
|
+
}
|
|
2363
|
+
const batch = options.ndjsonBatchBytes;
|
|
2364
|
+
if (batch !== void 0 && (typeof batch !== "number" || !Number.isSafeInteger(batch) || batch < 1)) {
|
|
2365
|
+
throw new ValidationError(
|
|
2366
|
+
"runExport: ndjsonBatchBytes must be a whole number of bytes, 1 or more"
|
|
2367
|
+
);
|
|
2368
|
+
}
|
|
2189
2369
|
const format = options.format ?? "roaring";
|
|
2190
2370
|
const ext = format === "roaring" ? ".roaring" : ".ndjson";
|
|
2191
2371
|
const batchCap = options.ndjsonBatchBytes ?? DEFAULT_NDJSON_BATCH_BYTES;
|
|
@@ -2389,6 +2569,7 @@ var MemoryRegistryDriver = class {
|
|
|
2389
2569
|
return { token };
|
|
2390
2570
|
}
|
|
2391
2571
|
async *list(namespace) {
|
|
2572
|
+
if (namespace !== void 0) validateSegmentRef({ segment: "x", namespace });
|
|
2392
2573
|
for (const row of this.rows.values()) {
|
|
2393
2574
|
if (namespace === void 0 || row.namespace === namespace) yield structuredClone(row);
|
|
2394
2575
|
}
|
|
@@ -2579,6 +2760,18 @@ async function fsyncDir(dir) {
|
|
|
2579
2760
|
} catch {
|
|
2580
2761
|
}
|
|
2581
2762
|
}
|
|
2763
|
+
async function writeAll(handle, bytes) {
|
|
2764
|
+
let offset = 0;
|
|
2765
|
+
while (offset < bytes.length) {
|
|
2766
|
+
const { bytesWritten } = await handle.write(bytes, offset, bytes.length - offset);
|
|
2767
|
+
if (bytesWritten <= 0) {
|
|
2768
|
+
throw Object.assign(new Error("a write made no progress: the device is full"), {
|
|
2769
|
+
code: "ENOSPC"
|
|
2770
|
+
});
|
|
2771
|
+
}
|
|
2772
|
+
offset += bytesWritten;
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2582
2775
|
|
|
2583
2776
|
// src/drivers/localfs/exact-case.ts
|
|
2584
2777
|
function firstCaseDifference(expected, actual) {
|
|
@@ -2718,7 +2911,7 @@ var LocalFsStorageDriver = class {
|
|
|
2718
2911
|
let size = 0;
|
|
2719
2912
|
const sink = {
|
|
2720
2913
|
async write(bytes) {
|
|
2721
|
-
await handle
|
|
2914
|
+
await writeAll(handle, bytes);
|
|
2722
2915
|
hash.update(bytes);
|
|
2723
2916
|
size += bytes.length;
|
|
2724
2917
|
}
|
|
@@ -2830,8 +3023,8 @@ var LocalFsStorageDriver = class {
|
|
|
2830
3023
|
// src/drivers/localfs/registry.ts
|
|
2831
3024
|
import { constants as FS3 } from "node:fs";
|
|
2832
3025
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
2833
|
-
import { mkdir as mkdir2, open as open3, readdir as readdir3, realpath as realpath2, rename, unlink as unlink2 } from "node:fs/promises";
|
|
2834
|
-
import { basename, dirname as dirname3, join as join3, resolve as resolve2 } from "node:path";
|
|
3026
|
+
import { mkdir as mkdir2, open as open3, opendir, readdir as readdir3, realpath as realpath2, rename, unlink as unlink2 } from "node:fs/promises";
|
|
3027
|
+
import { basename, dirname as dirname3, join as join3, relative as relative2, resolve as resolve2 } from "node:path";
|
|
2835
3028
|
var DEFAULT_MAX_ROW_BYTES = 1 * 1024 * 1024;
|
|
2836
3029
|
var rowChains = /* @__PURE__ */ new Map();
|
|
2837
3030
|
async function rowLockKey(path) {
|
|
@@ -2868,7 +3061,7 @@ var LocalFsRegistryDriver = class {
|
|
|
2868
3061
|
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
2869
3062
|
}
|
|
2870
3063
|
async get(ref) {
|
|
2871
|
-
const env = await this.readRow(registryRowPath(this.root, ref));
|
|
3064
|
+
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
2872
3065
|
return env && !env.deleted ? env.record : null;
|
|
2873
3066
|
}
|
|
2874
3067
|
async create(ref, record) {
|
|
@@ -2876,7 +3069,7 @@ var LocalFsRegistryDriver = class {
|
|
|
2876
3069
|
const path = registryRowPath(this.root, ref);
|
|
2877
3070
|
assertRegistryNamesFit(ref);
|
|
2878
3071
|
return this.withRowLock(path, async () => {
|
|
2879
|
-
const current = await this.readRow(path);
|
|
3072
|
+
const current = await this.readRow(path, ref);
|
|
2880
3073
|
if (current !== null && !current.deleted) {
|
|
2881
3074
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
2882
3075
|
}
|
|
@@ -2894,7 +3087,7 @@ var LocalFsRegistryDriver = class {
|
|
|
2894
3087
|
const path = registryRowPath(this.root, ref);
|
|
2895
3088
|
assertRegistryNamesFit(ref);
|
|
2896
3089
|
return this.withRowLock(path, async () => {
|
|
2897
|
-
const current = await this.readRow(path);
|
|
3090
|
+
const current = await this.readRow(path, ref);
|
|
2898
3091
|
if (current === null || current.deleted || current.record.token !== expected) {
|
|
2899
3092
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
2900
3093
|
}
|
|
@@ -2911,17 +3104,18 @@ var LocalFsRegistryDriver = class {
|
|
|
2911
3104
|
for (const ns of await this.namespaceDirs(namespace)) {
|
|
2912
3105
|
const dir = registryDir(this.root, ns);
|
|
2913
3106
|
if (await this.exactCase.differs(dir)) continue;
|
|
2914
|
-
let
|
|
3107
|
+
let entries;
|
|
2915
3108
|
try {
|
|
2916
|
-
|
|
3109
|
+
entries = await opendir(dir);
|
|
2917
3110
|
} catch (err) {
|
|
2918
3111
|
if (isAbsent(err)) continue;
|
|
2919
3112
|
throw mapFsError(err);
|
|
2920
3113
|
}
|
|
2921
|
-
for (const
|
|
2922
|
-
const segment = parseRegistryRow(name);
|
|
3114
|
+
for await (const entry of entries) {
|
|
3115
|
+
const segment = parseRegistryRow(entry.name);
|
|
2923
3116
|
if (segment === null) continue;
|
|
2924
|
-
const
|
|
3117
|
+
const ref = { namespace: ns, segment };
|
|
3118
|
+
const env = await this.readRow(registryRowPath(this.root, ref), ref);
|
|
2925
3119
|
if (env && !env.deleted) yield env.record;
|
|
2926
3120
|
}
|
|
2927
3121
|
}
|
|
@@ -2929,7 +3123,7 @@ var LocalFsRegistryDriver = class {
|
|
|
2929
3123
|
async delete(ref, expected) {
|
|
2930
3124
|
const path = registryRowPath(this.root, ref);
|
|
2931
3125
|
return this.withRowLock(path, async () => {
|
|
2932
|
-
const current = await this.readRow(path);
|
|
3126
|
+
const current = await this.readRow(path, ref);
|
|
2933
3127
|
if (expected !== void 0) {
|
|
2934
3128
|
if (current === null || current.deleted || current.record.token !== expected) {
|
|
2935
3129
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
@@ -2971,7 +3165,8 @@ var LocalFsRegistryDriver = class {
|
|
|
2971
3165
|
}
|
|
2972
3166
|
return namespaces;
|
|
2973
3167
|
}
|
|
2974
|
-
|
|
3168
|
+
/** Read the row at `path`, which must be the row of `ref`: one that names another segment is refused. */
|
|
3169
|
+
async readRow(path, ref) {
|
|
2975
3170
|
let handle;
|
|
2976
3171
|
try {
|
|
2977
3172
|
if (await this.exactCase.differs(path)) return null;
|
|
@@ -2986,7 +3181,7 @@ var LocalFsRegistryDriver = class {
|
|
|
2986
3181
|
throw new IntegrityError(`registry row ${size}B exceeds cap ${DEFAULT_MAX_ROW_BYTES}B`);
|
|
2987
3182
|
}
|
|
2988
3183
|
const text = (await handle.readFile()).toString("utf8");
|
|
2989
|
-
return parseRegistryEnvelope(text, path);
|
|
3184
|
+
return parseRegistryEnvelope(text, relative2(this.root, path), ref);
|
|
2990
3185
|
} finally {
|
|
2991
3186
|
await handle.close();
|
|
2992
3187
|
}
|
|
@@ -3008,11 +3203,16 @@ var LocalFsRegistryDriver = class {
|
|
|
3008
3203
|
throw mapFsError(err);
|
|
3009
3204
|
}
|
|
3010
3205
|
try {
|
|
3011
|
-
await handle
|
|
3206
|
+
await writeAll(handle, out);
|
|
3012
3207
|
await handle.sync();
|
|
3013
|
-
}
|
|
3014
|
-
await handle.close()
|
|
3208
|
+
} catch (err) {
|
|
3209
|
+
await handle.close().catch(() => {
|
|
3210
|
+
});
|
|
3211
|
+
await unlink2(tmp).catch(() => {
|
|
3212
|
+
});
|
|
3213
|
+
throw mapFsError(err);
|
|
3015
3214
|
}
|
|
3215
|
+
await handle.close();
|
|
3016
3216
|
await rename(tmp, path).catch(async (err) => {
|
|
3017
3217
|
await unlink2(tmp).catch(() => {
|
|
3018
3218
|
});
|
|
@@ -3063,36 +3263,6 @@ var LocalFsStorage = class {
|
|
|
3063
3263
|
registry;
|
|
3064
3264
|
};
|
|
3065
3265
|
|
|
3066
|
-
// src/core/audit.ts
|
|
3067
|
-
var NOOP_AUDIT = {
|
|
3068
|
-
onEvent() {
|
|
3069
|
-
}
|
|
3070
|
-
};
|
|
3071
|
-
function safeAudit(sink) {
|
|
3072
|
-
if (sink === NOOP_AUDIT) return sink;
|
|
3073
|
-
return {
|
|
3074
|
-
onEvent(event) {
|
|
3075
|
-
try {
|
|
3076
|
-
sink.onEvent(event);
|
|
3077
|
-
} catch {
|
|
3078
|
-
}
|
|
3079
|
-
}
|
|
3080
|
-
};
|
|
3081
|
-
}
|
|
3082
|
-
var RecordingAuditSink = class {
|
|
3083
|
-
recorded = [];
|
|
3084
|
-
onEvent(event) {
|
|
3085
|
-
this.recorded.push(event);
|
|
3086
|
-
}
|
|
3087
|
-
/** An independent copy of the recorded events, in emission order. */
|
|
3088
|
-
snapshot() {
|
|
3089
|
-
return this.recorded.map((e) => ({ ...e }));
|
|
3090
|
-
}
|
|
3091
|
-
reset() {
|
|
3092
|
-
this.recorded.length = 0;
|
|
3093
|
-
}
|
|
3094
|
-
};
|
|
3095
|
-
|
|
3096
3266
|
// src/core/cooperative.ts
|
|
3097
3267
|
var YIELD_EVERY = 1024;
|
|
3098
3268
|
function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
@@ -3263,6 +3433,12 @@ function magicMatches(bytes, offset) {
|
|
|
3263
3433
|
}
|
|
3264
3434
|
var sizePart = (size) => `${size}:`;
|
|
3265
3435
|
var fingerprintFor = (size, footerCrc) => `${sizePart(size)}${footerCrc}`;
|
|
3436
|
+
function compactFingerprint(fingerprint) {
|
|
3437
|
+
const colon = fingerprint.indexOf(":");
|
|
3438
|
+
const size = Number(fingerprint.slice(0, colon));
|
|
3439
|
+
const crc = Number(fingerprint.slice(colon + 1));
|
|
3440
|
+
return `${size.toString(36)}.${crc.toString(36)}`;
|
|
3441
|
+
}
|
|
3266
3442
|
function checkSize(tail, size) {
|
|
3267
3443
|
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3268
3444
|
throw new IntegrityError(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
@@ -3593,8 +3769,8 @@ var CrbmReader = class _CrbmReader {
|
|
|
3593
3769
|
/**
|
|
3594
3770
|
* Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
|
|
3595
3771
|
* absent. Throws `IntegrityError` on a CRC mismatch (the bytes must never reach the native
|
|
3596
|
-
* deserializer). The returned
|
|
3597
|
-
* not
|
|
3772
|
+
* deserializer). The returned bytes may be shared (a small generation's chunks are kept by the reader and handed out
|
|
3773
|
+
* again), so callers must not change them.
|
|
3598
3774
|
*/
|
|
3599
3775
|
async getChunk(chunkKey) {
|
|
3600
3776
|
const slot = this.slotOf(chunkKey);
|
|
@@ -3682,7 +3858,8 @@ var CrbmReader = class _CrbmReader {
|
|
|
3682
3858
|
* others; `options.now` times each request for `options.onRequest`, which hears of every range request that was sent, once, when
|
|
3683
3859
|
* it settles: taken, abandoned after the consumer stopped, or failed (a reader has no clock of its own, so `ms` is 0
|
|
3684
3860
|
* without one).
|
|
3685
|
-
* A plain chunk is a
|
|
3861
|
+
* A plain chunk is a view into the range it was read in, so keeping it keeps that whole range: copy a chunk to keep it
|
|
3862
|
+
* alone. Callers must not change it, as with {@link getChunk}.
|
|
3686
3863
|
*
|
|
3687
3864
|
* A stream that fails (a range that errors, a chunk that fails its check) raises at once and sends nothing further:
|
|
3688
3865
|
* the ranges it still had in flight finish in the background, are not retried and never raise. Its `settled()`
|
|
@@ -3745,7 +3922,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3745
3922
|
moved = bytes.length;
|
|
3746
3923
|
if (bytes.length !== read.length) {
|
|
3747
3924
|
throw new IntegrityError(
|
|
3748
|
-
`.crbm range [${read.offset}, +${read.length})
|
|
3925
|
+
`.crbm range [${read.offset}, +${read.length}) answered ${bytes.length} bytes, not the ${read.length} asked for`
|
|
3749
3926
|
);
|
|
3750
3927
|
}
|
|
3751
3928
|
return { bytes };
|
|
@@ -3953,7 +4130,7 @@ var CrbmWriter = class {
|
|
|
3953
4130
|
constructor(sink, options) {
|
|
3954
4131
|
this.sink = sink;
|
|
3955
4132
|
this.options = options;
|
|
3956
|
-
if (!Number.
|
|
4133
|
+
if (!Number.isSafeInteger(options.generation) || options.generation < 0) {
|
|
3957
4134
|
throw new ValidationError(
|
|
3958
4135
|
`generation must be a non-negative integer; got ${options.generation}`
|
|
3959
4136
|
);
|
|
@@ -3995,6 +4172,11 @@ var CrbmWriter = class {
|
|
|
3995
4172
|
);
|
|
3996
4173
|
}
|
|
3997
4174
|
if (payload.length === 0) throw new ValidationError("chunk payload must be non-empty");
|
|
4175
|
+
if (payload.length > DEFAULT_MAX_BITMAP_BYTES) {
|
|
4176
|
+
throw new ValidationError(
|
|
4177
|
+
`chunk payload of ${payload.length} bytes is over the ${DEFAULT_MAX_BITMAP_BYTES}-byte cap a reader opens`
|
|
4178
|
+
);
|
|
4179
|
+
}
|
|
3998
4180
|
await this.ensurePreamble();
|
|
3999
4181
|
const crypto = this.options.crypto;
|
|
4000
4182
|
const stored = crypto ? framed(crypto.aead.seal(payload, crypto.aadFor(chunkKey))) : payload;
|
|
@@ -4190,14 +4372,14 @@ function overCap(what, length) {
|
|
|
4190
4372
|
`${what} is ${length} bytes, more than any canonical 32-bit bitmap serializes to (${MAX_SERIALIZED_LOAD_BYTES}). If it was serialized from a bitmap, call runOptimize() on the bitmap first.`
|
|
4191
4373
|
);
|
|
4192
4374
|
}
|
|
4193
|
-
function prepareLoadInput(input, codec
|
|
4375
|
+
function prepareLoadInput(input, codec) {
|
|
4194
4376
|
if (typeof input !== "object" || input === null) {
|
|
4195
|
-
throw new ValidationError(
|
|
4377
|
+
throw new ValidationError("a load takes ids, { serialized } or { bitmap }");
|
|
4196
4378
|
}
|
|
4197
4379
|
const tag = tagOf(input);
|
|
4198
4380
|
if (tag === "Uint8Array" || tag === "Uint8ClampedArray") {
|
|
4199
4381
|
throw new ValidationError(
|
|
4200
|
-
|
|
4382
|
+
`a load's ids cannot be a ${tag} or Buffer: each byte would be loaded as an id. Pass portable Roaring bytes as { serialized }, and ids as a Uint32Array or an array of numbers.`
|
|
4201
4383
|
);
|
|
4202
4384
|
}
|
|
4203
4385
|
if (Symbol.iterator in input || Symbol.asyncIterator in input) return input;
|
|
@@ -4220,7 +4402,7 @@ function prepareLoadInput(input, codec, subject = "a load") {
|
|
|
4220
4402
|
return new DecodedLoadInput(decodeSerialized(bitmap.serialize("portable"), codec, what));
|
|
4221
4403
|
}
|
|
4222
4404
|
throw new ValidationError(
|
|
4223
|
-
|
|
4405
|
+
`a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
|
|
4224
4406
|
);
|
|
4225
4407
|
}
|
|
4226
4408
|
function decodeSerialized(bytes, codec, what = "the buffer") {
|
|
@@ -4374,6 +4556,19 @@ var notThePinned = (ref, generation) => new NotFoundError(
|
|
|
4374
4556
|
function versionOf(generation, lineage) {
|
|
4375
4557
|
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
4376
4558
|
}
|
|
4559
|
+
var readerVersions = /* @__PURE__ */ new WeakMap();
|
|
4560
|
+
function versionOfReader(reader) {
|
|
4561
|
+
let version = readerVersions.get(reader);
|
|
4562
|
+
if (version === void 0) {
|
|
4563
|
+
version = `${versionOf(reader.generation, reader.lineage)}#${compactFingerprint(reader.fingerprint)}`;
|
|
4564
|
+
readerVersions.set(reader, version);
|
|
4565
|
+
}
|
|
4566
|
+
return version;
|
|
4567
|
+
}
|
|
4568
|
+
var OBJECT_SUFFIX = /#[0-9a-z]+\.[0-9a-z]+$/;
|
|
4569
|
+
function rowVersionOf(version) {
|
|
4570
|
+
return version.replace(OBJECT_SUFFIX, "");
|
|
4571
|
+
}
|
|
4377
4572
|
var PinMoved = class extends Error {
|
|
4378
4573
|
};
|
|
4379
4574
|
function summaryOfGeneration(generation, description) {
|
|
@@ -4395,10 +4590,17 @@ function once(fn) {
|
|
|
4395
4590
|
};
|
|
4396
4591
|
}
|
|
4397
4592
|
var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
|
|
4593
|
+
async function readsObjectOf(snap, reader) {
|
|
4594
|
+
const settled2 = snap.settled;
|
|
4595
|
+
if (settled2 !== void 0) {
|
|
4596
|
+
const other = settled2.reader;
|
|
4597
|
+
return other !== null && (other === reader || versionOfReader(other) === versionOfReader(reader));
|
|
4598
|
+
}
|
|
4599
|
+
const live = await snap.target;
|
|
4600
|
+
return live !== null && versionOf(live.target.generation, live.target.lineage) === versionOf(reader.generation, reader.lineage);
|
|
4601
|
+
}
|
|
4398
4602
|
function describe(reader) {
|
|
4399
|
-
|
|
4400
|
-
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
4401
|
-
return { cardinality, metadata: reader.metadata };
|
|
4603
|
+
return { cardinality: reader.count(), metadata: reader.metadata };
|
|
4402
4604
|
}
|
|
4403
4605
|
var Snapshot = class _Snapshot {
|
|
4404
4606
|
constructor(target, open4, prior) {
|
|
@@ -4414,32 +4616,57 @@ var Snapshot = class _Snapshot {
|
|
|
4414
4616
|
/** Set by the cache, to hear of a reader as it is opened. */
|
|
4415
4617
|
onReader;
|
|
4416
4618
|
opened;
|
|
4619
|
+
outcome;
|
|
4417
4620
|
static live(target, open4, prior) {
|
|
4418
4621
|
return new _Snapshot(target, open4, prior);
|
|
4419
4622
|
}
|
|
4420
4623
|
static eager(reader) {
|
|
4421
4624
|
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
4422
|
-
snap.
|
|
4625
|
+
snap.hold(reader);
|
|
4423
4626
|
return snap;
|
|
4424
4627
|
}
|
|
4425
4628
|
/** The reader, opened by the first call. */
|
|
4426
4629
|
get reader() {
|
|
4427
4630
|
if (this.opened === void 0) {
|
|
4428
4631
|
const target = this.target;
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4632
|
+
const opened = this.hold(
|
|
4633
|
+
target.then(async (live) => {
|
|
4634
|
+
if (live === null) return null;
|
|
4635
|
+
const kept = await this.reuse(live);
|
|
4636
|
+
this.prior = void 0;
|
|
4637
|
+
return kept ?? this.open(live);
|
|
4638
|
+
})
|
|
4639
|
+
);
|
|
4640
|
+
this.onReader?.(opened);
|
|
4641
|
+
return opened;
|
|
4436
4642
|
}
|
|
4437
4643
|
return this.opened;
|
|
4438
4644
|
}
|
|
4645
|
+
/**
|
|
4646
|
+
* Take `reader` as this snapshot's, and note the reader once it has opened. A failed open is not noted: the reader
|
|
4647
|
+
* cache forgets the snapshot as the failure lands, and whoever awaits the reader meets the error.
|
|
4648
|
+
*/
|
|
4649
|
+
hold(reader) {
|
|
4650
|
+
this.opened = reader;
|
|
4651
|
+
reader.then(
|
|
4652
|
+
(r) => {
|
|
4653
|
+
this.outcome = { reader: r };
|
|
4654
|
+
},
|
|
4655
|
+
() => void 0
|
|
4656
|
+
);
|
|
4657
|
+
return reader;
|
|
4658
|
+
}
|
|
4439
4659
|
/** The reader if one has been asked for, without asking for one. */
|
|
4440
4660
|
get openedReader() {
|
|
4441
4661
|
return this.opened;
|
|
4442
4662
|
}
|
|
4663
|
+
/**
|
|
4664
|
+
* The reader once it has opened (`null` for no generation). Undefined while no reader has been asked for, while the
|
|
4665
|
+
* open is under way, and after an open that failed.
|
|
4666
|
+
*/
|
|
4667
|
+
get settled() {
|
|
4668
|
+
return this.outcome;
|
|
4669
|
+
}
|
|
4443
4670
|
/** The prior snapshot's reader, if it is of what this snapshot resolved. */
|
|
4444
4671
|
async reuse(live) {
|
|
4445
4672
|
const prior = this.prior;
|
|
@@ -4452,8 +4679,7 @@ var Snapshot = class _Snapshot {
|
|
|
4452
4679
|
const reader = await this.reuse(live);
|
|
4453
4680
|
this.prior = void 0;
|
|
4454
4681
|
if (reader !== null && this.opened === void 0) {
|
|
4455
|
-
this.
|
|
4456
|
-
this.onReader?.(this.opened);
|
|
4682
|
+
this.onReader?.(this.hold(Promise.resolve(reader)));
|
|
4457
4683
|
}
|
|
4458
4684
|
}
|
|
4459
4685
|
};
|
|
@@ -4488,8 +4714,13 @@ var CrbmStorageChunkSource = class {
|
|
|
4488
4714
|
this.keystore = keystore;
|
|
4489
4715
|
this.requireEncryption = requireEncryption ?? false;
|
|
4490
4716
|
this.readerOptions = readerOptions;
|
|
4491
|
-
|
|
4492
|
-
|
|
4717
|
+
if (currentGenTtlMs !== void 0 && currentGenTtlMs !== null && (typeof currentGenTtlMs !== "number" || !Number.isFinite(currentGenTtlMs) || currentGenTtlMs < 0)) {
|
|
4718
|
+
throw new ValidationError(
|
|
4719
|
+
`cache.genTtlMs must be a finite number of milliseconds, 0 or more (0: no timed refresh); got ${typeof currentGenTtlMs === "number" ? String(currentGenTtlMs) : `a ${typeof currentGenTtlMs}`}`
|
|
4720
|
+
);
|
|
4721
|
+
}
|
|
4722
|
+
this.timedRefresh = registry !== void 0 && clock !== void 0 && (currentGenTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS) > 0;
|
|
4723
|
+
this.keepChunkBytesUpTo = this.timedRefresh ? Math.floor(
|
|
4493
4724
|
(maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES) / (maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS)
|
|
4494
4725
|
) : 0;
|
|
4495
4726
|
this.clock = clock;
|
|
@@ -4514,7 +4745,8 @@ var CrbmStorageChunkSource = class {
|
|
|
4514
4745
|
* needs a clock). Within the TTL a segment's Storage bytes are treated as an immutable snapshot; when the TTL
|
|
4515
4746
|
* elapses the next read cheaply re-resolves `currentGen` and, only if it advanced (a load published),
|
|
4516
4747
|
* opens the new generation — so a long-lived source observes new generations within the TTL. The engine pairs
|
|
4517
|
-
* this with a
|
|
4748
|
+
* this with a cache keyed by {@link currentVersion}, which names the object a reader opened, so a bump never serves
|
|
4749
|
+
* a stale decoded chunk, and neither does a number taken again by another object. Without a clock or a
|
|
4518
4750
|
* registry there is no timed refresh, and a segment's reader is replaced only when it is evicted, when a read
|
|
4519
4751
|
* has to fetch from a generation a sweep deleted, or when {@link invalidate} drops it. A segment with no generation yet is not memoized, so
|
|
4520
4752
|
* it's re-checked until one exists. **Bounded** by a {@link BoundedLru} ({@link CrbmStorageChunkSourceOptions.maxOpenSegments}, default
|
|
@@ -4559,12 +4791,27 @@ var CrbmStorageChunkSource = class {
|
|
|
4559
4791
|
keepChunkBytesUpTo;
|
|
4560
4792
|
clock;
|
|
4561
4793
|
currentGenTtlMs;
|
|
4794
|
+
/**
|
|
4795
|
+
* Whether the segment's pointer is re-read on a timer: it needs a clock (the TTL), a registry (the cheap row read; with
|
|
4796
|
+
* none, resolving is a listing of the bucket) and a TTL above 0. It decides the keeping of chunk bytes, the TTL lapse
|
|
4797
|
+
* and the refresh interval the cost report prices, so it is decided once.
|
|
4798
|
+
*/
|
|
4799
|
+
timedRefresh;
|
|
4800
|
+
/**
|
|
4801
|
+
* The keys of the segments this source reads, kept rather than encoded again: a read asks for its segment's snapshot
|
|
4802
|
+
* once per operand, and a stream before every cached chunk it serves, and encoding the names cost about a
|
|
4803
|
+
* microsecond each time.
|
|
4804
|
+
*/
|
|
4805
|
+
keys = new KeptSegmentKeys(segmentKey);
|
|
4806
|
+
keyOf(ref) {
|
|
4807
|
+
return this.keys.of(ref);
|
|
4808
|
+
}
|
|
4562
4809
|
/**
|
|
4563
4810
|
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
|
|
4564
4811
|
* target, and its reader once a read has asked for one.
|
|
4565
4812
|
*/
|
|
4566
4813
|
liveSnapshot(ref) {
|
|
4567
|
-
const key =
|
|
4814
|
+
const key = this.keyOf(ref);
|
|
4568
4815
|
const existing = this.snapshots.get(key);
|
|
4569
4816
|
if (existing === void 0) {
|
|
4570
4817
|
return this.install(
|
|
@@ -4656,10 +4903,19 @@ var CrbmStorageChunkSource = class {
|
|
|
4656
4903
|
* this.
|
|
4657
4904
|
*/
|
|
4658
4905
|
get pointerRefreshMs() {
|
|
4659
|
-
return this.
|
|
4906
|
+
return this.timedRefresh ? this.currentGenTtlMs : 0;
|
|
4907
|
+
}
|
|
4908
|
+
/**
|
|
4909
|
+
* Whether a stream reading from `snap` must resolve its segment again before its next chunk, as a read of one chunk
|
|
4910
|
+
* would: the snapshot's TTL has lapsed, an invalidation has happened since `epoch`, or the snapshot is no longer the
|
|
4911
|
+
* one the reader cache holds for the segment (it let it go, or another read installed a newer one). Else nothing can
|
|
4912
|
+
* have moved it, and the check costs a compare and a lookup.
|
|
4913
|
+
*/
|
|
4914
|
+
mayHaveMoved(key, snap, epoch) {
|
|
4915
|
+
return this.invalidations !== epoch || this.expired(snap.installedAtMs) || this.snapshots.get(key) !== snap;
|
|
4660
4916
|
}
|
|
4661
4917
|
expired(installedAtMs) {
|
|
4662
|
-
return this.
|
|
4918
|
+
return this.timedRefresh && this.now() - installedAtMs >= this.currentGenTtlMs;
|
|
4663
4919
|
}
|
|
4664
4920
|
/**
|
|
4665
4921
|
* The segment's current generation number — the engine keys its chunk cache by this so a generation bump
|
|
@@ -4707,13 +4963,16 @@ var CrbmStorageChunkSource = class {
|
|
|
4707
4963
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
4708
4964
|
}
|
|
4709
4965
|
/**
|
|
4710
|
-
* `<generation>` for a registry-less source, or `<generation>:<row token>` with one
|
|
4711
|
-
*
|
|
4712
|
-
*
|
|
4713
|
-
*
|
|
4714
|
-
*
|
|
4715
|
-
*
|
|
4716
|
-
*
|
|
4966
|
+
* `<generation>#<object>` for a registry-less source, or `<generation>:<row token>#<object>` with one: the generation
|
|
4967
|
+
* the segment's snapshot resolved, and the object its reader opened, by that object's size and footer checksum in
|
|
4968
|
+
* base 36, which the open read anyway. The token moves on every row write, so an unrelated write (a `setRetention`)
|
|
4969
|
+
* costs the segment's decoded chunks once — bounded, and on an admin path. Two objects do not share a version
|
|
4970
|
+
* string, as far as their sizes and footer checksums tell them apart, whichever way a number came to name a second
|
|
4971
|
+
* one: a name purged and loaded again (a new row, or with no registry the same bare number), a number taken again
|
|
4972
|
+
* within one row once its object was deleted, or an object put back under its key from outside the library. That
|
|
4973
|
+
* holds however long after its row read a reader opens: a snapshot that a `count()` resolved from the row opens its
|
|
4974
|
+
* reader only when a later read needs it, and the object under the key may have changed in between. The version is
|
|
4975
|
+
* opaque: compare two for equality.
|
|
4717
4976
|
*
|
|
4718
4977
|
* This is the call the engine makes **once per operand of every read**, before any chunk fetch, to key its
|
|
4719
4978
|
* chunk cache, so it heals a swept generation exactly as {@link currentGeneration} does: unhealed, a cold read
|
|
@@ -4731,7 +4990,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4731
4990
|
this.dropStale(segmentKey(ref), snap);
|
|
4732
4991
|
reader = await this.resolvedReader(ref);
|
|
4733
4992
|
}
|
|
4734
|
-
return reader === null ? null :
|
|
4993
|
+
return reader === null ? null : versionOfReader(reader);
|
|
4735
4994
|
}
|
|
4736
4995
|
/**
|
|
4737
4996
|
* Resolve the segment **once** and report what a pin should hold: the generation, and the version that
|
|
@@ -4743,6 +5002,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4743
5002
|
* once the swept one is gone is exactly what a pin taken now should hold. One retry, then propagate.
|
|
4744
5003
|
*/
|
|
4745
5004
|
async pinGeneration(ref, lease) {
|
|
5005
|
+
validateUserRef(ref);
|
|
4746
5006
|
for (let moved = 0; ; moved++) {
|
|
4747
5007
|
try {
|
|
4748
5008
|
try {
|
|
@@ -4774,6 +5034,7 @@ var CrbmStorageChunkSource = class {
|
|
|
4774
5034
|
* of the row's summary against it, since that check is made where the row's own generation is read.
|
|
4775
5035
|
*/
|
|
4776
5036
|
async pinGenerationAt(ref, at, lease) {
|
|
5037
|
+
validateUserRef(ref);
|
|
4777
5038
|
const { generation, fingerprint } = at;
|
|
4778
5039
|
const gone = (why) => new NotFoundError(
|
|
4779
5040
|
`segment "${ref.segment}" generation ${generation} cannot be pinned: ${why}`
|
|
@@ -5060,7 +5321,11 @@ var CrbmStorageChunkSource = class {
|
|
|
5060
5321
|
* tell apart.
|
|
5061
5322
|
*/
|
|
5062
5323
|
async exists(ref) {
|
|
5063
|
-
|
|
5324
|
+
validateUserRef(ref);
|
|
5325
|
+
if (this.registry !== void 0) {
|
|
5326
|
+
const row = await this.registry.get(ref);
|
|
5327
|
+
return row !== null && row.status !== "destroyed";
|
|
5328
|
+
}
|
|
5064
5329
|
for await (const key of this.driver.list(ref)) {
|
|
5065
5330
|
void key;
|
|
5066
5331
|
return true;
|
|
@@ -5169,7 +5434,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5169
5434
|
const keystore = this.keystore;
|
|
5170
5435
|
if (keystore === void 0) {
|
|
5171
5436
|
throw new KeyUnavailableError(
|
|
5172
|
-
`segment "${ref.segment}" is encrypted
|
|
5437
|
+
`segment "${ref.segment}" is encrypted, and the store reading it was built with no keystore (encryption.keystore)`
|
|
5173
5438
|
);
|
|
5174
5439
|
}
|
|
5175
5440
|
const aead = await (unwrap ?? (() => keystore.openDek(wrappedDeks)))();
|
|
@@ -5193,10 +5458,12 @@ var CrbmStorageChunkSource = class {
|
|
|
5193
5458
|
for (const chunkKey of keys) validateChunkRef({ ...ref, chunkKey });
|
|
5194
5459
|
validateChunkKeyOrder(keys);
|
|
5195
5460
|
const retry = options?.retry;
|
|
5461
|
+
const key = segmentKey(ref);
|
|
5196
5462
|
let yielded = 0;
|
|
5197
5463
|
let healed = false;
|
|
5198
5464
|
for (; ; ) {
|
|
5199
5465
|
let snap = this.liveSnapshot(ref);
|
|
5466
|
+
let epoch = this.invalidations;
|
|
5200
5467
|
let pending = snap.reader;
|
|
5201
5468
|
let chunks;
|
|
5202
5469
|
try {
|
|
@@ -5215,7 +5482,8 @@ var CrbmStorageChunkSource = class {
|
|
|
5215
5482
|
}
|
|
5216
5483
|
return;
|
|
5217
5484
|
}
|
|
5218
|
-
const version =
|
|
5485
|
+
const version = versionOfReader(reader);
|
|
5486
|
+
let seen = snap.settled;
|
|
5219
5487
|
chunks = reader.readChunks(keys.slice(yielded), {
|
|
5220
5488
|
...options?.concurrency === void 0 ? {} : { concurrency: options.concurrency },
|
|
5221
5489
|
...options?.ramp === void 0 ? {} : { ramp: options.ramp },
|
|
@@ -5227,14 +5495,26 @@ var CrbmStorageChunkSource = class {
|
|
|
5227
5495
|
let movedOn = false;
|
|
5228
5496
|
try {
|
|
5229
5497
|
while (await pull.advance()) {
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5498
|
+
if (this.mayHaveMoved(key, snap, epoch) || snap.settled !== seen) {
|
|
5499
|
+
const live = this.liveSnapshot(ref);
|
|
5500
|
+
if (live !== snap || live.settled !== seen) {
|
|
5501
|
+
const looked = live.settled;
|
|
5502
|
+
let same;
|
|
5503
|
+
try {
|
|
5504
|
+
same = await readsObjectOf(live, reader);
|
|
5505
|
+
} catch (err) {
|
|
5506
|
+
if (!isTransientError(err)) throw err;
|
|
5507
|
+
movedOn = true;
|
|
5508
|
+
break;
|
|
5509
|
+
}
|
|
5510
|
+
if (!same) {
|
|
5511
|
+
movedOn = true;
|
|
5512
|
+
break;
|
|
5513
|
+
}
|
|
5514
|
+
snap = live;
|
|
5515
|
+
seen = looked;
|
|
5236
5516
|
}
|
|
5237
|
-
|
|
5517
|
+
epoch = this.invalidations;
|
|
5238
5518
|
}
|
|
5239
5519
|
yield pull.take((chunk) => {
|
|
5240
5520
|
yielded += 1;
|
|
@@ -5254,7 +5534,7 @@ var CrbmStorageChunkSource = class {
|
|
|
5254
5534
|
if (!isNotFoundError(err) && !await this.replacedUnder(ref, pending, err, retry))
|
|
5255
5535
|
throw err;
|
|
5256
5536
|
await chunks?.settled();
|
|
5257
|
-
this.dropStale(
|
|
5537
|
+
this.dropStale(key, snap);
|
|
5258
5538
|
healed = true;
|
|
5259
5539
|
}
|
|
5260
5540
|
}
|
|
@@ -5795,6 +6075,11 @@ async function provesOwnObject(storage, key, fingerprint) {
|
|
|
5795
6075
|
throw err;
|
|
5796
6076
|
}
|
|
5797
6077
|
}
|
|
6078
|
+
function cleartextUnderRequiredEncryption(ref, currentGen) {
|
|
6079
|
+
return new ValidationError(
|
|
6080
|
+
`requireEncryption: segment "${ref.segment}" already has generation ${currentGen} in cleartext, so this load cannot be encrypted \u2014 encryption is chosen when a segment is first loaded. Load into a new segment with the keystore wired, then drop this one.`
|
|
6081
|
+
);
|
|
6082
|
+
}
|
|
5798
6083
|
async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
5799
6084
|
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
5800
6085
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
@@ -5831,11 +6116,8 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
5831
6116
|
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
5832
6117
|
crypto = { aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
5833
6118
|
} else if (existing !== null && existing.currentGen !== null) {
|
|
5834
|
-
if (options.requireEncryption === true)
|
|
5835
|
-
throw
|
|
5836
|
-
`requireEncryption: segment "${ref.segment}" already has generation ${existing.currentGen} in cleartext, so this load cannot be encrypted \u2014 encryption is chosen when a segment is first loaded. Load into a new segment with the keystore wired, then drop this one.`
|
|
5837
|
-
);
|
|
5838
|
-
}
|
|
6119
|
+
if (options.requireEncryption === true)
|
|
6120
|
+
throw cleartextUnderRequiredEncryption(ref, existing.currentGen);
|
|
5839
6121
|
} else {
|
|
5840
6122
|
const minted = await options.keystore.createDek();
|
|
5841
6123
|
newWrapped = minted.wrapped;
|
|
@@ -6147,6 +6429,30 @@ function applyJitter(policy, delayMs, rng) {
|
|
|
6147
6429
|
function retryRead(op, retry) {
|
|
6148
6430
|
return retry === void 0 ? op() : withRetry(op, retry.policy ?? DEFAULT_RETRY_POLICY, retry);
|
|
6149
6431
|
}
|
|
6432
|
+
function checkedRetryPolicy(policy) {
|
|
6433
|
+
const { maxAttempts, baseDelayMs, maxDelayMs, backoffFactor, jitter } = policy;
|
|
6434
|
+
const fail = (field, rule, v) => {
|
|
6435
|
+
throw new ValidationError(
|
|
6436
|
+
`retry.${field} must be ${rule}; got ${typeof v === "number" ? String(v) : `a ${typeof v}`}`
|
|
6437
|
+
);
|
|
6438
|
+
};
|
|
6439
|
+
if (typeof maxAttempts !== "number" || !Number.isFinite(maxAttempts) || maxAttempts < 1) {
|
|
6440
|
+
fail("maxAttempts", "a finite number >= 1", maxAttempts);
|
|
6441
|
+
}
|
|
6442
|
+
if (typeof baseDelayMs !== "number" || !Number.isFinite(baseDelayMs) || baseDelayMs < 0) {
|
|
6443
|
+
fail("baseDelayMs", "a finite number of milliseconds >= 0", baseDelayMs);
|
|
6444
|
+
}
|
|
6445
|
+
if (typeof maxDelayMs !== "number" || !Number.isFinite(maxDelayMs) || maxDelayMs < 0) {
|
|
6446
|
+
fail("maxDelayMs", "a finite number of milliseconds >= 0", maxDelayMs);
|
|
6447
|
+
}
|
|
6448
|
+
if (typeof backoffFactor !== "number" || !Number.isFinite(backoffFactor) || backoffFactor < 1) {
|
|
6449
|
+
fail("backoffFactor", "a finite number >= 1", backoffFactor);
|
|
6450
|
+
}
|
|
6451
|
+
if (jitter !== "full" && jitter !== "none") {
|
|
6452
|
+
throw new ValidationError(`retry.jitter must be 'full' or 'none'`);
|
|
6453
|
+
}
|
|
6454
|
+
return policy;
|
|
6455
|
+
}
|
|
6150
6456
|
async function withRetry(op, policy, deps) {
|
|
6151
6457
|
const retryable = deps.isRetryable ?? isTransientError;
|
|
6152
6458
|
if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
|
|
@@ -6174,6 +6480,106 @@ async function withRetry(op, policy, deps) {
|
|
|
6174
6480
|
}
|
|
6175
6481
|
|
|
6176
6482
|
// src/core/load.ts
|
|
6483
|
+
function checkedGuard(guard, where) {
|
|
6484
|
+
if (guard === void 0 || guard === null) return void 0;
|
|
6485
|
+
const shown2 = (v) => typeof v === "number" ? String(v) : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
6486
|
+
if (typeof guard !== "object" || Array.isArray(guard)) {
|
|
6487
|
+
throw new ValidationError(
|
|
6488
|
+
`${where}guard must be an object of bounds, such as { minRetained: 0.5 }; got ${shown2(guard)}`
|
|
6489
|
+
);
|
|
6490
|
+
}
|
|
6491
|
+
const bounds = guard;
|
|
6492
|
+
const unknown = Object.keys(bounds).filter(
|
|
6493
|
+
(k) => k !== "minCardinality" && k !== "minRetained" && k !== "maxGrowth" && bounds[k] !== void 0
|
|
6494
|
+
);
|
|
6495
|
+
if (unknown.length > 0) {
|
|
6496
|
+
throw new ValidationError(
|
|
6497
|
+
`${where}guard: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { minCardinality, minRetained, maxGrowth }`
|
|
6498
|
+
);
|
|
6499
|
+
}
|
|
6500
|
+
const { minCardinality, minRetained, maxGrowth } = guard;
|
|
6501
|
+
if (minCardinality !== void 0 && (!Number.isInteger(minCardinality) || minCardinality < 0)) {
|
|
6502
|
+
throw new ValidationError(
|
|
6503
|
+
`${where}guard.minCardinality must be a non-negative integer; got ${shown2(minCardinality)}`
|
|
6504
|
+
);
|
|
6505
|
+
}
|
|
6506
|
+
if (minRetained !== void 0 && (!Number.isFinite(minRetained) || minRetained < 0 || minRetained > 1)) {
|
|
6507
|
+
throw new ValidationError(
|
|
6508
|
+
`${where}guard.minRetained must be a fraction in 0..1; got ${shown2(minRetained)}`
|
|
6509
|
+
);
|
|
6510
|
+
}
|
|
6511
|
+
if (maxGrowth !== void 0 && (!Number.isFinite(maxGrowth) || maxGrowth !== 0 && maxGrowth < 1)) {
|
|
6512
|
+
throw new ValidationError(
|
|
6513
|
+
`${where}guard.maxGrowth must be 0 (no bound) or a factor of at least 1; got ${shown2(maxGrowth)}`
|
|
6514
|
+
);
|
|
6515
|
+
}
|
|
6516
|
+
return Object.freeze({
|
|
6517
|
+
...minCardinality === void 0 ? {} : { minCardinality },
|
|
6518
|
+
...minRetained === void 0 ? {} : { minRetained },
|
|
6519
|
+
...maxGrowth === void 0 ? {} : { maxGrowth }
|
|
6520
|
+
});
|
|
6521
|
+
}
|
|
6522
|
+
function guardRefusal(cardinality, before, guard, allowEmpty) {
|
|
6523
|
+
if (cardinality === 0 && allowEmpty !== true && (before ?? 0) > 0) return "empty";
|
|
6524
|
+
if (guard?.minCardinality !== void 0 && cardinality < guard.minCardinality) {
|
|
6525
|
+
return "min-cardinality";
|
|
6526
|
+
}
|
|
6527
|
+
if (guard?.minRetained !== void 0 && before !== null && cardinality < before * guard.minRetained) {
|
|
6528
|
+
return "min-retained";
|
|
6529
|
+
}
|
|
6530
|
+
if (guard?.maxGrowth !== void 0 && guard.maxGrowth !== 0 && before !== null && before > 0 && cardinality > before * guard.maxGrowth) {
|
|
6531
|
+
return "max-growth";
|
|
6532
|
+
}
|
|
6533
|
+
return void 0;
|
|
6534
|
+
}
|
|
6535
|
+
async function judgeLoad(ref, cardinality, deps, options = {}) {
|
|
6536
|
+
validateUserRef(ref);
|
|
6537
|
+
if (!Number.isSafeInteger(cardinality) || cardinality < 0) {
|
|
6538
|
+
throw new ValidationError("judgeLoad: cardinality must be a non-negative integer");
|
|
6539
|
+
}
|
|
6540
|
+
const guard = checkedGuard(options.guard, "");
|
|
6541
|
+
assertRegistryCanWrite(deps.registry, "load");
|
|
6542
|
+
if (deps.requireEncryption === true && deps.keystore === void 0) {
|
|
6543
|
+
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
6544
|
+
}
|
|
6545
|
+
const row = await deps.registry.get(ref);
|
|
6546
|
+
if (row?.status === "destroyed") {
|
|
6547
|
+
throw new ValidationError(
|
|
6548
|
+
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
6549
|
+
);
|
|
6550
|
+
}
|
|
6551
|
+
const cleartext = row?.wrappedDeks === void 0 || row.wrappedDeks.length === 0;
|
|
6552
|
+
if (deps.requireEncryption === true && row !== null && row.currentGen !== null && cleartext) {
|
|
6553
|
+
throw cleartextUnderRequiredEncryption(ref, row.currentGen);
|
|
6554
|
+
}
|
|
6555
|
+
const wrapped = row?.wrappedDeks;
|
|
6556
|
+
let aead;
|
|
6557
|
+
if (row !== null && wrapped !== void 0 && wrapped.length > 0) {
|
|
6558
|
+
if (deps.keystore === void 0) {
|
|
6559
|
+
throw new KeyUnavailableError(
|
|
6560
|
+
`segment "${ref.segment}" is encrypted but load was given no keystore`
|
|
6561
|
+
);
|
|
6562
|
+
}
|
|
6563
|
+
aead = Promise.resolve(await deps.keystore.openDek(wrapped));
|
|
6564
|
+
}
|
|
6565
|
+
let current;
|
|
6566
|
+
try {
|
|
6567
|
+
current = await currentCardinality(ref, deps, row, aead);
|
|
6568
|
+
} catch (err) {
|
|
6569
|
+
if (sizeNeeded(guard, options.allowEmpty)) throw err;
|
|
6570
|
+
current = { cardinality: null, fromSummary: false };
|
|
6571
|
+
}
|
|
6572
|
+
const wouldRefuse = guardRefusal(cardinality, current.cardinality, guard, options.allowEmpty);
|
|
6573
|
+
const openedObject = row !== null && row.currentGen !== null && !current.fromSummary;
|
|
6574
|
+
return {
|
|
6575
|
+
cardinalityBefore: current.cardinality,
|
|
6576
|
+
...wouldRefuse === void 0 ? {} : { wouldRefuse },
|
|
6577
|
+
reads: openedObject ? 2 : 1
|
|
6578
|
+
};
|
|
6579
|
+
}
|
|
6580
|
+
function sizeNeeded(guard, allowEmpty) {
|
|
6581
|
+
return guard?.minRetained !== void 0 || guard?.maxGrowth !== void 0 || allowEmpty !== true;
|
|
6582
|
+
}
|
|
6177
6583
|
function settled(promise) {
|
|
6178
6584
|
promise.catch(() => void 0);
|
|
6179
6585
|
return promise;
|
|
@@ -6232,26 +6638,13 @@ function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
|
6232
6638
|
}
|
|
6233
6639
|
async function runLoad(ref, prepare, deps, options) {
|
|
6234
6640
|
validateUserRef(ref);
|
|
6641
|
+
checkedAuditSink(options.audit, "load");
|
|
6235
6642
|
const codec = requireCodec(deps.codec, "loadSegment");
|
|
6236
6643
|
const keep = options.keep ?? 1;
|
|
6237
6644
|
if (!Number.isInteger(keep) || keep < 0) {
|
|
6238
6645
|
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
6239
6646
|
}
|
|
6240
|
-
const guard = options.guard;
|
|
6241
|
-
if (guard?.minCardinality !== void 0) {
|
|
6242
|
-
const m = guard.minCardinality;
|
|
6243
|
-
if (!Number.isInteger(m) || m < 0) {
|
|
6244
|
-
throw new ValidationError(
|
|
6245
|
-
`guard.minCardinality must be a non-negative integer; got ${String(m)}`
|
|
6246
|
-
);
|
|
6247
|
-
}
|
|
6248
|
-
}
|
|
6249
|
-
if (guard?.minRetained !== void 0) {
|
|
6250
|
-
const s = guard.minRetained;
|
|
6251
|
-
if (!Number.isFinite(s) || s < 0 || s > 1) {
|
|
6252
|
-
throw new ValidationError(`guard.minRetained must be a fraction in 0..1; got ${String(s)}`);
|
|
6253
|
-
}
|
|
6254
|
-
}
|
|
6647
|
+
const guard = checkedGuard(options.guard, "");
|
|
6255
6648
|
const metadata = copiedMetadata(options.metadata, (message) => {
|
|
6256
6649
|
throw new ValidationError(message);
|
|
6257
6650
|
});
|
|
@@ -6264,7 +6657,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6264
6657
|
const wrappedKeys = row?.wrappedDeks;
|
|
6265
6658
|
const keystore = deps.keystore;
|
|
6266
6659
|
const unwrapped = row !== null && row.status !== "destroyed" && wrappedKeys !== void 0 && wrappedKeys.length > 0 && keystore !== void 0 ? { wrapped: wrappedKeys, aead: settled((async () => keystore.openDek(wrappedKeys))()) } : void 0;
|
|
6267
|
-
const needsBefore = guard
|
|
6660
|
+
const needsBefore = sizeNeeded(guard, options.allowEmpty);
|
|
6268
6661
|
const current = needsBefore ? await currentCardinality(ref, deps, row, unwrapped?.aead) : { cardinality: null, fromSummary: false };
|
|
6269
6662
|
const before = current.cardinality;
|
|
6270
6663
|
const numbering = settled(nextLoadGeneration(ref, deps, row));
|
|
@@ -6323,8 +6716,9 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6323
6716
|
}
|
|
6324
6717
|
};
|
|
6325
6718
|
let unanswered = false;
|
|
6719
|
+
const tidy = () => reclaim().catch(() => void 0);
|
|
6326
6720
|
const refuse = async (reason) => {
|
|
6327
|
-
await
|
|
6721
|
+
await tidy();
|
|
6328
6722
|
audit.onEvent({
|
|
6329
6723
|
kind: "segment.load-refused",
|
|
6330
6724
|
segment: ref.segment,
|
|
@@ -6346,15 +6740,8 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6346
6740
|
collected: []
|
|
6347
6741
|
};
|
|
6348
6742
|
};
|
|
6349
|
-
|
|
6350
|
-
|
|
6351
|
-
}
|
|
6352
|
-
if (guard?.minCardinality !== void 0 && written.cardinality < guard.minCardinality) {
|
|
6353
|
-
return refuse("min-cardinality");
|
|
6354
|
-
}
|
|
6355
|
-
if (guard?.minRetained !== void 0 && before !== null) {
|
|
6356
|
-
if (written.cardinality < before * guard.minRetained) return refuse("min-retained");
|
|
6357
|
-
}
|
|
6743
|
+
const refusal = guardRefusal(written.cardinality, before, guard, options.allowEmpty);
|
|
6744
|
+
if (refusal !== void 0) return refuse(refusal);
|
|
6358
6745
|
const leasesNow = deps.clock === void 0 ? void 0 : () => deps.clock.now();
|
|
6359
6746
|
let published;
|
|
6360
6747
|
try {
|
|
@@ -6367,14 +6754,14 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6367
6754
|
cleartext: !written.encrypted,
|
|
6368
6755
|
...fromToken === void 0 ? {} : { expectToken: fromToken, expectRow: row },
|
|
6369
6756
|
...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
|
|
6370
|
-
//
|
|
6371
|
-
//
|
|
6372
|
-
//
|
|
6373
|
-
//
|
|
6374
|
-
//
|
|
6375
|
-
//
|
|
6376
|
-
//
|
|
6377
|
-
...
|
|
6757
|
+
// A load that found NO ROW fences on that absence, guarded or not, so a row that appeared since refuses it.
|
|
6758
|
+
// `expectFrom` and `expectToken` both compare against a value read from a row, so with no row both are omitted,
|
|
6759
|
+
// and the publish would otherwise be a bare forward-only advance over whatever appeared in between. For a guarded
|
|
6760
|
+
// load, `before` was `null` and its bounds passed vacuously, so an empty generation would land over the ids
|
|
6761
|
+
// another writer had just loaded. For an unguarded one, the pointer would move over another first load's
|
|
6762
|
+
// generation or onto a row `setRetention` made, and, where an erasure had already deleted this load's object
|
|
6763
|
+
// above that row's pointer, onto an object that is gone, so that every read of the segment would fail.
|
|
6764
|
+
...row === null ? { expectAbsent: true } : {},
|
|
6378
6765
|
// A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
|
|
6379
6766
|
// is this load's publish only over the object this load wrote, which one footer read proves.
|
|
6380
6767
|
holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
|
|
@@ -6386,7 +6773,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
6386
6773
|
}
|
|
6387
6774
|
});
|
|
6388
6775
|
} catch (err) {
|
|
6389
|
-
if (isValidationError(err) || err
|
|
6776
|
+
if (isValidationError(err) || isKeyUnavailableError(err)) await tidy();
|
|
6390
6777
|
throw err;
|
|
6391
6778
|
}
|
|
6392
6779
|
if (!published.published) return refuse("superseded");
|
|
@@ -6925,9 +7312,12 @@ var FeedCursor = class {
|
|
|
6925
7312
|
this.finished = true;
|
|
6926
7313
|
}
|
|
6927
7314
|
// ---- the checks -------------------------------------------------------------------------------------------
|
|
6928
|
-
|
|
7315
|
+
/** How many records the feed has handed over, the one being read included: what names a record in an error. */
|
|
7316
|
+
records = 0;
|
|
7317
|
+
/** Refuse the record being read, named by its position in the feed: its key narrows the ids it holds to a range. */
|
|
7318
|
+
refuse(operand, why) {
|
|
6929
7319
|
throw new ValidationError(
|
|
6930
|
-
`materializeMany feed: record
|
|
7320
|
+
`materializeMany feed: record ${this.records}${operand === void 0 ? "" : `, operand "${operand}"`}: ${why}`
|
|
6931
7321
|
);
|
|
6932
7322
|
}
|
|
6933
7323
|
charge(bytes) {
|
|
@@ -6944,20 +7334,17 @@ var FeedCursor = class {
|
|
|
6944
7334
|
this.deps.release(bytes);
|
|
6945
7335
|
}
|
|
6946
7336
|
async convert(record) {
|
|
7337
|
+
this.records += 1;
|
|
6947
7338
|
if (typeof record !== "object" || record === null) {
|
|
6948
|
-
this.refuse(
|
|
7339
|
+
this.refuse(void 0, "a record is an object with a key and operands");
|
|
6949
7340
|
}
|
|
6950
7341
|
const key = record.key;
|
|
6951
7342
|
const operands = record.operands;
|
|
6952
7343
|
if (typeof key !== "number" || !Number.isInteger(key) || key < 0 || key >= KEY_IDS) {
|
|
6953
|
-
this.refuse(
|
|
7344
|
+
this.refuse(void 0, "the key must be an integer from 0 to 65535");
|
|
6954
7345
|
}
|
|
6955
7346
|
if (key < this.prevKey) {
|
|
6956
|
-
this.refuse(
|
|
6957
|
-
key,
|
|
6958
|
-
void 0,
|
|
6959
|
-
`the key is below the previous record's (${this.prevKey}): keys never go back`
|
|
6960
|
-
);
|
|
7347
|
+
this.refuse(void 0, `the key is below the previous record's: keys never go back`);
|
|
6961
7348
|
}
|
|
6962
7349
|
if (key !== this.prevKey) {
|
|
6963
7350
|
for (const i of this.touched) this.seen[i] = 0;
|
|
@@ -6966,7 +7353,6 @@ var FeedCursor = class {
|
|
|
6966
7353
|
}
|
|
6967
7354
|
if (typeof operands !== "object" || operands === null || !isPlainObject(operands)) {
|
|
6968
7355
|
this.refuse(
|
|
6969
|
-
key,
|
|
6970
7356
|
void 0,
|
|
6971
7357
|
"operands must be a plain object of fed operand name to Uint32Array (not a Map, an array or a class instance)"
|
|
6972
7358
|
);
|
|
@@ -6976,31 +7362,29 @@ var FeedCursor = class {
|
|
|
6976
7362
|
const charged = [];
|
|
6977
7363
|
try {
|
|
6978
7364
|
for (const name of Reflect.ownKeys(operands)) {
|
|
6979
|
-
if (typeof name !== "string") this.refuse(
|
|
7365
|
+
if (typeof name !== "string") this.refuse(void 0, "operands has a symbol key");
|
|
6980
7366
|
const at = this.index.get(name);
|
|
6981
7367
|
if (at === void 0) {
|
|
6982
|
-
this.refuse(
|
|
7368
|
+
this.refuse(name, "not a declared fed operand (it is not in feed.names)");
|
|
6983
7369
|
}
|
|
6984
7370
|
const value = operands[name];
|
|
6985
7371
|
const view = plainUint32(value);
|
|
6986
7372
|
if (view === void 0) {
|
|
6987
7373
|
this.refuse(
|
|
6988
|
-
key,
|
|
6989
7374
|
name,
|
|
6990
7375
|
"the ids must be a Uint32Array (any other typed array, array or look-alike is refused)"
|
|
6991
7376
|
);
|
|
6992
7377
|
}
|
|
6993
|
-
if (view === "detached") this.refuse(
|
|
7378
|
+
if (view === "detached") this.refuse(name, "the array's buffer is detached");
|
|
6994
7379
|
if (view === "out of bounds") {
|
|
6995
7380
|
this.refuse(
|
|
6996
|
-
key,
|
|
6997
7381
|
name,
|
|
6998
7382
|
"the array is out of bounds of its resizable buffer, which shrank under it"
|
|
6999
7383
|
);
|
|
7000
7384
|
}
|
|
7001
7385
|
if (view.length === 0) continue;
|
|
7002
7386
|
if (this.seen[at] === 1) {
|
|
7003
|
-
this.refuse(
|
|
7387
|
+
this.refuse(name, "named twice at one key: an operand appears at most once per key");
|
|
7004
7388
|
}
|
|
7005
7389
|
const n = this.scan(view, key, name);
|
|
7006
7390
|
this.seen[at] = 1;
|
|
@@ -7034,17 +7418,16 @@ var FeedCursor = class {
|
|
|
7034
7418
|
*/
|
|
7035
7419
|
scan(view, key, name) {
|
|
7036
7420
|
const n = view.length;
|
|
7037
|
-
if (n > KEY_IDS) this.refuse(
|
|
7421
|
+
if (n > KEY_IDS) this.refuse(name, `${n} ids, more than one chunk key holds`);
|
|
7038
7422
|
const scratch = this.scratch;
|
|
7039
7423
|
let prev = -1;
|
|
7040
7424
|
for (let i = 0; i < n; i++) {
|
|
7041
7425
|
const id = view[i];
|
|
7042
7426
|
if (id >>> 16 !== key) {
|
|
7043
|
-
this.refuse(
|
|
7427
|
+
this.refuse(name, `the id at position ${i} is not inside the key`);
|
|
7044
7428
|
}
|
|
7045
7429
|
if (id <= prev) {
|
|
7046
7430
|
this.refuse(
|
|
7047
|
-
key,
|
|
7048
7431
|
name,
|
|
7049
7432
|
`the id at position ${i} is not above the one before it: ids are ascending and unique`
|
|
7050
7433
|
);
|
|
@@ -7191,13 +7574,6 @@ function compileCombineMany(request) {
|
|
|
7191
7574
|
}
|
|
7192
7575
|
const index = /* @__PURE__ */ new Map();
|
|
7193
7576
|
operands.forEach((op, i) => index.set(op.name, i));
|
|
7194
|
-
for (const op of operands) {
|
|
7195
|
-
if (op.held !== void 0 && op.held.chunks.empty && !op.held.mayBeEmpty) {
|
|
7196
|
-
throw new ValidationError(
|
|
7197
|
-
`materializeMany: operand "${op.name}" holds no id, so an include would contribute none and an exclude would subtract none. Check what it was made from, or name it in \`mayBeEmpty\` if it may be empty.`
|
|
7198
|
-
);
|
|
7199
|
-
}
|
|
7200
|
-
}
|
|
7201
7577
|
const feed = request.feed === void 0 ? void 0 : checkFeed(request.feed, new Set(index.keys()));
|
|
7202
7578
|
feed?.names.forEach((name, j) => index.set(name, operands.length + j));
|
|
7203
7579
|
let totalNodes = 0;
|
|
@@ -7227,24 +7603,9 @@ function compileCombineMany(request) {
|
|
|
7227
7603
|
if (output.allowEmpty !== void 0 && typeof output.allowEmpty !== "boolean") {
|
|
7228
7604
|
throw new ValidationError(`${where}.allowEmpty must be a boolean`);
|
|
7229
7605
|
}
|
|
7230
|
-
|
|
7231
|
-
if (guard
|
|
7232
|
-
|
|
7233
|
-
throw new ValidationError(`${where}.guard must be an object`);
|
|
7234
|
-
}
|
|
7235
|
-
const min = guard.minCardinality;
|
|
7236
|
-
if (min !== void 0 && (!Number.isInteger(min) || min < 0)) {
|
|
7237
|
-
throw new ValidationError(
|
|
7238
|
-
`${where}.guard.minCardinality must be a non-negative integer; got ${String(min)}`
|
|
7239
|
-
);
|
|
7240
|
-
}
|
|
7241
|
-
const retained = guard.minRetained;
|
|
7242
|
-
if (retained !== void 0 && (!Number.isFinite(retained) || retained < 0 || retained > 1)) {
|
|
7243
|
-
throw new ValidationError(
|
|
7244
|
-
`${where}.guard.minRetained must be a fraction in 0..1; got ${String(retained)}`
|
|
7245
|
-
);
|
|
7246
|
-
}
|
|
7247
|
-
}
|
|
7606
|
+
checkedAuditSink(output.audit, where);
|
|
7607
|
+
if (output.guard === null) throw new ValidationError(`${where}.guard must be an object`);
|
|
7608
|
+
const guard = checkedGuard(output.guard, `${where}.`);
|
|
7248
7609
|
const metadata = copiedMetadata(output.metadata, (message) => {
|
|
7249
7610
|
throw new ValidationError(`${where}.metadata: ${message}`);
|
|
7250
7611
|
});
|
|
@@ -7423,6 +7784,7 @@ var Run = class {
|
|
|
7423
7784
|
codec;
|
|
7424
7785
|
clock;
|
|
7425
7786
|
maxBitmapBytes;
|
|
7787
|
+
metrics;
|
|
7426
7788
|
ledger;
|
|
7427
7789
|
gate = new Gate(MAX_RANGES_IN_FLIGHT2);
|
|
7428
7790
|
operands;
|
|
@@ -7437,6 +7799,8 @@ var Run = class {
|
|
|
7437
7799
|
publishes = 0;
|
|
7438
7800
|
opens = 0;
|
|
7439
7801
|
planCharge = 0;
|
|
7802
|
+
/** The groups {@link formGroups} made, whose kept plans {@link dropKeptPlans} can release. */
|
|
7803
|
+
formed = [];
|
|
7440
7804
|
attributedGet = 0;
|
|
7441
7805
|
attributedPut = 0;
|
|
7442
7806
|
planned = 0;
|
|
@@ -7462,6 +7826,7 @@ var Run = class {
|
|
|
7462
7826
|
this.codec = deps.codec;
|
|
7463
7827
|
this.clock = deps.clock;
|
|
7464
7828
|
this.maxBitmapBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
7829
|
+
this.metrics = deps.metrics === void 0 ? void 0 : safeMetrics(deps.metrics);
|
|
7465
7830
|
this.ledger = new ResidentLedger(this.req.maxBufferedBytes);
|
|
7466
7831
|
this.tick = yieldEvery(deps.clock, 256);
|
|
7467
7832
|
this.operands = this.req.operands.map((spec, index) => ({
|
|
@@ -7501,7 +7866,6 @@ var Run = class {
|
|
|
7501
7866
|
async execute() {
|
|
7502
7867
|
this.window = windowOf2(this.req.after, this.req.through);
|
|
7503
7868
|
if (this.checkedFeed !== void 0) this.feed = this.openFeed(this.checkedFeed);
|
|
7504
|
-
this.indexHeld();
|
|
7505
7869
|
await this.readIndexes();
|
|
7506
7870
|
await this.refuseAbsent();
|
|
7507
7871
|
this.planOutputs();
|
|
@@ -7565,11 +7929,13 @@ var Run = class {
|
|
|
7565
7929
|
}
|
|
7566
7930
|
}
|
|
7567
7931
|
/**
|
|
7568
|
-
* Make room in the ledger.
|
|
7569
|
-
*
|
|
7570
|
-
*
|
|
7932
|
+
* Make room in the ledger. The plans kept for later groups give way first: they are rebuilt when their group starts,
|
|
7933
|
+
* and the outputs were judged without them. A fed output cannot be run again (the feed is read once), so a fed call
|
|
7934
|
+
* gives up its fed outputs and the feed next, as soon as the ledger passes the budget; only then is a stored-only
|
|
7935
|
+
* buffer evicted, to run again alone.
|
|
7571
7936
|
*/
|
|
7572
7937
|
reclaim() {
|
|
7938
|
+
if (this.dropKeptPlans()) return true;
|
|
7573
7939
|
if (this.feed !== void 0) {
|
|
7574
7940
|
let fed = this.feed.heldBytes;
|
|
7575
7941
|
for (const o of this.running)
|
|
@@ -7585,70 +7951,13 @@ var Run = class {
|
|
|
7585
7951
|
return { ok: false, error: new Error("materializeMany: output did not run") };
|
|
7586
7952
|
}
|
|
7587
7953
|
// ---- index ------------------------------------------------------------------------------------------------
|
|
7588
|
-
/** What an output that reads the held operand `op` gets when an erasure ran in the store since it was made. */
|
|
7589
|
-
staleHeldError(op) {
|
|
7590
|
-
return new StaleOperandError(
|
|
7591
|
-
`materializeMany: an erasure ran in this store after operand "${op.spec.name}" was made, so it may hold an erased id and the outputs that read it were not published. Make it again from the source of truth, after the erasure, and run the call again.`,
|
|
7592
|
-
op.spec.name,
|
|
7593
|
-
"erased"
|
|
7594
|
-
);
|
|
7595
|
-
}
|
|
7596
|
-
/** The error of the first held operand `o` reads that an erasure has made stale, if any. */
|
|
7597
|
-
staleHeld(o) {
|
|
7598
|
-
for (const i of o.compiled.operands) {
|
|
7599
|
-
const op = this.operands[i];
|
|
7600
|
-
if (op.spec.held?.moved() === true) return this.staleHeldError(op);
|
|
7601
|
-
}
|
|
7602
|
-
return void 0;
|
|
7603
|
-
}
|
|
7604
|
-
/**
|
|
7605
|
-
* Index the held operands the outputs read, before any request: their keys and exact cardinalities are in memory, and
|
|
7606
|
-
* one that an erasure has made stale is failed here, so the outputs that read it fail without a read.
|
|
7607
|
-
*/
|
|
7608
|
-
indexHeld() {
|
|
7609
|
-
const wanted = /* @__PURE__ */ new Set();
|
|
7610
|
-
for (const o of this.outputs) for (const op of o.compiled.operands) wanted.add(op);
|
|
7611
|
-
if (this.window === "empty") return;
|
|
7612
|
-
const w = this.window;
|
|
7613
|
-
const charged = /* @__PURE__ */ new Set();
|
|
7614
|
-
let resident = 0;
|
|
7615
|
-
for (const index of wanted) {
|
|
7616
|
-
const op = this.operands[index];
|
|
7617
|
-
const held = op.spec.held;
|
|
7618
|
-
if (held === void 0) continue;
|
|
7619
|
-
op.read = true;
|
|
7620
|
-
if (held.moved()) {
|
|
7621
|
-
op.error = { error: this.staleHeldError(op) };
|
|
7622
|
-
continue;
|
|
7623
|
-
}
|
|
7624
|
-
if (!charged.has(held.chunks)) {
|
|
7625
|
-
charged.add(held.chunks);
|
|
7626
|
-
resident += held.chunks.residentBytes;
|
|
7627
|
-
}
|
|
7628
|
-
const all = held.chunks.keys;
|
|
7629
|
-
const cards = held.chunks.cardinalities;
|
|
7630
|
-
const keep = [];
|
|
7631
|
-
const card = [];
|
|
7632
|
-
for (let i = 0; i < all.length; i++) {
|
|
7633
|
-
const key = all[i];
|
|
7634
|
-
if (w !== null && (key < w.loKey || key > w.hiKey)) continue;
|
|
7635
|
-
keep.push(key);
|
|
7636
|
-
card.push(cards[i]);
|
|
7637
|
-
}
|
|
7638
|
-
op.bounds = { keys: Uint16Array.from(keep), card: Uint32Array.from(card) };
|
|
7639
|
-
op.indexKeys = keep.length;
|
|
7640
|
-
op.chunkless = all.length === 0;
|
|
7641
|
-
}
|
|
7642
|
-
if (resident > 0) this.chargePlan(resident, "the held operands");
|
|
7643
|
-
}
|
|
7644
7954
|
async readIndexes() {
|
|
7645
7955
|
const wanted = /* @__PURE__ */ new Set();
|
|
7646
7956
|
for (const o of this.outputs) for (const op of o.compiled.operands) wanted.add(op);
|
|
7647
7957
|
if (this.window === "empty") return;
|
|
7648
7958
|
for (const index of wanted) this.operands[index].read = true;
|
|
7649
7959
|
const w = this.window;
|
|
7650
|
-
|
|
7651
|
-
await mapWithConcurrency(stored, INDEX_PARALLELISM, async (index) => {
|
|
7960
|
+
await mapWithConcurrency([...wanted], INDEX_PARALLELISM, async (index) => {
|
|
7652
7961
|
const op = this.operands[index];
|
|
7653
7962
|
try {
|
|
7654
7963
|
const listed = checkedChunkKeys(await this.source.listChunkKeys(op.spec.ref));
|
|
@@ -7686,7 +7995,7 @@ var Run = class {
|
|
|
7686
7995
|
async refuseAbsent() {
|
|
7687
7996
|
if (this.req.allowAbsentOperands === true || this.source.exists === void 0) return;
|
|
7688
7997
|
const judged = this.operands.filter(
|
|
7689
|
-
(op) => op.
|
|
7998
|
+
(op) => op.error === void 0 && (op.read ? op.chunkless : true)
|
|
7690
7999
|
);
|
|
7691
8000
|
if (judged.length === 0) return;
|
|
7692
8001
|
const exists = this.source.exists.bind(this.source);
|
|
@@ -7697,7 +8006,7 @@ var Run = class {
|
|
|
7697
8006
|
const absent = checked.filter((c) => !c.exists).map((c) => c.op.spec.name);
|
|
7698
8007
|
if (absent.length === 0) return;
|
|
7699
8008
|
throw new ValidationError(
|
|
7700
|
-
`materializeMany: operand ${absent.map((n) => `"${n}"`).join(", ")} names a segment that does not exist, so it would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
8009
|
+
`materializeMany: operand ${absent.map((n) => `"${n}"`).join(", ")} names a segment that does not exist or was dropped, so it would contribute nothing \u2014 an exclude would suppress no ids and an include would contribute none. Check the name and the namespace (a segment addressed without its \`namespace\` is a DIFFERENT segment). Pass \`allowAbsentOperands: true\` if you meant it.`
|
|
7701
8010
|
);
|
|
7702
8011
|
}
|
|
7703
8012
|
// ---- plan -------------------------------------------------------------------------------------------------
|
|
@@ -7759,12 +8068,15 @@ var Run = class {
|
|
|
7759
8068
|
o.status = "waiting";
|
|
7760
8069
|
}
|
|
7761
8070
|
}
|
|
7762
|
-
/**
|
|
7763
|
-
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
|
|
8071
|
+
/**
|
|
8072
|
+
* Outputs in call order, a group at a time while the ledger has room for the next. Every output is judged against the
|
|
8073
|
+
* room the call's own plan leaves, `base`: a group's plan kept for its start is released when it starts, or sooner when
|
|
8074
|
+
* a group needs the room ({@link dropKeptPlans}), so it is not resident while a later output runs, and counting it
|
|
8075
|
+
* would refuse an output, or cut a group short, by its position in the call.
|
|
8076
|
+
*/
|
|
7767
8077
|
formGroups(waiting, single) {
|
|
8078
|
+
const base = this.planCharge;
|
|
8079
|
+
const room = this.req.maxBufferedBytes - base;
|
|
7768
8080
|
const groups = [];
|
|
7769
8081
|
const fresh = () => new GroupCost((i) => this.operands[i].bytes, this.req.publishConcurrency);
|
|
7770
8082
|
let cost = fresh();
|
|
@@ -7790,22 +8102,36 @@ var Run = class {
|
|
|
7790
8102
|
};
|
|
7791
8103
|
for (const o of waiting) {
|
|
7792
8104
|
const alone = fresh().with(o);
|
|
7793
|
-
if (alone >
|
|
8105
|
+
if (alone > room) {
|
|
7794
8106
|
this.fail(
|
|
7795
8107
|
o,
|
|
7796
8108
|
new BudgetExceededError(
|
|
7797
|
-
`materializeMany: output ${o.index} needs about ${alone} bytes resident with its operands, and the plan holds ${
|
|
8109
|
+
`materializeMany: output ${o.index} needs about ${alone} bytes resident with its operands, and the plan holds ${base}, over maxBufferedBytes (${this.req.maxBufferedBytes}) \u2014 raise maxBufferedBytes to at least ${alone + base}`
|
|
7798
8110
|
)
|
|
7799
8111
|
);
|
|
7800
8112
|
continue;
|
|
7801
8113
|
}
|
|
7802
|
-
if (!single && cost.size > 0 && cost.with(o) >
|
|
8114
|
+
if (!single && cost.size > 0 && cost.with(o) > room) flush();
|
|
7803
8115
|
current.push(o);
|
|
7804
8116
|
cost.add(o);
|
|
7805
8117
|
}
|
|
7806
8118
|
flush();
|
|
8119
|
+
this.formed = groups;
|
|
7807
8120
|
return groups;
|
|
7808
8121
|
}
|
|
8122
|
+
/** Release every group plan still kept for its start; each is built again when its group starts. */
|
|
8123
|
+
dropKeptPlans() {
|
|
8124
|
+
let dropped = false;
|
|
8125
|
+
for (const summary of this.formed) {
|
|
8126
|
+
const plan = summary.plan;
|
|
8127
|
+
if (plan === void 0) continue;
|
|
8128
|
+
summary.plan = void 0;
|
|
8129
|
+
this.ledger.release(plan.planBytes);
|
|
8130
|
+
this.planCharge -= plan.planBytes;
|
|
8131
|
+
dropped = true;
|
|
8132
|
+
}
|
|
8133
|
+
return dropped;
|
|
8134
|
+
}
|
|
7809
8135
|
/** The keys each operand is fetched at for `members`, and the keys the pass visits. */
|
|
7810
8136
|
buildGroup(members) {
|
|
7811
8137
|
combineWork.planBuilds++;
|
|
@@ -7848,6 +8174,11 @@ var Run = class {
|
|
|
7848
8174
|
pruned += this.operands[operand].indexKeys - set.size;
|
|
7849
8175
|
fixed += this.operands[operand].bytes;
|
|
7850
8176
|
}
|
|
8177
|
+
const named = /* @__PURE__ */ new Set();
|
|
8178
|
+
for (const o of members) for (const operand of o.compiled.operands) named.add(operand);
|
|
8179
|
+
for (const operand of named) {
|
|
8180
|
+
if (!bits.has(operand)) pruned += this.operands[operand].indexKeys;
|
|
8181
|
+
}
|
|
7851
8182
|
const visited = Uint16Array.from(visit.toKeys());
|
|
7852
8183
|
return {
|
|
7853
8184
|
outputs: members.map((o) => o.index),
|
|
@@ -7892,13 +8223,13 @@ var Run = class {
|
|
|
7892
8223
|
// ---- one group --------------------------------------------------------------------------------------------
|
|
7893
8224
|
async runGroup(indices, summary) {
|
|
7894
8225
|
const members = indices.map((i) => this.outputs[i]).filter((o) => o.status === "waiting");
|
|
7895
|
-
if (members.length === 0) return;
|
|
7896
8226
|
const kept = summary?.plan;
|
|
7897
8227
|
if (summary !== void 0) summary.plan = void 0;
|
|
7898
8228
|
if (kept !== void 0) {
|
|
7899
8229
|
this.ledger.release(kept.planBytes);
|
|
7900
8230
|
this.planCharge -= kept.planBytes;
|
|
7901
8231
|
}
|
|
8232
|
+
if (members.length === 0) return;
|
|
7902
8233
|
const plan = kept !== void 0 && members.length === indices.length ? kept : this.buildGroup(members);
|
|
7903
8234
|
const group = this.groups++;
|
|
7904
8235
|
const budget = this.limit;
|
|
@@ -7918,13 +8249,13 @@ var Run = class {
|
|
|
7918
8249
|
o.chunkCount = 0;
|
|
7919
8250
|
o.peak = 0;
|
|
7920
8251
|
}
|
|
7921
|
-
this.pruned += plan.pruned;
|
|
7922
8252
|
const fixed = plan.fixedBytes + plan.planBytes;
|
|
7923
|
-
if (!this.ledger.tryCharge(fixed)) {
|
|
8253
|
+
if (!this.ledger.tryCharge(fixed) && !(this.dropKeptPlans() && this.ledger.tryCharge(fixed))) {
|
|
7924
8254
|
const error = budgetError(this.req.maxBufferedBytes, "the operand streams");
|
|
7925
8255
|
for (const o of members) this.fail(o, error);
|
|
7926
8256
|
return;
|
|
7927
8257
|
}
|
|
8258
|
+
this.pruned += plan.pruned;
|
|
7928
8259
|
this.running = members;
|
|
7929
8260
|
try {
|
|
7930
8261
|
await this.pass(plan, members);
|
|
@@ -7973,12 +8304,19 @@ var Run = class {
|
|
|
7973
8304
|
});
|
|
7974
8305
|
}
|
|
7975
8306
|
const retry = (request) => this.gate.run(request);
|
|
7976
|
-
const
|
|
8307
|
+
const onRequestOf = (op) => (r) => {
|
|
7977
8308
|
this.rangeReads++;
|
|
7978
8309
|
this.rangeBytes += r.bytes;
|
|
8310
|
+
this.metrics?.onEvent({
|
|
8311
|
+
kind: "storage.get",
|
|
8312
|
+
namespace: op.spec.ref.namespace,
|
|
8313
|
+
segment: op.spec.ref.segment,
|
|
8314
|
+
bytes: r.bytes,
|
|
8315
|
+
ms: r.ms
|
|
8316
|
+
});
|
|
7979
8317
|
};
|
|
7980
8318
|
const streamOf = (c) => {
|
|
7981
|
-
c.stream ??= new ChunkStream(this.readKeys(c.op, c.keys, retry,
|
|
8319
|
+
c.stream ??= new ChunkStream(this.readKeys(c.op, c.keys, retry, onRequestOf(c.op)));
|
|
7982
8320
|
return c.stream;
|
|
7983
8321
|
};
|
|
7984
8322
|
const stop = (c) => {
|
|
@@ -8146,22 +8484,8 @@ var Run = class {
|
|
|
8146
8484
|
/** The operand's chunks at `keys` as a stream: the source's coalesced ranges, or chunk by chunk where it has none. */
|
|
8147
8485
|
readKeys(op, wanted, retry, onRequest) {
|
|
8148
8486
|
const source = this.source;
|
|
8487
|
+
const clock = this.clock;
|
|
8149
8488
|
const keys = Array.from(wanted);
|
|
8150
|
-
const held = op.spec.held;
|
|
8151
|
-
if (held !== void 0) {
|
|
8152
|
-
const stale = () => this.staleHeldError(op);
|
|
8153
|
-
return {
|
|
8154
|
-
// Each chunk is checked and read in one step with no wait between, so a release or an erasure cannot land after
|
|
8155
|
-
// the check and before the bytes.
|
|
8156
|
-
async *[Symbol.asyncIterator]() {
|
|
8157
|
-
for (const key of keys) {
|
|
8158
|
-
if (held.moved()) throw stale();
|
|
8159
|
-
const bytes = held.chunks.payload(key);
|
|
8160
|
-
yield { key, bytes: bytes === null ? null : new Uint8Array(bytes), version: null };
|
|
8161
|
-
}
|
|
8162
|
-
}
|
|
8163
|
-
};
|
|
8164
|
-
}
|
|
8165
8489
|
const ref = op.spec.ref;
|
|
8166
8490
|
if (source.getChunks !== void 0) {
|
|
8167
8491
|
return source.getChunks(ref, keys, { concurrency: this.req.concurrency, retry, onRequest });
|
|
@@ -8169,8 +8493,9 @@ var Run = class {
|
|
|
8169
8493
|
return {
|
|
8170
8494
|
async *[Symbol.asyncIterator]() {
|
|
8171
8495
|
for (const key of keys) {
|
|
8496
|
+
const startedAt = clock.now();
|
|
8172
8497
|
const bytes = await retry(() => source.getChunk({ ...ref, chunkKey: key }));
|
|
8173
|
-
onRequest({ bytes: bytes?.length ?? 0, ms: 0 });
|
|
8498
|
+
onRequest({ bytes: bytes?.length ?? 0, ms: Math.max(0, clock.now() - startedAt) });
|
|
8174
8499
|
yield { key, bytes, version: null };
|
|
8175
8500
|
}
|
|
8176
8501
|
}
|
|
@@ -8287,13 +8612,6 @@ var Run = class {
|
|
|
8287
8612
|
wake();
|
|
8288
8613
|
continue;
|
|
8289
8614
|
}
|
|
8290
|
-
const staleHeld = this.staleHeld(o);
|
|
8291
|
-
if (staleHeld !== void 0) {
|
|
8292
|
-
this.ledger.release(transient);
|
|
8293
|
-
this.fail(o, staleHeld);
|
|
8294
|
-
wake();
|
|
8295
|
-
continue;
|
|
8296
|
-
}
|
|
8297
8615
|
if (o.compiled.fed.length > 0 && this.erased()) {
|
|
8298
8616
|
this.ledger.release(transient);
|
|
8299
8617
|
this.fail(o, this.staleError(o));
|
|
@@ -8301,7 +8619,7 @@ var Run = class {
|
|
|
8301
8619
|
continue;
|
|
8302
8620
|
}
|
|
8303
8621
|
inFlight++;
|
|
8304
|
-
this.publishes++;
|
|
8622
|
+
if (this.req.dryRun !== true) this.publishes++;
|
|
8305
8623
|
const chunks = o.chunks;
|
|
8306
8624
|
try {
|
|
8307
8625
|
const value = await o.spec.publish(
|
|
@@ -8431,7 +8749,7 @@ function pinnedStillCurrent(spec, now) {
|
|
|
8431
8749
|
}
|
|
8432
8750
|
function liveMoved(op) {
|
|
8433
8751
|
if (op.endRow === void 0 || !op.read) return void 0;
|
|
8434
|
-
const was = tokenOfVersion(op.startVersion);
|
|
8752
|
+
const was = op.startVersion == null ? void 0 : tokenOfVersion(rowVersionOf(op.startVersion));
|
|
8435
8753
|
if (was !== void 0 && op.endRow !== null && !sameRow2(was, op.endRow.token)) return true;
|
|
8436
8754
|
return op.startGeneration === void 0 ? void 0 : op.endGeneration !== op.startGeneration;
|
|
8437
8755
|
}
|
|
@@ -8550,106 +8868,6 @@ function evaluate(node, kc) {
|
|
|
8550
8868
|
return acc;
|
|
8551
8869
|
}
|
|
8552
8870
|
|
|
8553
|
-
// src/core/combine-held.ts
|
|
8554
|
-
var PER_CHUNK_BYTES = 96;
|
|
8555
|
-
var HeldChunks = class {
|
|
8556
|
-
/** The chunk keys, ascending. */
|
|
8557
|
-
keys;
|
|
8558
|
-
/** How many ids each chunk holds. */
|
|
8559
|
-
cardinalities;
|
|
8560
|
-
/** What it holds in memory, for a call's resident-bytes ledger. */
|
|
8561
|
-
residentBytes;
|
|
8562
|
-
payloads;
|
|
8563
|
-
constructor(chunks) {
|
|
8564
|
-
const keys = [];
|
|
8565
|
-
const cards = [];
|
|
8566
|
-
const payloads = [];
|
|
8567
|
-
let bytes = 0;
|
|
8568
|
-
let last = -1;
|
|
8569
|
-
for (const chunk of chunks) {
|
|
8570
|
-
const { chunkKey, payload, cardinality } = chunk;
|
|
8571
|
-
if (!Number.isInteger(chunkKey) || chunkKey < 0 || chunkKey > MAX_REMAINDER || chunkKey <= last) {
|
|
8572
|
-
throw new ValidationError(
|
|
8573
|
-
`a held chunk's key must be an integer in [0, 65535] above the last`
|
|
8574
|
-
);
|
|
8575
|
-
}
|
|
8576
|
-
assertChunkCardinalityInRange(cardinality);
|
|
8577
|
-
last = chunkKey;
|
|
8578
|
-
keys.push(chunkKey);
|
|
8579
|
-
cards.push(cardinality);
|
|
8580
|
-
payloads.push(payload);
|
|
8581
|
-
bytes += payload.byteLength + PER_CHUNK_BYTES;
|
|
8582
|
-
}
|
|
8583
|
-
this.keys = Uint16Array.from(keys);
|
|
8584
|
-
this.cardinalities = Uint32Array.from(cards);
|
|
8585
|
-
this.payloads = payloads;
|
|
8586
|
-
this.residentBytes = bytes;
|
|
8587
|
-
}
|
|
8588
|
-
/** Whether it holds no id. */
|
|
8589
|
-
get empty() {
|
|
8590
|
-
return this.keys.length === 0;
|
|
8591
|
-
}
|
|
8592
|
-
/** Whether {@link release} ran. */
|
|
8593
|
-
get released() {
|
|
8594
|
-
return this.payloads === void 0;
|
|
8595
|
-
}
|
|
8596
|
-
/**
|
|
8597
|
-
* The payload of the chunk at `key`, `null` when it holds none there. A view into what is held: not to be written to.
|
|
8598
|
-
* @throws {ValidationError} once released.
|
|
8599
|
-
*/
|
|
8600
|
-
payload(key) {
|
|
8601
|
-
const payloads = this.payloads;
|
|
8602
|
-
if (payloads === void 0) {
|
|
8603
|
-
throw new ValidationError(
|
|
8604
|
-
"this memory operand was released: make another with store.memory()"
|
|
8605
|
-
);
|
|
8606
|
-
}
|
|
8607
|
-
let lo = 0;
|
|
8608
|
-
let hi = this.keys.length - 1;
|
|
8609
|
-
while (lo <= hi) {
|
|
8610
|
-
const mid = lo + hi >>> 1;
|
|
8611
|
-
const at = this.keys[mid];
|
|
8612
|
-
if (at === key) return payloads[mid];
|
|
8613
|
-
if (at < key) lo = mid + 1;
|
|
8614
|
-
else hi = mid - 1;
|
|
8615
|
-
}
|
|
8616
|
-
return null;
|
|
8617
|
-
}
|
|
8618
|
-
/** Zero and drop the bytes. Every later read throws; a second call does nothing. */
|
|
8619
|
-
release() {
|
|
8620
|
-
const payloads = this.payloads;
|
|
8621
|
-
if (payloads === void 0) return;
|
|
8622
|
-
this.payloads = void 0;
|
|
8623
|
-
for (const payload of payloads) payload.fill(0);
|
|
8624
|
-
}
|
|
8625
|
-
};
|
|
8626
|
-
async function prepareHeld(input, codec, clock) {
|
|
8627
|
-
if (typeof input === "object" && input !== null && !(Symbol.iterator in input) && !(Symbol.asyncIterator in input) && Object.prototype.hasOwnProperty.call(input, "ascending")) {
|
|
8628
|
-
throw new ValidationError(
|
|
8629
|
-
"a memory operand takes ids, { serialized } or { bitmap }, with no `ascending` option: ids that arrive in chunk-key order, one key at a time, are a feed of materializeMany"
|
|
8630
|
-
);
|
|
8631
|
-
}
|
|
8632
|
-
const real = plainUint32(input);
|
|
8633
|
-
if (real === "detached") {
|
|
8634
|
-
throw new ValidationError("a memory operand's ids are a Uint32Array over a detached buffer");
|
|
8635
|
-
}
|
|
8636
|
-
if (real === "out of bounds") {
|
|
8637
|
-
throw new ValidationError(
|
|
8638
|
-
"a memory operand's ids are a resizable-buffer Uint32Array whose view is out of bounds"
|
|
8639
|
-
);
|
|
8640
|
-
}
|
|
8641
|
-
const prepared = real !== void 0 ? new DecodedLoadInput(codec.fromValues(real)) : prepareLoadInput(input, codec, "a memory operand");
|
|
8642
|
-
const encoded = await encodedChunksOfInput(prepared, codec, clock);
|
|
8643
|
-
const tick = yieldEvery(clock, 256);
|
|
8644
|
-
const chunks = [];
|
|
8645
|
-
for (const chunk of encoded) {
|
|
8646
|
-
chunks.push(chunk);
|
|
8647
|
-
const pause = tick();
|
|
8648
|
-
if (pause !== null) await pause;
|
|
8649
|
-
}
|
|
8650
|
-
return new HeldChunks(chunks);
|
|
8651
|
-
}
|
|
8652
|
-
|
|
8653
8871
|
// src/core/rollback.ts
|
|
8654
8872
|
async function listGenerations(ref, deps) {
|
|
8655
8873
|
validateUserRef(ref);
|
|
@@ -8679,6 +8897,7 @@ async function currentDescription(ref, record, deps) {
|
|
|
8679
8897
|
}
|
|
8680
8898
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
8681
8899
|
validateUserRef(ref);
|
|
8900
|
+
checkedAuditSink(options.audit, "rollback");
|
|
8682
8901
|
if (!Number.isInteger(toGeneration) || toGeneration < 0) {
|
|
8683
8902
|
throw new ValidationError(
|
|
8684
8903
|
`rollback: generation must be a non-negative integer; got ${String(toGeneration)}`
|
|
@@ -8704,9 +8923,9 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
8704
8923
|
`rollback: generation ${toGeneration} of "${ref.segment}" is not in the bucket` + (available.length === 0 ? " (no generations remain)" : ` \u2014 present: ${available.join(", ")}`)
|
|
8705
8924
|
);
|
|
8706
8925
|
}
|
|
8707
|
-
if (record.currentGen
|
|
8926
|
+
if ((record.currentGen === null || toGeneration > record.currentGen) && options.allowForward !== true) {
|
|
8708
8927
|
throw new ValidationError(
|
|
8709
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" is above the current pointer (${record.currentGen}) \u2014 it may never have been published. Pass { allowForward: true } if you are undoing an earlier rollback.`
|
|
8928
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" is above the current pointer (${record.currentGen ?? "none"}) \u2014 it may never have been published. Pass { allowForward: true } if you are undoing an earlier rollback.`
|
|
8710
8929
|
);
|
|
8711
8930
|
}
|
|
8712
8931
|
const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
@@ -8824,7 +9043,21 @@ function describingSummary(record) {
|
|
|
8824
9043
|
// src/core/erase-id.ts
|
|
8825
9044
|
var HOLDS_CONCURRENCY = 4;
|
|
8826
9045
|
async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
9046
|
+
const first = {};
|
|
9047
|
+
const result = await eraseOnce(ref, id, deps, options, first);
|
|
9048
|
+
if (result.reason === "destroyed" && first.tombstoned === false) {
|
|
9049
|
+
const again = await eraseOnce(ref, id, deps, options, {});
|
|
9050
|
+
if (again.reason === "destroyed") return result;
|
|
9051
|
+
if (!again.erased) return again;
|
|
9052
|
+
const collected = [.../* @__PURE__ */ new Set([...result.collected, ...again.collected])].sort((a, b) => a - b);
|
|
9053
|
+
const newest = Math.max(result.fromGeneration ?? -1, again.fromGeneration ?? -1);
|
|
9054
|
+
return { ...again, collected, ...newest >= 0 ? { fromGeneration: newest } : {} };
|
|
9055
|
+
}
|
|
9056
|
+
return result;
|
|
9057
|
+
}
|
|
9058
|
+
async function eraseOnce(ref, id, deps, options, seen) {
|
|
8827
9059
|
validateUserRef(ref);
|
|
9060
|
+
checkedAuditSink(options.audit, "eraseIdFromSegment");
|
|
8828
9061
|
const { chunkKey, remainder } = splitId(id);
|
|
8829
9062
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
8830
9063
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
@@ -8832,21 +9065,23 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8832
9065
|
const read = (op) => retryRead(op, deps.readRetry);
|
|
8833
9066
|
const record = await deps.registry.get(ref);
|
|
8834
9067
|
if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
|
|
8835
|
-
|
|
8836
|
-
|
|
8837
|
-
|
|
8838
|
-
|
|
8839
|
-
const from = record.currentGen;
|
|
9068
|
+
const tombstoned = record.status === "destroyed";
|
|
9069
|
+
seen.tombstoned = tombstoned;
|
|
9070
|
+
const pointerless = !tombstoned && record.currentGen === null;
|
|
9071
|
+
const from = record.currentGen ?? -1;
|
|
8840
9072
|
const fromToken = record.token;
|
|
9073
|
+
let premise = record;
|
|
9074
|
+
let premiseToken = fromToken;
|
|
8841
9075
|
let aead;
|
|
8842
|
-
if (
|
|
9076
|
+
if (tombstoned) {
|
|
9077
|
+
} else if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
8843
9078
|
if (deps.keystore === void 0) {
|
|
8844
9079
|
throw new KeyUnavailableError(
|
|
8845
9080
|
`segment "${ref.segment}" is encrypted but eraseIdFromSegment has no keystore`
|
|
8846
9081
|
);
|
|
8847
9082
|
}
|
|
8848
9083
|
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
8849
|
-
} else if (deps.requireEncryption === true) {
|
|
9084
|
+
} else if (!pointerless && deps.requireEncryption === true) {
|
|
8850
9085
|
throw new ValidationError(`requireEncryption: segment "${ref.segment}" is cleartext`);
|
|
8851
9086
|
}
|
|
8852
9087
|
const cryptoAt = (generation2) => aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation2, scope) };
|
|
@@ -8863,8 +9098,9 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8863
9098
|
if (row === null) return "absent";
|
|
8864
9099
|
if (row.status === "destroyed") return "destroyed";
|
|
8865
9100
|
if (row.currentGen === null) return "no-generation";
|
|
8866
|
-
return row.currentGen === from && (row.token ===
|
|
9101
|
+
return row.currentGen === from && (row.token === premiseToken || onlyLeasesDiffer(premise, row)) ? null : "superseded";
|
|
8867
9102
|
};
|
|
9103
|
+
const sealedUnpublished = /* @__PURE__ */ new Set();
|
|
8868
9104
|
const holds = async (generation2) => {
|
|
8869
9105
|
const key2 = { ...base, generation: generation2 };
|
|
8870
9106
|
const chunkIn = (crypto) => read(async () => (await openGenerationReader(deps.storage, key2, crypto)).getChunk(chunkKey));
|
|
@@ -8873,7 +9109,15 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8873
9109
|
try {
|
|
8874
9110
|
bytes = await chunkIn(cryptoAt(generation2));
|
|
8875
9111
|
} catch (err) {
|
|
8876
|
-
if (
|
|
9112
|
+
if (cryptoAt(generation2) === void 0) {
|
|
9113
|
+
if ((tombstoned || pointerless) && await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
9114
|
+
if (tombstoned) return false;
|
|
9115
|
+
sealedUnpublished.add(generation2);
|
|
9116
|
+
return true;
|
|
9117
|
+
}
|
|
9118
|
+
throw err;
|
|
9119
|
+
}
|
|
9120
|
+
if (!isIntegrityError(err) || await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
8877
9121
|
throw err;
|
|
8878
9122
|
}
|
|
8879
9123
|
bytes = await chunkIn(void 0);
|
|
@@ -8919,6 +9163,51 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8919
9163
|
const cannotRemove = (generation2) => new WriteConflictError(
|
|
8920
9164
|
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which still holds the id and is still in the bucket; re-run`
|
|
8921
9165
|
);
|
|
9166
|
+
const fenceInFlight = async () => {
|
|
9167
|
+
const churn = leaseChurn({ clock: deps.clock, rng: deps.rng ?? deps.readRetry?.rng });
|
|
9168
|
+
let row = await deps.registry.get(ref);
|
|
9169
|
+
for (; ; ) {
|
|
9170
|
+
const verdict = rowVerdict(row);
|
|
9171
|
+
if (verdict !== null) return verdict;
|
|
9172
|
+
const current = row;
|
|
9173
|
+
const keptGens = current.keptGens !== void 0 && current.keptGens.length === 0 ? void 0 : [];
|
|
9174
|
+
try {
|
|
9175
|
+
const { token } = await deps.registry.compareAndSwap(ref, current.token, { keptGens });
|
|
9176
|
+
premise = { ...current, keptGens, token };
|
|
9177
|
+
premiseToken = token;
|
|
9178
|
+
return null;
|
|
9179
|
+
} catch (err) {
|
|
9180
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9181
|
+
}
|
|
9182
|
+
const now = await deps.registry.get(ref);
|
|
9183
|
+
const settled2 = await churn.settle(current, now, () => deps.registry.get(ref));
|
|
9184
|
+
if (settled2 === void 0) return rowVerdict(now) ?? "superseded";
|
|
9185
|
+
row = settled2.row;
|
|
9186
|
+
}
|
|
9187
|
+
};
|
|
9188
|
+
const unpublished = async () => {
|
|
9189
|
+
const generations = [];
|
|
9190
|
+
for await (const key2 of deps.storage.list(ref)) generations.push(key2.generation);
|
|
9191
|
+
const newestFirst = generations.sort((a, b) => b - a);
|
|
9192
|
+
const holder = (await holdsEach(newestFirst, true)).find((h) => h.held === true);
|
|
9193
|
+
if (holder === void 0)
|
|
9194
|
+
return { ...base, erased: false, reason: "no-generation", collected: [] };
|
|
9195
|
+
const what = sealedUnpublished.has(holder.generation) ? "is sealed under a key that load has not published, so it cannot be searched" : "holds the id";
|
|
9196
|
+
throw new WriteConflictError(
|
|
9197
|
+
`eraseIdFromSegment: segment "${ref.segment}" has no published generation, and an object a first load wrote and never published (generation ${holder.generation}) ${what}. It cannot be deleted while that load may still publish it: re-run once the segment is loaded, which makes the object collectable, or drop the segment`
|
|
9198
|
+
);
|
|
9199
|
+
};
|
|
9200
|
+
const underTombstone = async () => {
|
|
9201
|
+
const generations = [];
|
|
9202
|
+
for await (const key2 of deps.storage.list(ref)) generations.push(key2.generation);
|
|
9203
|
+
const newestFirst = generations.sort((a, b) => b - a);
|
|
9204
|
+
const holder = (await holdsEach(newestFirst, true)).find((h) => h.held === true);
|
|
9205
|
+
if (holder === void 0) return { ...base, erased: false, reason: "destroyed", collected: [] };
|
|
9206
|
+
const collected2 = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
9207
|
+
const left2 = await holderLeft(/* @__PURE__ */ new Set());
|
|
9208
|
+
if (left2 !== void 0) throw cannotRemove(left2);
|
|
9209
|
+
return { ...base, erased: true, fromGeneration: holder.generation, collected: collected2 };
|
|
9210
|
+
};
|
|
8922
9211
|
const notInCurrent = async () => {
|
|
8923
9212
|
const notMember = {
|
|
8924
9213
|
...base,
|
|
@@ -8955,8 +9244,9 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
8955
9244
|
const newest = holdersAbove[0] ?? holderBelow;
|
|
8956
9245
|
if (newest === void 0) return notMember;
|
|
8957
9246
|
const collected2 = [...await gcOrphanGenerations(ref, deps, { keep: 0 })];
|
|
8958
|
-
let moved = null;
|
|
9247
|
+
let moved = holdersAbove.length > 0 ? await fenceInFlight() : null;
|
|
8959
9248
|
for (const generation2 of holdersAbove) {
|
|
9249
|
+
if (moved !== null) break;
|
|
8960
9250
|
moved = rowVerdict(await deps.registry.get(ref));
|
|
8961
9251
|
if (moved !== null) break;
|
|
8962
9252
|
await deps.storage.delete({ ...base, generation: generation2 });
|
|
@@ -9021,10 +9311,17 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
9021
9311
|
};
|
|
9022
9312
|
const discardRefused = async (written) => {
|
|
9023
9313
|
const row = await deps.registry.get(ref);
|
|
9024
|
-
if (row
|
|
9025
|
-
|
|
9314
|
+
if (row !== null && row.status === "destroyed") {
|
|
9315
|
+
await deps.storage.delete({ ...base, generation: written });
|
|
9316
|
+
return row;
|
|
9317
|
+
}
|
|
9318
|
+
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
9319
|
+
if (row.currentGen <= from || row.currentGen >= written) return row;
|
|
9026
9320
|
await deps.storage.delete({ ...base, generation: written });
|
|
9321
|
+
return row;
|
|
9027
9322
|
};
|
|
9323
|
+
if (tombstoned) return await underTombstone();
|
|
9324
|
+
if (pointerless) return await unpublished();
|
|
9028
9325
|
const staged = await stage();
|
|
9029
9326
|
if ("erased" in staged) {
|
|
9030
9327
|
if (!staged.erased && staged.generation !== void 0) await discardRefused(staged.generation);
|
|
@@ -9048,11 +9345,11 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
9048
9345
|
rng: deps.rng ?? deps.readRetry?.rng
|
|
9049
9346
|
});
|
|
9050
9347
|
if (!published) {
|
|
9051
|
-
await discardRefused(generation);
|
|
9348
|
+
const now = await discardRefused(generation);
|
|
9052
9349
|
return {
|
|
9053
9350
|
...base,
|
|
9054
9351
|
erased: false,
|
|
9055
|
-
reason: "superseded",
|
|
9352
|
+
reason: rowVerdict(now) ?? "superseded",
|
|
9056
9353
|
fromGeneration: from,
|
|
9057
9354
|
generation,
|
|
9058
9355
|
collected: []
|
|
@@ -9111,9 +9408,12 @@ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
|
9111
9408
|
var KEY_BYTES = 32;
|
|
9112
9409
|
var NONCE_BYTES2 = 12;
|
|
9113
9410
|
var TAG_BYTES2 = 16;
|
|
9411
|
+
var isBytes = (v) => Object.prototype.toString.call(v) === "[object Uint8Array]";
|
|
9114
9412
|
var NodeAead = class {
|
|
9115
9413
|
key;
|
|
9116
9414
|
constructor(key) {
|
|
9415
|
+
if (!isBytes(key))
|
|
9416
|
+
throw new ValidationError("AEAD key must be a Uint8Array of key bytes, not text");
|
|
9117
9417
|
if (key.length !== KEY_BYTES)
|
|
9118
9418
|
throw new ValidationError(`AEAD key must be ${KEY_BYTES} bytes, got ${key.length}`);
|
|
9119
9419
|
this.key = Buffer.from(key);
|
|
@@ -9161,6 +9461,8 @@ var InProcessKeystore = class {
|
|
|
9161
9461
|
const entries = Object.entries(options.keys);
|
|
9162
9462
|
if (entries.length === 0) throw new ValidationError("keystore needs at least one KEK");
|
|
9163
9463
|
for (const [id, k] of entries) {
|
|
9464
|
+
if (!isBytes(k))
|
|
9465
|
+
throw new ValidationError(`KEK "${id}" must be a Uint8Array of key bytes, not text`);
|
|
9164
9466
|
if (k.length !== KEY_BYTES)
|
|
9165
9467
|
throw new ValidationError(`KEK "${id}" must be ${KEY_BYTES} bytes, got ${k.length}`);
|
|
9166
9468
|
}
|
|
@@ -9198,20 +9500,186 @@ var InProcessKeystore = class {
|
|
|
9198
9500
|
}
|
|
9199
9501
|
}
|
|
9200
9502
|
if (anyKekHeld) throw lastUnwrapError;
|
|
9201
|
-
const have = [...this.keys.keys()].join(", ") || "(none)";
|
|
9202
9503
|
const need = wrapped.map((w) => w.keyId).join(", ") || "(none)";
|
|
9203
9504
|
throw new KeyUnavailableError(
|
|
9204
|
-
`no held KEK can unwrap this DEK \u2014
|
|
9505
|
+
`no held KEK can unwrap this DEK \u2014 the wrappings reference [${need}]. Restore the missing KEK (or its recovery KEK), or rebuild the segment from source.`
|
|
9205
9506
|
);
|
|
9206
9507
|
}
|
|
9207
9508
|
};
|
|
9208
9509
|
|
|
9510
|
+
// src/core/retention.ts
|
|
9511
|
+
var EXPIRES_AT = "expiresAt";
|
|
9512
|
+
var RETENTION_CAS_ATTEMPTS = 5;
|
|
9513
|
+
var MIN_EXPIRES_AT_MS = 1e12;
|
|
9514
|
+
var SWEEP_MARK = "retiredBySweepAt";
|
|
9515
|
+
function withoutSweepMark(meta) {
|
|
9516
|
+
if (meta === void 0 || meta === null || typeof meta !== "object" || Array.isArray(meta))
|
|
9517
|
+
return null;
|
|
9518
|
+
if (!Object.prototype.hasOwnProperty.call(meta, SWEEP_MARK)) return null;
|
|
9519
|
+
const rest = Object.fromEntries(Object.entries(meta).filter(([key]) => key !== SWEEP_MARK));
|
|
9520
|
+
return Object.keys(rest).length === 0 ? void 0 : rest;
|
|
9521
|
+
}
|
|
9522
|
+
var MAX_DATE_MS = 864e13;
|
|
9523
|
+
function validateRetentionPolicy(policy) {
|
|
9524
|
+
if (policy === null || typeof policy !== "object" || Array.isArray(policy)) {
|
|
9525
|
+
throw new ValidationError(
|
|
9526
|
+
`retention policy must be an object such as { expiresAt }; got ${policy === null ? "null" : Array.isArray(policy) ? "an array" : `a ${typeof policy}`}`
|
|
9527
|
+
);
|
|
9528
|
+
}
|
|
9529
|
+
const { expiresAt } = policy;
|
|
9530
|
+
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
9531
|
+
throw new ValidationError(
|
|
9532
|
+
`retention.expiresAt must be an integer epoch-ms; got ${String(expiresAt)}`
|
|
9533
|
+
);
|
|
9534
|
+
}
|
|
9535
|
+
if (expiresAt > MAX_DATE_MS) {
|
|
9536
|
+
throw new ValidationError(
|
|
9537
|
+
`retention.expiresAt (${expiresAt}) is past the latest time a date can name (${MAX_DATE_MS}); to keep a segment, leave it without a policy or call clearRetention`
|
|
9538
|
+
);
|
|
9539
|
+
}
|
|
9540
|
+
if (expiresAt < MIN_EXPIRES_AT_MS) {
|
|
9541
|
+
throw new ValidationError(
|
|
9542
|
+
`retention.expiresAt (${expiresAt}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds, which would read as "already expired" and retire the segment on the next sweep. Use \`Date.now() + days * 86_400_000\`. To retire a segment now, call \`dropSegment\`.`
|
|
9543
|
+
);
|
|
9544
|
+
}
|
|
9545
|
+
}
|
|
9546
|
+
function readRetentionPolicy(meta) {
|
|
9547
|
+
if (meta === void 0) return null;
|
|
9548
|
+
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "invalid";
|
|
9549
|
+
if (!(EXPIRES_AT in meta)) return null;
|
|
9550
|
+
const raw = meta[EXPIRES_AT];
|
|
9551
|
+
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < MIN_EXPIRES_AT_MS)
|
|
9552
|
+
return "invalid";
|
|
9553
|
+
return { expiresAt: raw };
|
|
9554
|
+
}
|
|
9555
|
+
async function getSegmentRetention(ref, deps) {
|
|
9556
|
+
validateUserRef(ref);
|
|
9557
|
+
const record = await deps.registry.get(ref);
|
|
9558
|
+
if (record === null || record.status === "destroyed") return null;
|
|
9559
|
+
return readRetentionPolicy(record.retention);
|
|
9560
|
+
}
|
|
9561
|
+
function readExpiresAt(record) {
|
|
9562
|
+
const parsed = record === null ? null : readRetentionPolicy(record.retention);
|
|
9563
|
+
return parsed === null || parsed === "invalid" ? void 0 : parsed.expiresAt;
|
|
9564
|
+
}
|
|
9565
|
+
async function reindex(ref, deps, expiresAt, previousExpiresAt) {
|
|
9566
|
+
if (!canIndex(ref)) return false;
|
|
9567
|
+
const nextBucket = expiresAt === void 0 ? void 0 : dueBucket(expiresAt);
|
|
9568
|
+
const oldBucket = previousExpiresAt === void 0 ? void 0 : dueBucket(previousExpiresAt);
|
|
9569
|
+
let indexed = expiresAt === void 0;
|
|
9570
|
+
if (nextBucket !== void 0) {
|
|
9571
|
+
try {
|
|
9572
|
+
await deps.registry.create(dueIndexRef(nextBucket, ref), { currentGen: null });
|
|
9573
|
+
indexed = true;
|
|
9574
|
+
} catch (err) {
|
|
9575
|
+
if (isWriteConflictError(err)) indexed = true;
|
|
9576
|
+
}
|
|
9577
|
+
}
|
|
9578
|
+
if (oldBucket !== void 0 && oldBucket !== nextBucket) {
|
|
9579
|
+
try {
|
|
9580
|
+
await deps.registry.delete(dueIndexRef(oldBucket, ref));
|
|
9581
|
+
} catch {
|
|
9582
|
+
}
|
|
9583
|
+
}
|
|
9584
|
+
return indexed;
|
|
9585
|
+
}
|
|
9586
|
+
async function setSegmentRetention(ref, deps, policy) {
|
|
9587
|
+
validateUserRef(ref);
|
|
9588
|
+
validateRetentionPolicy(policy);
|
|
9589
|
+
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9590
|
+
let previousExpiresAt;
|
|
9591
|
+
const churn = leaseChurn(deps);
|
|
9592
|
+
let lost;
|
|
9593
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
9594
|
+
let record = await deps.registry.get(ref);
|
|
9595
|
+
if (lost !== void 0) {
|
|
9596
|
+
const settled2 = await churn.settle(lost, record, () => deps.registry.get(ref));
|
|
9597
|
+
if (settled2 !== void 0) {
|
|
9598
|
+
record = settled2.row;
|
|
9599
|
+
attempt -= 1;
|
|
9600
|
+
}
|
|
9601
|
+
}
|
|
9602
|
+
try {
|
|
9603
|
+
previousExpiresAt = readExpiresAt(record);
|
|
9604
|
+
if (record === null) {
|
|
9605
|
+
await deps.registry.create(ref, {
|
|
9606
|
+
currentGen: null,
|
|
9607
|
+
retention: { [EXPIRES_AT]: policy.expiresAt }
|
|
9608
|
+
});
|
|
9609
|
+
return {
|
|
9610
|
+
...base,
|
|
9611
|
+
expiresAt: policy.expiresAt,
|
|
9612
|
+
createdRow: true,
|
|
9613
|
+
indexed: await reindex(ref, deps, policy.expiresAt, void 0)
|
|
9614
|
+
};
|
|
9615
|
+
}
|
|
9616
|
+
assertNotDestroyed(record, ref);
|
|
9617
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
9618
|
+
retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt }
|
|
9619
|
+
});
|
|
9620
|
+
return {
|
|
9621
|
+
...base,
|
|
9622
|
+
expiresAt: policy.expiresAt,
|
|
9623
|
+
createdRow: false,
|
|
9624
|
+
indexed: await reindex(ref, deps, policy.expiresAt, previousExpiresAt)
|
|
9625
|
+
};
|
|
9626
|
+
} catch (err) {
|
|
9627
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9628
|
+
lost = record ?? void 0;
|
|
9629
|
+
}
|
|
9630
|
+
}
|
|
9631
|
+
throw new WriteConflictError(
|
|
9632
|
+
`setRetention: contention writing the retention policy for segment "${ref.segment}"`
|
|
9633
|
+
);
|
|
9634
|
+
}
|
|
9635
|
+
async function clearSegmentRetention(ref, deps) {
|
|
9636
|
+
validateUserRef(ref);
|
|
9637
|
+
const churn = leaseChurn(deps);
|
|
9638
|
+
let lost;
|
|
9639
|
+
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
9640
|
+
let record = await deps.registry.get(ref);
|
|
9641
|
+
if (lost !== void 0) {
|
|
9642
|
+
const settled2 = await churn.settle(lost, record, () => deps.registry.get(ref));
|
|
9643
|
+
if (settled2 !== void 0) {
|
|
9644
|
+
record = settled2.row;
|
|
9645
|
+
attempt -= 1;
|
|
9646
|
+
}
|
|
9647
|
+
}
|
|
9648
|
+
if (record === null) return false;
|
|
9649
|
+
if (record.status === "destroyed") return false;
|
|
9650
|
+
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) return false;
|
|
9651
|
+
const rest = { ...record.retention };
|
|
9652
|
+
delete rest[EXPIRES_AT];
|
|
9653
|
+
try {
|
|
9654
|
+
await deps.registry.compareAndSwap(ref, record.token, {
|
|
9655
|
+
retention: Object.keys(rest).length === 0 ? void 0 : rest
|
|
9656
|
+
});
|
|
9657
|
+
await reindex(ref, deps, void 0, readExpiresAt(record));
|
|
9658
|
+
return true;
|
|
9659
|
+
} catch (err) {
|
|
9660
|
+
if (!isWriteConflictError(err)) throw err;
|
|
9661
|
+
lost = record;
|
|
9662
|
+
}
|
|
9663
|
+
}
|
|
9664
|
+
throw new WriteConflictError(
|
|
9665
|
+
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"`
|
|
9666
|
+
);
|
|
9667
|
+
}
|
|
9668
|
+
function assertNotDestroyed(record, ref) {
|
|
9669
|
+
if (record.status === "destroyed") {
|
|
9670
|
+
throw new ValidationError(
|
|
9671
|
+
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to set a retention policy on a tombstone; there is nothing left to retire. Use a new segment.`
|
|
9672
|
+
);
|
|
9673
|
+
}
|
|
9674
|
+
}
|
|
9675
|
+
|
|
9209
9676
|
// src/core/erasure.ts
|
|
9210
9677
|
var MAX_CAS_ATTEMPTS = 8;
|
|
9211
9678
|
var ERASE_CONCURRENCY = 8;
|
|
9212
9679
|
var MAX_STORAGE_SWEEPS = 3;
|
|
9213
9680
|
async function destroySegment(ref, deps, options) {
|
|
9214
9681
|
validateUserRef(ref);
|
|
9682
|
+
checkedAuditSink(options.audit, "destroySegment");
|
|
9215
9683
|
if (options.confirmSegment !== ref.segment) {
|
|
9216
9684
|
throw new ValidationError(
|
|
9217
9685
|
`destroySegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental crypto-shred)`
|
|
@@ -9232,6 +9700,7 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9232
9700
|
throw new ValidationError("eraseNamespace: namespace must be a non-empty string");
|
|
9233
9701
|
}
|
|
9234
9702
|
validateUserNamespace(namespace);
|
|
9703
|
+
checkedAuditSink(options.audit, "eraseNamespace");
|
|
9235
9704
|
if (options.confirmNamespace !== namespace) {
|
|
9236
9705
|
throw new ValidationError(
|
|
9237
9706
|
`eraseNamespace: confirmNamespace must equal the namespace "${namespace}" (guard against accidental erasure)`
|
|
@@ -9273,7 +9742,11 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
9273
9742
|
return { destroyed };
|
|
9274
9743
|
}
|
|
9275
9744
|
async function dropSegment(ref, deps, options) {
|
|
9745
|
+
return await dropSegmentFor(ref, deps, options, void 0);
|
|
9746
|
+
}
|
|
9747
|
+
async function dropSegmentFor(ref, deps, options, sweep) {
|
|
9276
9748
|
validateUserRef(ref);
|
|
9749
|
+
checkedAuditSink(options.audit, "dropSegment");
|
|
9277
9750
|
if (options.confirmSegment !== ref.segment) {
|
|
9278
9751
|
throw new ValidationError(
|
|
9279
9752
|
`dropSegment: confirmSegment must equal the segment name "${ref.segment}" (guard against accidental deletion)`
|
|
@@ -9294,7 +9767,9 @@ async function dropSegment(ref, deps, options) {
|
|
|
9294
9767
|
reason: record === null ? wouldDelete.length === 0 ? "absent" : void 0 : record.status === "destroyed" ? "already" : void 0
|
|
9295
9768
|
};
|
|
9296
9769
|
}
|
|
9297
|
-
|
|
9770
|
+
const first = await shredSegment(ref, deps, true, "dropSegment", sweep);
|
|
9771
|
+
if (first === null) return null;
|
|
9772
|
+
let shred = first;
|
|
9298
9773
|
if (shred.reason === "absent") {
|
|
9299
9774
|
const orphans = await listGenerations2(deps.storage, ref);
|
|
9300
9775
|
if (orphans.length === 0) {
|
|
@@ -9371,7 +9846,7 @@ async function listGenerations2(storage, ref) {
|
|
|
9371
9846
|
for await (const key of storage.list(ref)) generations.push(key.generation);
|
|
9372
9847
|
return generations.sort((a, b) => a - b);
|
|
9373
9848
|
}
|
|
9374
|
-
async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
9849
|
+
async function shredSegment(ref, deps, allowCleartext, op = "destroySegment", sweep) {
|
|
9375
9850
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9376
9851
|
const churn = leaseChurn(deps);
|
|
9377
9852
|
let lost;
|
|
@@ -9384,6 +9859,9 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
9384
9859
|
attempt -= 1;
|
|
9385
9860
|
}
|
|
9386
9861
|
}
|
|
9862
|
+
if (sweep !== void 0 && (record === null || record.status === "destroyed" || !sweep.wanted(record))) {
|
|
9863
|
+
return null;
|
|
9864
|
+
}
|
|
9387
9865
|
if (record === null) {
|
|
9388
9866
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "absent" };
|
|
9389
9867
|
}
|
|
@@ -9394,6 +9872,7 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
9394
9872
|
if (!encrypted && !allowCleartext) {
|
|
9395
9873
|
return { ...base, destroyed: false, cryptoShredded: false, reason: "cleartext" };
|
|
9396
9874
|
}
|
|
9875
|
+
const unmarked = sweep === void 0 ? withoutSweepMark(record.retention) : null;
|
|
9397
9876
|
try {
|
|
9398
9877
|
await deps.registry.compareAndSwap(ref, record.token, {
|
|
9399
9878
|
status: "destroyed",
|
|
@@ -9404,7 +9883,10 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
9404
9883
|
// the wrappings; clear, they would outlive the segment on its tombstone.
|
|
9405
9884
|
summary: void 0,
|
|
9406
9885
|
// A tombstone holds nothing: a lease names a generation of a segment that now resolves none.
|
|
9407
|
-
leases: void 0
|
|
9886
|
+
leases: void 0,
|
|
9887
|
+
// A sweep's mark, in this same write, so no other tombstone ever carries it: any other tombstone write removes
|
|
9888
|
+
// one a restored or hand-edited row carried.
|
|
9889
|
+
...sweep !== void 0 ? { retention: sweep.stamp(record.retention) } : unmarked === null ? {} : { retention: unmarked }
|
|
9408
9890
|
});
|
|
9409
9891
|
return { ...base, destroyed: true, cryptoShredded: encrypted };
|
|
9410
9892
|
} catch (err) {
|
|
@@ -9415,153 +9897,6 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
9415
9897
|
throw new WriteConflictError(`${op}: contention shredding "${ref.segment}" \u2014 retry`);
|
|
9416
9898
|
}
|
|
9417
9899
|
|
|
9418
|
-
// src/core/retention.ts
|
|
9419
|
-
var EXPIRES_AT = "expiresAt";
|
|
9420
|
-
var RETENTION_CAS_ATTEMPTS = 5;
|
|
9421
|
-
var MIN_EXPIRES_AT_MS = 1e12;
|
|
9422
|
-
function validateRetentionPolicy(policy) {
|
|
9423
|
-
const { expiresAt } = policy;
|
|
9424
|
-
if (typeof expiresAt !== "number" || !Number.isInteger(expiresAt)) {
|
|
9425
|
-
throw new ValidationError(
|
|
9426
|
-
`retention.expiresAt must be an integer epoch-ms; got ${String(expiresAt)}`
|
|
9427
|
-
);
|
|
9428
|
-
}
|
|
9429
|
-
if (expiresAt < MIN_EXPIRES_AT_MS) {
|
|
9430
|
-
throw new ValidationError(
|
|
9431
|
-
`retention.expiresAt (${expiresAt}) is before ${MIN_EXPIRES_AT_MS} \u2014 that is almost certainly epoch SECONDS rather than milliseconds, which would read as "already expired" and retire the segment on the next sweep. Use \`Date.now() + days * 86_400_000\`. To retire a segment now, call \`dropSegment\`.`
|
|
9432
|
-
);
|
|
9433
|
-
}
|
|
9434
|
-
}
|
|
9435
|
-
function readRetentionPolicy(meta) {
|
|
9436
|
-
if (meta === void 0) return null;
|
|
9437
|
-
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) return "invalid";
|
|
9438
|
-
if (!(EXPIRES_AT in meta)) return null;
|
|
9439
|
-
const raw = meta[EXPIRES_AT];
|
|
9440
|
-
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < MIN_EXPIRES_AT_MS)
|
|
9441
|
-
return "invalid";
|
|
9442
|
-
return { expiresAt: raw };
|
|
9443
|
-
}
|
|
9444
|
-
async function getSegmentRetention(ref, deps) {
|
|
9445
|
-
validateUserRef(ref);
|
|
9446
|
-
const record = await deps.registry.get(ref);
|
|
9447
|
-
if (record === null || record.status === "destroyed") return null;
|
|
9448
|
-
return readRetentionPolicy(record.retention);
|
|
9449
|
-
}
|
|
9450
|
-
function readExpiresAt(record) {
|
|
9451
|
-
const parsed = record === null ? null : readRetentionPolicy(record.retention);
|
|
9452
|
-
return parsed === null || parsed === "invalid" ? void 0 : parsed.expiresAt;
|
|
9453
|
-
}
|
|
9454
|
-
async function reindex(ref, deps, expiresAt, previousExpiresAt) {
|
|
9455
|
-
if (!canIndex(ref)) return false;
|
|
9456
|
-
const nextBucket = expiresAt === void 0 ? void 0 : dueBucket(expiresAt);
|
|
9457
|
-
const oldBucket = previousExpiresAt === void 0 ? void 0 : dueBucket(previousExpiresAt);
|
|
9458
|
-
let indexed = expiresAt === void 0;
|
|
9459
|
-
if (nextBucket !== void 0) {
|
|
9460
|
-
try {
|
|
9461
|
-
await deps.registry.create(dueIndexRef(nextBucket, ref), { currentGen: null });
|
|
9462
|
-
indexed = true;
|
|
9463
|
-
} catch (err) {
|
|
9464
|
-
if (isWriteConflictError(err)) indexed = true;
|
|
9465
|
-
}
|
|
9466
|
-
}
|
|
9467
|
-
if (oldBucket !== void 0 && oldBucket !== nextBucket) {
|
|
9468
|
-
try {
|
|
9469
|
-
await deps.registry.delete(dueIndexRef(oldBucket, ref));
|
|
9470
|
-
} catch {
|
|
9471
|
-
}
|
|
9472
|
-
}
|
|
9473
|
-
return indexed;
|
|
9474
|
-
}
|
|
9475
|
-
async function setSegmentRetention(ref, deps, policy) {
|
|
9476
|
-
validateUserRef(ref);
|
|
9477
|
-
validateRetentionPolicy(policy);
|
|
9478
|
-
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9479
|
-
let previousExpiresAt;
|
|
9480
|
-
const churn = leaseChurn(deps);
|
|
9481
|
-
let lost;
|
|
9482
|
-
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
9483
|
-
let record = await deps.registry.get(ref);
|
|
9484
|
-
if (lost !== void 0) {
|
|
9485
|
-
const settled2 = await churn.settle(lost, record, () => deps.registry.get(ref));
|
|
9486
|
-
if (settled2 !== void 0) {
|
|
9487
|
-
record = settled2.row;
|
|
9488
|
-
attempt -= 1;
|
|
9489
|
-
}
|
|
9490
|
-
}
|
|
9491
|
-
try {
|
|
9492
|
-
previousExpiresAt = readExpiresAt(record);
|
|
9493
|
-
if (record === null) {
|
|
9494
|
-
await deps.registry.create(ref, {
|
|
9495
|
-
currentGen: null,
|
|
9496
|
-
retention: { [EXPIRES_AT]: policy.expiresAt }
|
|
9497
|
-
});
|
|
9498
|
-
return {
|
|
9499
|
-
...base,
|
|
9500
|
-
expiresAt: policy.expiresAt,
|
|
9501
|
-
createdRow: true,
|
|
9502
|
-
indexed: await reindex(ref, deps, policy.expiresAt, void 0)
|
|
9503
|
-
};
|
|
9504
|
-
}
|
|
9505
|
-
assertNotDestroyed(record, ref);
|
|
9506
|
-
await deps.registry.compareAndSwap(ref, record.token, {
|
|
9507
|
-
retention: { ...record.retention, [EXPIRES_AT]: policy.expiresAt }
|
|
9508
|
-
});
|
|
9509
|
-
return {
|
|
9510
|
-
...base,
|
|
9511
|
-
expiresAt: policy.expiresAt,
|
|
9512
|
-
createdRow: false,
|
|
9513
|
-
indexed: await reindex(ref, deps, policy.expiresAt, previousExpiresAt)
|
|
9514
|
-
};
|
|
9515
|
-
} catch (err) {
|
|
9516
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9517
|
-
lost = record ?? void 0;
|
|
9518
|
-
}
|
|
9519
|
-
}
|
|
9520
|
-
throw new WriteConflictError(
|
|
9521
|
-
`setRetention: contention writing the retention policy for segment "${ref.segment}"`
|
|
9522
|
-
);
|
|
9523
|
-
}
|
|
9524
|
-
async function clearSegmentRetention(ref, deps) {
|
|
9525
|
-
validateUserRef(ref);
|
|
9526
|
-
const churn = leaseChurn(deps);
|
|
9527
|
-
let lost;
|
|
9528
|
-
for (let attempt = 0; attempt < RETENTION_CAS_ATTEMPTS; attempt += 1) {
|
|
9529
|
-
let record = await deps.registry.get(ref);
|
|
9530
|
-
if (lost !== void 0) {
|
|
9531
|
-
const settled2 = await churn.settle(lost, record, () => deps.registry.get(ref));
|
|
9532
|
-
if (settled2 !== void 0) {
|
|
9533
|
-
record = settled2.row;
|
|
9534
|
-
attempt -= 1;
|
|
9535
|
-
}
|
|
9536
|
-
}
|
|
9537
|
-
if (record === null) return false;
|
|
9538
|
-
if (record.status === "destroyed") return false;
|
|
9539
|
-
if (record.retention === void 0 || !(EXPIRES_AT in record.retention)) return false;
|
|
9540
|
-
const rest = { ...record.retention };
|
|
9541
|
-
delete rest[EXPIRES_AT];
|
|
9542
|
-
try {
|
|
9543
|
-
await deps.registry.compareAndSwap(ref, record.token, {
|
|
9544
|
-
retention: Object.keys(rest).length === 0 ? void 0 : rest
|
|
9545
|
-
});
|
|
9546
|
-
await reindex(ref, deps, void 0, readExpiresAt(record));
|
|
9547
|
-
return true;
|
|
9548
|
-
} catch (err) {
|
|
9549
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9550
|
-
lost = record;
|
|
9551
|
-
}
|
|
9552
|
-
}
|
|
9553
|
-
throw new WriteConflictError(
|
|
9554
|
-
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"`
|
|
9555
|
-
);
|
|
9556
|
-
}
|
|
9557
|
-
function assertNotDestroyed(record, ref) {
|
|
9558
|
-
if (record.status === "destroyed") {
|
|
9559
|
-
throw new ValidationError(
|
|
9560
|
-
`segment "${ref.segment}" is destroyed (crypto-shredded) \u2014 refusing to set a retention policy on a tombstone; there is nothing left to retire. Use a new segment.`
|
|
9561
|
-
);
|
|
9562
|
-
}
|
|
9563
|
-
}
|
|
9564
|
-
|
|
9565
9900
|
// src/core/retention-sweep.ts
|
|
9566
9901
|
var DEFAULT_RETIRE_LIMIT = 100;
|
|
9567
9902
|
var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
@@ -9590,7 +9925,7 @@ async function rowsFromDueIndex(registry, options) {
|
|
|
9590
9925
|
continue;
|
|
9591
9926
|
}
|
|
9592
9927
|
if (gone.has(key)) {
|
|
9593
|
-
litter.push(found);
|
|
9928
|
+
if (litter.length < options.maxScanSegments) litter.push(found);
|
|
9594
9929
|
continue;
|
|
9595
9930
|
}
|
|
9596
9931
|
if (rows.length >= options.maxScanSegments) {
|
|
@@ -9600,8 +9935,8 @@ async function rowsFromDueIndex(registry, options) {
|
|
|
9600
9935
|
}
|
|
9601
9936
|
const live = await registry.get(ref);
|
|
9602
9937
|
if (live === null) {
|
|
9603
|
-
gone.add(key);
|
|
9604
|
-
litter.push(found);
|
|
9938
|
+
if (gone.size < options.maxScanSegments) gone.add(key);
|
|
9939
|
+
if (litter.length < options.maxScanSegments) litter.push(found);
|
|
9605
9940
|
continue;
|
|
9606
9941
|
}
|
|
9607
9942
|
rows.push(live);
|
|
@@ -9667,6 +10002,23 @@ async function forgetPurgePointers(registry, ref, purgeAt, found, onFault) {
|
|
|
9667
10002
|
}
|
|
9668
10003
|
}
|
|
9669
10004
|
async function retireExpired(deps, options) {
|
|
10005
|
+
checkedAuditSink(options.audit, "retireExpired");
|
|
10006
|
+
if (options.shards !== void 0) {
|
|
10007
|
+
const total = options.totalShards;
|
|
10008
|
+
if (typeof total !== "number" || !Number.isInteger(total) || total < 1) {
|
|
10009
|
+
throw new ValidationError("retireExpired: shards needs totalShards, an integer of 1 or more");
|
|
10010
|
+
}
|
|
10011
|
+
if (!Array.isArray(options.shards)) {
|
|
10012
|
+
throw new ValidationError("retireExpired: shards must be an array of shard numbers");
|
|
10013
|
+
}
|
|
10014
|
+
for (const shard of options.shards) {
|
|
10015
|
+
if (typeof shard !== "number" || !Number.isInteger(shard) || shard < 0 || shard >= total) {
|
|
10016
|
+
throw new ValidationError(
|
|
10017
|
+
`retireExpired: a shard is an integer from 0 to totalShards - 1 (${total - 1}); got ${String(shard)}`
|
|
10018
|
+
);
|
|
10019
|
+
}
|
|
10020
|
+
}
|
|
10021
|
+
}
|
|
9670
10022
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
9671
10023
|
const now = options.now;
|
|
9672
10024
|
if (!Number.isFinite(now)) {
|
|
@@ -9742,8 +10094,24 @@ async function retireExpired(deps, options) {
|
|
|
9742
10094
|
attempted += 1;
|
|
9743
10095
|
try {
|
|
9744
10096
|
if (!await isFullyReclaimed(deps, ref)) {
|
|
9745
|
-
|
|
10097
|
+
let collectFault;
|
|
10098
|
+
if (!dryRun) {
|
|
10099
|
+
await gcOrphanGenerations(ref, deps).catch((error) => {
|
|
10100
|
+
collectFault = { error };
|
|
10101
|
+
});
|
|
10102
|
+
}
|
|
9746
10103
|
if (!await isFullyReclaimed(deps, ref)) {
|
|
10104
|
+
const lostRace = collectFault !== void 0 && isWriteConflictError(collectFault.error);
|
|
10105
|
+
if (!dryRun && !lostRace) {
|
|
10106
|
+
attempted -= 1;
|
|
10107
|
+
purgeRun += 1;
|
|
10108
|
+
if (purgeRun >= MAX_CONSECUTIVE_PURGE_FAULTS) purging = false;
|
|
10109
|
+
if (collectFault !== void 0) noteFault(collectFault.error);
|
|
10110
|
+
else {
|
|
10111
|
+
purgeFaults += 1;
|
|
10112
|
+
firstPurgeFault ??= "failed: the tombstone's objects were still there after its collection";
|
|
10113
|
+
}
|
|
10114
|
+
}
|
|
9747
10115
|
entries.push({ ...base, action: "skipped", reason: "tombstone-not-empty" });
|
|
9748
10116
|
continue;
|
|
9749
10117
|
}
|
|
@@ -9796,11 +10164,23 @@ async function retireExpired(deps, options) {
|
|
|
9796
10164
|
continue;
|
|
9797
10165
|
}
|
|
9798
10166
|
attempted += 1;
|
|
9799
|
-
const result = await
|
|
9800
|
-
|
|
9801
|
-
|
|
9802
|
-
audit: options.audit
|
|
9803
|
-
|
|
10167
|
+
const result = await dropSegmentFor(
|
|
10168
|
+
ref,
|
|
10169
|
+
deps,
|
|
10170
|
+
{ confirmSegment: rec.segment, dryRun, audit: options.audit },
|
|
10171
|
+
dryRun ? void 0 : {
|
|
10172
|
+
wanted: (row) => {
|
|
10173
|
+
const p = readRetentionPolicy(row.retention);
|
|
10174
|
+
return p !== null && p !== "invalid" && p.expiresAt <= now;
|
|
10175
|
+
},
|
|
10176
|
+
stamp: (retention) => ({ ...retention, [RETIRED_AT]: now })
|
|
10177
|
+
}
|
|
10178
|
+
);
|
|
10179
|
+
if (result === null) {
|
|
10180
|
+
attempted -= 1;
|
|
10181
|
+
entries.push({ ...base, action: "skipped", reason: "policy-changed" });
|
|
10182
|
+
continue;
|
|
10183
|
+
}
|
|
9804
10184
|
if (dryRun) {
|
|
9805
10185
|
wouldRetire += 1;
|
|
9806
10186
|
entries.push({ ...base, action: "would-retire", expiresAt: livePolicy.expiresAt, result });
|
|
@@ -9830,13 +10210,11 @@ async function retireExpired(deps, options) {
|
|
|
9830
10210
|
if (deleted) continue;
|
|
9831
10211
|
}
|
|
9832
10212
|
}
|
|
9833
|
-
|
|
9834
|
-
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
10213
|
+
if (removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
9835
10214
|
} catch (err) {
|
|
9836
10215
|
const after = await deps.registry.get(ref).catch(() => null);
|
|
9837
|
-
if (after?.status === "destroyed") {
|
|
9838
|
-
|
|
9839
|
-
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
10216
|
+
if (after?.status === "destroyed" && after.retention?.[RETIRED_AT] === now) {
|
|
10217
|
+
if (removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
9840
10218
|
retired += 1;
|
|
9841
10219
|
entries.push({
|
|
9842
10220
|
...base,
|
|
@@ -9884,7 +10262,7 @@ async function retireExpired(deps, options) {
|
|
|
9884
10262
|
entries
|
|
9885
10263
|
};
|
|
9886
10264
|
}
|
|
9887
|
-
var RETIRED_AT =
|
|
10265
|
+
var RETIRED_AT = SWEEP_MARK;
|
|
9888
10266
|
function retirementStamp(meta) {
|
|
9889
10267
|
if (meta === void 0 || meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
|
9890
10268
|
return null;
|
|
@@ -9892,21 +10270,6 @@ function retirementStamp(meta) {
|
|
|
9892
10270
|
const raw = meta[RETIRED_AT];
|
|
9893
10271
|
return typeof raw === "number" && Number.isInteger(raw) && raw >= MIN_EXPIRES_AT_MS ? raw : null;
|
|
9894
10272
|
}
|
|
9895
|
-
async function stampRetirement(registry, ref, now) {
|
|
9896
|
-
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
9897
|
-
const rec = await registry.get(ref);
|
|
9898
|
-
if (rec === null || rec.status !== "destroyed") return false;
|
|
9899
|
-
try {
|
|
9900
|
-
await registry.compareAndSwap(ref, rec.token, {
|
|
9901
|
-
retention: { ...rec.retention, [RETIRED_AT]: now }
|
|
9902
|
-
});
|
|
9903
|
-
return true;
|
|
9904
|
-
} catch (err) {
|
|
9905
|
-
if (!isWriteConflictError(err)) throw err;
|
|
9906
|
-
}
|
|
9907
|
-
}
|
|
9908
|
-
return false;
|
|
9909
|
-
}
|
|
9910
10273
|
async function isFullyReclaimed(deps, ref) {
|
|
9911
10274
|
for await (const key of deps.storage.list(ref)) {
|
|
9912
10275
|
void key;
|
|
@@ -9919,31 +10282,6 @@ function failureReason(err) {
|
|
|
9919
10282
|
return `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
9920
10283
|
}
|
|
9921
10284
|
|
|
9922
|
-
// src/core/registry-reap.ts
|
|
9923
|
-
var DEFAULT_REAP_LIMIT = 1e3;
|
|
9924
|
-
async function reapRegistryTombstones(registry, options = {}) {
|
|
9925
|
-
const dryRun = options.dryRun === true;
|
|
9926
|
-
if (!dryRun && options.confirmNoLegacyWriters !== true) {
|
|
9927
|
-
throw new ValidationError(
|
|
9928
|
-
"reapRegistryTombstones: a real run needs `confirmNoLegacyWriters: true`, your statement that no process on a release before 0.12 writes this registry (one that did would issue a removed row\u2019s tokens again). Pass `dryRun: true` to see what would be removed."
|
|
9929
|
-
);
|
|
9930
|
-
}
|
|
9931
|
-
const limit = options.limit ?? DEFAULT_REAP_LIMIT;
|
|
9932
|
-
if (!Number.isInteger(limit) || limit < 1) {
|
|
9933
|
-
throw new ValidationError(
|
|
9934
|
-
`reapRegistryTombstones: \`limit\` must be a positive integer; got ${String(limit)}`
|
|
9935
|
-
);
|
|
9936
|
-
}
|
|
9937
|
-
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
9938
|
-
if (registry.reapLegacyTombstones === void 0) {
|
|
9939
|
-
throw new UnsupportedError(
|
|
9940
|
-
"reapRegistryTombstones: this registry keeps no deleted rows of a release before 0.12 to remove (only the object-store registries, S3, GCS and Azure Blob, do)"
|
|
9941
|
-
);
|
|
9942
|
-
}
|
|
9943
|
-
const request = options.namespace === void 0 ? { dryRun, limit } : { namespace: options.namespace, dryRun, limit };
|
|
9944
|
-
return registry.reapLegacyTombstones(request);
|
|
9945
|
-
}
|
|
9946
|
-
|
|
9947
10285
|
// src/core/consistency.ts
|
|
9948
10286
|
var DEFAULT_CHECK_CONCURRENCY = 8;
|
|
9949
10287
|
async function generationsPresent(storage, ref) {
|
|
@@ -9960,7 +10298,7 @@ async function checkSummary(ref, live, deps) {
|
|
|
9960
10298
|
try {
|
|
9961
10299
|
aead = await deps.keystore.openDek(live.wrappedDeks);
|
|
9962
10300
|
} catch (err) {
|
|
9963
|
-
if (isIntegrityError(err) ||
|
|
10301
|
+
if (isIntegrityError(err) || isKeyUnavailableError(err))
|
|
9964
10302
|
return { kind: "ok", unchecked: true };
|
|
9965
10303
|
throw err;
|
|
9966
10304
|
}
|
|
@@ -10048,8 +10386,11 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
10048
10386
|
|
|
10049
10387
|
// src/drivers/retry/retrying-chunk-source.ts
|
|
10050
10388
|
function toRetry(opts) {
|
|
10389
|
+
if (opts.onRetry !== void 0 && typeof opts.onRetry !== "function") {
|
|
10390
|
+
throw new ValidationError("retry.onRetry must be a function");
|
|
10391
|
+
}
|
|
10051
10392
|
return {
|
|
10052
|
-
policy: opts.policy ?? DEFAULT_RETRY_POLICY,
|
|
10393
|
+
policy: checkedRetryPolicy(opts.policy ?? DEFAULT_RETRY_POLICY),
|
|
10053
10394
|
deps: {
|
|
10054
10395
|
clock: opts.clock,
|
|
10055
10396
|
rng: opts.rng,
|
|
@@ -10182,24 +10523,27 @@ export {
|
|
|
10182
10523
|
excludingReservedRows,
|
|
10183
10524
|
getSegmentRetention,
|
|
10184
10525
|
groundedReport,
|
|
10526
|
+
isBudgetExceededError,
|
|
10527
|
+
isCapabilityError,
|
|
10185
10528
|
isCloudRoaringError,
|
|
10186
10529
|
isIntegrityError,
|
|
10530
|
+
isKeyUnavailableError,
|
|
10187
10531
|
isLeaseExpiredError,
|
|
10188
10532
|
isLeaseLimitError,
|
|
10189
10533
|
isNotFoundError,
|
|
10190
10534
|
isStaleOperandError,
|
|
10191
10535
|
isStorageBackend,
|
|
10192
10536
|
isTransientError,
|
|
10537
|
+
isUnsupportedError,
|
|
10193
10538
|
isValidationError,
|
|
10194
10539
|
isWriteConflictError,
|
|
10540
|
+
judgeLoad,
|
|
10195
10541
|
listGenerations,
|
|
10196
10542
|
listSegments,
|
|
10197
10543
|
loadSegment,
|
|
10198
10544
|
loadSegmentChunks,
|
|
10199
10545
|
mapWithConcurrency,
|
|
10200
|
-
prepareHeld,
|
|
10201
10546
|
readRetentionPolicy,
|
|
10202
|
-
reapRegistryTombstones,
|
|
10203
10547
|
rebindCombineMany,
|
|
10204
10548
|
releaseLease,
|
|
10205
10549
|
resolveBudget,
|