@cloudbitmaps/core 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -5
- package/dist/{chunk-IGLIZBR7.js → chunk-OSU7W2Y3.js} +74 -30
- package/dist/chunk-OSU7W2Y3.js.map +7 -0
- package/dist/core/bit-route.d.ts +7 -0
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/combine-expr.d.ts +9 -5
- package/dist/core/combine-feed.d.ts +11 -3
- package/dist/core/combine-many.d.ts +17 -13
- package/dist/core/determinism.d.ts +2 -1
- package/dist/core/erase-id.d.ts +18 -8
- package/dist/core/erasure.d.ts +1 -1
- package/dist/core/errors.d.ts +53 -9
- package/dist/core/load.d.ts +48 -14
- package/dist/core/metrics.d.ts +1 -1
- package/dist/core/ports.d.ts +7 -6
- package/dist/core/registry-scan.d.ts +0 -5
- package/dist/core/retention-sweep.d.ts +1 -1
- package/dist/core/retention.d.ts +1 -1
- package/dist/core/retry.d.ts +4 -0
- package/dist/core/rollback.d.ts +1 -0
- package/dist/driver-kit.d.ts +7 -2
- package/dist/driver-kit.js +5 -10
- package/dist/driver-kit.js.map +1 -1
- package/dist/drivers/backends.d.ts +2 -0
- package/dist/drivers/crypto.d.ts +1 -1
- package/dist/export/index.d.ts +9 -8
- package/dist/index.d.ts +11 -10
- package/dist/index.js +766 -634
- package/dist/index.js.map +4 -4
- package/package.json +7 -2
- package/dist/chunk-IGLIZBR7.js.map +0 -7
- package/dist/core/absent-operands.d.ts +0 -8
- package/dist/core/bulk-load.d.ts +0 -148
- package/dist/core/cas-retry.d.ts +0 -27
- package/dist/core/check.d.ts +0 -22
- package/dist/core/chunk-checks.d.ts +0 -39
- package/dist/core/chunk-stream.d.ts +0 -22
- package/dist/core/chunk-window.d.ts +0 -37
- package/dist/core/combine-ledger.d.ts +0 -43
- package/dist/core/crbm/crc32c.d.ts +0 -10
- package/dist/core/crbm/fingerprint.d.ts +0 -23
- package/dist/core/crbm/format.d.ts +0 -150
- package/dist/core/crbm/frame.d.ts +0 -10
- package/dist/core/crbm/plan-reads.d.ts +0 -50
- package/dist/core/crbm/varint.d.ts +0 -14
- package/dist/core/crbm/writer.d.ts +0 -72
- package/dist/core/crbm-open.d.ts +0 -154
- package/dist/core/crbm-write.d.ts +0 -75
- package/dist/core/due-index.d.ts +0 -53
- package/dist/core/generation-gc.d.ts +0 -293
- package/dist/core/generation-list.d.ts +0 -11
- package/dist/core/id-window.d.ts +0 -46
- package/dist/core/invalidation-scope.d.ts +0 -77
- package/dist/core/item-pull.d.ts +0 -29
- package/dist/core/known-generations-shape.d.ts +0 -13
- package/dist/core/known-generations.d.ts +0 -147
- package/dist/core/ledger-seed.d.ts +0 -17
- package/dist/core/metadata.d.ts +0 -42
- package/dist/core/order.d.ts +0 -2
- package/dist/core/publish.d.ts +0 -207
- package/dist/core/reader-defaults.d.ts +0 -29
- package/dist/core/summary.d.ts +0 -110
- package/dist/core/token.d.ts +0 -41
- package/dist/drivers/_shared/entropy.d.ts +0 -5
- package/dist/drivers/_shared/registry.d.ts +0 -107
- package/dist/drivers/localfs/exact-case.d.ts +0 -34
- package/dist/drivers/localfs/fs-util.d.ts +0 -55
- package/dist/drivers/localfs/paths.d.ts +0 -40
- package/dist/drivers/localfs/registry.d.ts +0 -50
- package/dist/drivers/localfs/storage.d.ts +0 -28
- package/dist/drivers/memory.d.ts +0 -64
- package/dist/testing/fuzz-core.d.ts +0 -15
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BudgetExceededError,
|
|
3
3
|
CHUNK_COUNT,
|
|
4
|
+
CONTENTION_NEXT_STEP,
|
|
4
5
|
CapabilityError,
|
|
5
6
|
CloudRoaringError,
|
|
6
7
|
DEFAULT_NAMESPACE,
|
|
@@ -47,6 +48,7 @@ import {
|
|
|
47
48
|
encodeNameForPath,
|
|
48
49
|
encodedNameLength,
|
|
49
50
|
entropyIsAvailable,
|
|
51
|
+
errorKind,
|
|
50
52
|
fallbackBound,
|
|
51
53
|
fingerprintFor,
|
|
52
54
|
fingerprintParts,
|
|
@@ -54,21 +56,15 @@ import {
|
|
|
54
56
|
heldGenerations,
|
|
55
57
|
incarnationField,
|
|
56
58
|
incarnationOf,
|
|
57
|
-
|
|
58
|
-
isCapabilityError,
|
|
59
|
-
isCloudRoaringError,
|
|
59
|
+
isBadIdError,
|
|
60
60
|
isFullLedger,
|
|
61
61
|
isIntegrityError,
|
|
62
62
|
isKeyUnavailableError,
|
|
63
|
-
isLeaseExpiredError,
|
|
64
|
-
isLeaseLimitError,
|
|
65
63
|
isLive,
|
|
66
64
|
isNotFoundError,
|
|
67
65
|
isReservedNamespace,
|
|
68
|
-
isStaleOperandError,
|
|
69
66
|
isStorageBackend,
|
|
70
67
|
isTransientError,
|
|
71
|
-
isUnsupportedError,
|
|
72
68
|
isValidationError,
|
|
73
69
|
isWriteConflictError,
|
|
74
70
|
jitteredWait,
|
|
@@ -121,7 +117,7 @@ import {
|
|
|
121
117
|
webCryptoEntropy,
|
|
122
118
|
windowOf,
|
|
123
119
|
withRetry
|
|
124
|
-
} from "./chunk-
|
|
120
|
+
} from "./chunk-OSU7W2Y3.js";
|
|
125
121
|
|
|
126
122
|
// src/core/budget.ts
|
|
127
123
|
var DEFAULT_BUDGET = { maxRequests: 1e6 };
|
|
@@ -152,7 +148,8 @@ function resolvePerOpBudget(opt, storeBudget) {
|
|
|
152
148
|
function checkBudget(budget, projectedRequests, op) {
|
|
153
149
|
if (budget !== null && projectedRequests > budget.maxRequests) {
|
|
154
150
|
throw new BudgetExceededError(
|
|
155
|
-
`${op} would fan out to ${projectedRequests} units, over the per-op budget of ${budget.maxRequests} \u2014 raise \`budget.maxRequests\`, override it per-op, or set \`budget: false
|
|
151
|
+
`${op} would fan out to ${projectedRequests} units, over the per-op budget of ${budget.maxRequests} \u2014 raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
152
|
+
{ option: "budget.maxRequests", limit: budget.maxRequests, spent: projectedRequests }
|
|
156
153
|
);
|
|
157
154
|
}
|
|
158
155
|
}
|
|
@@ -162,7 +159,9 @@ async function collectWithinBudget(source, budget, op) {
|
|
|
162
159
|
out.push(item);
|
|
163
160
|
if (budget !== null && out.length > budget.maxRequests) {
|
|
164
161
|
throw new BudgetExceededError(
|
|
165
|
-
`${op} would fan out to more than ${budget.maxRequests} units, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false
|
|
162
|
+
`${op} would fan out to more than ${budget.maxRequests} units, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
163
|
+
// The scan stops at the first item past the bound, so `spent` is a lower bound on the true total.
|
|
164
|
+
{ option: "budget.maxRequests", limit: budget.maxRequests, spent: out.length }
|
|
166
165
|
);
|
|
167
166
|
}
|
|
168
167
|
}
|
|
@@ -172,8 +171,8 @@ async function collectWithinBudget(source, budget, op) {
|
|
|
172
171
|
// src/core/absent-operands.ts
|
|
173
172
|
async function refuseAbsent(op, candidates, limit, exists, nameOf) {
|
|
174
173
|
if (candidates.length === 0) return;
|
|
175
|
-
const
|
|
176
|
-
const absent = candidates.filter((_, i) => !
|
|
174
|
+
const found2 = await boundedAll(candidates, limit, exists);
|
|
175
|
+
const absent = candidates.filter((_, i) => !found2[i]);
|
|
177
176
|
if (absent.length === 0) return;
|
|
178
177
|
const named = absent.map((c) => `"${nameOf(c)}"`).join(", ");
|
|
179
178
|
throw new ValidationError(
|
|
@@ -549,8 +548,8 @@ var KeptSegmentKeys = class {
|
|
|
549
548
|
/** The segment's encoded key. */
|
|
550
549
|
of(ref) {
|
|
551
550
|
let names = this.byNamespace.get(ref.namespace);
|
|
552
|
-
const
|
|
553
|
-
if (
|
|
551
|
+
const found2 = names?.get(ref.segment);
|
|
552
|
+
if (found2 !== void 0) return found2;
|
|
554
553
|
const encoded = this.encode(ref);
|
|
555
554
|
if (this.kept >= this.max) {
|
|
556
555
|
this.byNamespace.clear();
|
|
@@ -919,9 +918,9 @@ var SegmentEngine = class _SegmentEngine {
|
|
|
919
918
|
*/
|
|
920
919
|
async stat(seg) {
|
|
921
920
|
if (this.storage.stat) {
|
|
922
|
-
const
|
|
923
|
-
if (
|
|
924
|
-
return
|
|
921
|
+
const found2 = await this.storage.stat(seg);
|
|
922
|
+
if (found2 === null) return { generation: null, cardinality: 0, sizeBytes: null };
|
|
923
|
+
return found2;
|
|
925
924
|
}
|
|
926
925
|
if (this.storage.summary) {
|
|
927
926
|
const summary = await this.storage.summary(seg);
|
|
@@ -2312,13 +2311,12 @@ function dueBucketsAt(now, lookbackBuckets) {
|
|
|
2312
2311
|
// src/core/registry-scan.ts
|
|
2313
2312
|
var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
|
|
2314
2313
|
function validateMaxScanSegments(value, op) {
|
|
2315
|
-
requirePositiveInteger(value, `${op}: maxScanSegments
|
|
2314
|
+
requirePositiveInteger(value, `${op}: \`maxScanSegments\``);
|
|
2316
2315
|
}
|
|
2317
2316
|
async function drainRegistry(registry, options) {
|
|
2318
2317
|
const { maxScanSegments, op } = options;
|
|
2319
2318
|
validateMaxScanSegments(maxScanSegments, op);
|
|
2320
|
-
const
|
|
2321
|
-
const advice = options.narrowable === false ? `${raise[0].toUpperCase()}${raise.slice(1)} if the namespace really is that large and the memory is ` : `Narrow it with \`namespace\`, or ${raise} if the fleet really is that large and the memory is `;
|
|
2319
|
+
const advice = options.narrowable === false ? "Raise `maxScanSegments` if the namespace really is that large and the memory is " : "Narrow it with `namespace`, or raise `maxScanSegments` if the fleet really is that large and the memory is ";
|
|
2322
2320
|
const rows = [];
|
|
2323
2321
|
for await (const rec of registry.list(options.namespace)) {
|
|
2324
2322
|
if (options.namespace === void 0 && isReservedRow(rec)) {
|
|
@@ -2327,7 +2325,9 @@ async function drainRegistry(registry, options) {
|
|
|
2327
2325
|
}
|
|
2328
2326
|
if (rows.length >= maxScanSegments) {
|
|
2329
2327
|
throw new BudgetExceededError(
|
|
2330
|
-
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident)
|
|
2328
|
+
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident).`,
|
|
2329
|
+
// The scan stops at the first row past the bound, so `spent` is a lower bound on the true total.
|
|
2330
|
+
{ option: "maxScanSegments", limit: maxScanSegments, spent: rows.length + 1 }
|
|
2331
2331
|
);
|
|
2332
2332
|
}
|
|
2333
2333
|
rows.push(rec);
|
|
@@ -3415,6 +3415,9 @@ var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
|
|
|
3415
3415
|
var RESOLUTIONS_PER_OPEN_SEGMENT = 8;
|
|
3416
3416
|
var RESOLUTION_BYTES_DIVISOR = 16;
|
|
3417
3417
|
|
|
3418
|
+
// src/core/crbm/integrity.ts
|
|
3419
|
+
var crbmIntegrity = (message) => new IntegrityError(`${message}; re-load from source or roll back`);
|
|
3420
|
+
|
|
3418
3421
|
// src/core/crbm/crc32c.ts
|
|
3419
3422
|
var TABLES = (() => {
|
|
3420
3423
|
const POLY = 2197175160;
|
|
@@ -3470,7 +3473,7 @@ function* planChunkReads(extents, region) {
|
|
|
3470
3473
|
for (const extent of extents) {
|
|
3471
3474
|
const extentEnd = extent.offset + extent.length;
|
|
3472
3475
|
if (!(Number.isSafeInteger(extent.offset) && Number.isSafeInteger(extent.length) && extent.length >= 1 && extent.offset >= previousEnd && extentEnd <= region.end)) {
|
|
3473
|
-
throw
|
|
3476
|
+
throw crbmIntegrity(`chunk ${extent.key} is not inside the chunk region, in order`);
|
|
3474
3477
|
}
|
|
3475
3478
|
previousEnd = extentEnd;
|
|
3476
3479
|
if (chunks.length > 0 && extent.offset - end <= MAX_COALESCE_GAP_BYTES && extentEnd - start <= MAX_COALESCED_READ_BYTES) {
|
|
@@ -3509,12 +3512,12 @@ function readVarint(bytes, offset) {
|
|
|
3509
3512
|
let pos = offset;
|
|
3510
3513
|
for (let i = 0; i < 5; i++) {
|
|
3511
3514
|
if (pos >= bytes.length) {
|
|
3512
|
-
throw
|
|
3515
|
+
throw crbmIntegrity("varint truncated past end of buffer");
|
|
3513
3516
|
}
|
|
3514
3517
|
const byte = bytes[pos];
|
|
3515
3518
|
pos += 1;
|
|
3516
3519
|
if (i === 4 && byte > 15) {
|
|
3517
|
-
throw
|
|
3520
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
3518
3521
|
}
|
|
3519
3522
|
result = (result | (byte & 127) << shift) >>> 0;
|
|
3520
3523
|
if ((byte & 128) === 0) {
|
|
@@ -3522,7 +3525,7 @@ function readVarint(bytes, offset) {
|
|
|
3522
3525
|
}
|
|
3523
3526
|
shift += 7;
|
|
3524
3527
|
}
|
|
3525
|
-
throw
|
|
3528
|
+
throw crbmIntegrity("varint exceeds u32 range");
|
|
3526
3529
|
}
|
|
3527
3530
|
|
|
3528
3531
|
// src/core/crbm/reader.ts
|
|
@@ -3537,22 +3540,22 @@ function magicMatches(bytes, offset) {
|
|
|
3537
3540
|
}
|
|
3538
3541
|
function checkSize(tail, size) {
|
|
3539
3542
|
if (!Number.isSafeInteger(size) || size < tail.length) {
|
|
3540
|
-
throw
|
|
3543
|
+
throw crbmIntegrity(`.crbm size is not a byte count its tail fits in: ${size}`);
|
|
3541
3544
|
}
|
|
3542
3545
|
}
|
|
3543
3546
|
function checkedFooter(tail, size) {
|
|
3544
3547
|
checkSize(tail, size);
|
|
3545
3548
|
if (size < PREAMBLE_BYTES + FOOTER_BYTES || tail.length < FOOTER_BYTES) {
|
|
3546
|
-
throw
|
|
3549
|
+
throw crbmIntegrity(`.crbm too small: ${size}B`);
|
|
3547
3550
|
}
|
|
3548
3551
|
const footer = tail.subarray(tail.length - FOOTER_BYTES);
|
|
3549
3552
|
if (!magicMatches(footer, FOOTER.endMagic)) {
|
|
3550
|
-
throw
|
|
3553
|
+
throw crbmIntegrity(".crbm end magic mismatch");
|
|
3551
3554
|
}
|
|
3552
3555
|
const fview = new DataView(footer.buffer, footer.byteOffset, footer.byteLength);
|
|
3553
3556
|
const storedFooterCrc = fview.getUint32(FOOTER.footerCrc32c, true);
|
|
3554
3557
|
if (crc32c(footer.subarray(0, FOOTER_CRC_COVERAGE)) !== storedFooterCrc) {
|
|
3555
|
-
throw
|
|
3558
|
+
throw crbmIntegrity(".crbm footer CRC mismatch");
|
|
3556
3559
|
}
|
|
3557
3560
|
return { footer, fview, storedFooterCrc };
|
|
3558
3561
|
}
|
|
@@ -3570,7 +3573,7 @@ async function footerSaysEncrypted(blob) {
|
|
|
3570
3573
|
function readU64(view, offset, field) {
|
|
3571
3574
|
const big = view.getBigUint64(offset, true);
|
|
3572
3575
|
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
3573
|
-
throw
|
|
3576
|
+
throw crbmIntegrity(`.crbm ${field} ${big} exceeds safe-integer range`);
|
|
3574
3577
|
}
|
|
3575
3578
|
return Number(big);
|
|
3576
3579
|
}
|
|
@@ -3678,7 +3681,10 @@ var CrbmReader = class _CrbmReader {
|
|
|
3678
3681
|
const { footer, fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
3679
3682
|
const versionMajor = footer[FOOTER.versionMajor];
|
|
3680
3683
|
if (versionMajor !== VERSION_MAJOR) {
|
|
3681
|
-
|
|
3684
|
+
const next = versionMajor > VERSION_MAJOR ? "upgrade @cloudbitmaps/core" : "no release writes that version, so this library did not write the object";
|
|
3685
|
+
throw new UnsupportedError(
|
|
3686
|
+
`.crbm major version ${versionMajor} not supported; this build reads major version ${VERSION_MAJOR}: ${next}`
|
|
3687
|
+
);
|
|
3682
3688
|
}
|
|
3683
3689
|
const flags = fview.getUint32(FOOTER.flags, true);
|
|
3684
3690
|
const encrypted = (flags & FLAG_ENCRYPTED) !== 0;
|
|
@@ -3716,24 +3722,22 @@ var CrbmReader = class _CrbmReader {
|
|
|
3716
3722
|
const totalCardinality = readU64(fview, FOOTER.totalCardinality, "total_cardinality");
|
|
3717
3723
|
const generation = readU64(fview, FOOTER.generation, "generation");
|
|
3718
3724
|
if (!encrypted && options.crypto !== void 0) {
|
|
3719
|
-
throw
|
|
3725
|
+
throw crbmIntegrity(
|
|
3720
3726
|
`.crbm generation ${generation} is not encrypted, but it was opened with a key: a cleartext object where an encrypted one belongs is forged, corrupt, or a write that never published (a crash, before the segment's key was made); a load's collection, dropSegment or deleting the object removes it`
|
|
3721
3727
|
);
|
|
3722
3728
|
}
|
|
3723
3729
|
const versionMinor = footer[FOOTER.versionMinor];
|
|
3724
3730
|
const hasExtension = (flags & FLAG_EXTENSION) !== 0;
|
|
3725
3731
|
if (indexOffset < PAYLOAD_START || indexOffset + indexLength > size - FOOTER_BYTES) {
|
|
3726
|
-
throw
|
|
3727
|
-
`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`
|
|
3728
|
-
);
|
|
3732
|
+
throw crbmIntegrity(`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`);
|
|
3729
3733
|
}
|
|
3730
3734
|
if (indexLength > maxIndexBytes) {
|
|
3731
|
-
throw
|
|
3735
|
+
throw crbmIntegrity(`.crbm index ${indexLength}B exceeds cap ${maxIndexBytes}B`);
|
|
3732
3736
|
}
|
|
3733
3737
|
const tailCoversFrom = size - tail.length;
|
|
3734
3738
|
if (tailCoversFrom === 0) {
|
|
3735
3739
|
if (!magicMatches(tail, 0) || tail[4] !== versionMajor || tail[5] !== versionMinor) {
|
|
3736
|
-
throw
|
|
3740
|
+
throw crbmIntegrity(".crbm preamble magic/version mismatch with footer");
|
|
3737
3741
|
}
|
|
3738
3742
|
}
|
|
3739
3743
|
const lowest = hasExtension ? Math.max(PAYLOAD_START, indexOffset - EXT_TRAILER_BYTES - MAX_EXT_BYTES) : indexOffset;
|
|
@@ -3754,7 +3758,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3754
3758
|
indexOffset - regionStart + indexLength
|
|
3755
3759
|
);
|
|
3756
3760
|
if (crc32c(indexBytes) !== indexCrc) {
|
|
3757
|
-
throw
|
|
3761
|
+
throw crbmIntegrity(".crbm index CRC mismatch");
|
|
3758
3762
|
}
|
|
3759
3763
|
const indexForParse = encrypted ? options.crypto.aead.open(
|
|
3760
3764
|
{
|
|
@@ -3770,7 +3774,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3770
3774
|
if (hasExtension) {
|
|
3771
3775
|
const trailerStart = indexOffset - EXT_TRAILER_BYTES;
|
|
3772
3776
|
if (trailerStart < PAYLOAD_START) {
|
|
3773
|
-
throw
|
|
3777
|
+
throw crbmIntegrity(".crbm flags an extension block, but has no room for one");
|
|
3774
3778
|
}
|
|
3775
3779
|
let window = { bytes: region, start: regionStart };
|
|
3776
3780
|
if (trailerStart < regionStart) {
|
|
@@ -3780,17 +3784,17 @@ var CrbmReader = class _CrbmReader {
|
|
|
3780
3784
|
const trailer = within(window.bytes, window.start, trailerStart, indexOffset);
|
|
3781
3785
|
const tview = new DataView(trailer.buffer, trailer.byteOffset, trailer.byteLength);
|
|
3782
3786
|
if (!EXT_MAGIC.every((b, i) => trailer[8 + i] === b)) {
|
|
3783
|
-
throw
|
|
3787
|
+
throw crbmIntegrity(".crbm extension block trailer magic mismatch");
|
|
3784
3788
|
}
|
|
3785
3789
|
const sectionsLength = tview.getUint32(0, true);
|
|
3786
3790
|
if (sectionsLength > MAX_EXT_BYTES) {
|
|
3787
|
-
throw
|
|
3791
|
+
throw crbmIntegrity(
|
|
3788
3792
|
`.crbm extension block ${sectionsLength}B exceeds cap ${MAX_EXT_BYTES}B`
|
|
3789
3793
|
);
|
|
3790
3794
|
}
|
|
3791
3795
|
const extStart = trailerStart - sectionsLength;
|
|
3792
3796
|
if (extStart < PAYLOAD_START) {
|
|
3793
|
-
throw
|
|
3797
|
+
throw crbmIntegrity(`.crbm extension block of ${sectionsLength}B out of bounds`);
|
|
3794
3798
|
}
|
|
3795
3799
|
let covered;
|
|
3796
3800
|
if (extStart >= window.start) {
|
|
@@ -3799,11 +3803,11 @@ var CrbmReader = class _CrbmReader {
|
|
|
3799
3803
|
covered = await blob.getRange(extStart, sectionsLength + 4);
|
|
3800
3804
|
servedFromTail = false;
|
|
3801
3805
|
if (covered.length !== sectionsLength + 4) {
|
|
3802
|
-
throw
|
|
3806
|
+
throw crbmIntegrity(".crbm extension block read short");
|
|
3803
3807
|
}
|
|
3804
3808
|
}
|
|
3805
3809
|
if (crc32c(covered) !== tview.getUint32(4, true)) {
|
|
3806
|
-
throw
|
|
3810
|
+
throw crbmIntegrity(".crbm extension block CRC mismatch");
|
|
3807
3811
|
}
|
|
3808
3812
|
({ metadata, metadataLength } = extensionContents(
|
|
3809
3813
|
covered.subarray(0, sectionsLength),
|
|
@@ -3819,18 +3823,16 @@ var CrbmReader = class _CrbmReader {
|
|
|
3819
3823
|
);
|
|
3820
3824
|
if (encrypted) {
|
|
3821
3825
|
if (chunkCount !== 0 || totalCardinality !== 0) {
|
|
3822
|
-
throw
|
|
3823
|
-
".crbm encrypted footer must zero chunk_count + total_cardinality"
|
|
3824
|
-
);
|
|
3826
|
+
throw crbmIntegrity(".crbm encrypted footer must zero chunk_count + total_cardinality");
|
|
3825
3827
|
}
|
|
3826
3828
|
} else {
|
|
3827
3829
|
if (index.keys.length !== chunkCount) {
|
|
3828
|
-
throw
|
|
3830
|
+
throw crbmIntegrity(
|
|
3829
3831
|
`.crbm chunk_count ${chunkCount} != ${index.keys.length} index entries`
|
|
3830
3832
|
);
|
|
3831
3833
|
}
|
|
3832
3834
|
if (index.cardinalitySum !== totalCardinality) {
|
|
3833
|
-
throw
|
|
3835
|
+
throw crbmIntegrity(
|
|
3834
3836
|
`.crbm total_cardinality ${totalCardinality} != \u03A3 index cardinality ${index.cardinalitySum}`
|
|
3835
3837
|
);
|
|
3836
3838
|
}
|
|
@@ -3932,7 +3934,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
3932
3934
|
const offset = this.index.offsets[slot];
|
|
3933
3935
|
const length = this.index.lengths[slot];
|
|
3934
3936
|
if (offset < PAYLOAD_START || offset + length > this.objectSize - FOOTER_BYTES) {
|
|
3935
|
-
throw
|
|
3937
|
+
throw crbmIntegrity(`chunk ${chunkKey} payload out of bounds`);
|
|
3936
3938
|
}
|
|
3937
3939
|
const held = this.keptRange(offset, length);
|
|
3938
3940
|
return this.openChunk(slot, chunkKey, held ?? await this.blob.getRange(offset, length));
|
|
@@ -3959,11 +3961,11 @@ var CrbmReader = class _CrbmReader {
|
|
|
3959
3961
|
*/
|
|
3960
3962
|
openChunk(slot, chunkKey, bytes) {
|
|
3961
3963
|
if (crc32c(bytes) !== this.index.crcs[slot]) {
|
|
3962
|
-
throw
|
|
3964
|
+
throw crbmIntegrity(`chunk ${chunkKey} payload CRC mismatch`);
|
|
3963
3965
|
}
|
|
3964
3966
|
if (this.crypto === void 0) return bytes;
|
|
3965
3967
|
if (bytes.length < AEAD_NONCE_BYTES + AEAD_TAG_BYTES) {
|
|
3966
|
-
throw
|
|
3968
|
+
throw crbmIntegrity(`chunk ${chunkKey} encrypted payload too short`);
|
|
3967
3969
|
}
|
|
3968
3970
|
return this.crypto.aead.open(
|
|
3969
3971
|
{
|
|
@@ -4067,7 +4069,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
4067
4069
|
});
|
|
4068
4070
|
moved = bytes.length;
|
|
4069
4071
|
if (bytes.length !== read.length) {
|
|
4070
|
-
throw
|
|
4072
|
+
throw crbmIntegrity(
|
|
4071
4073
|
`.crbm range [${read.offset}, +${read.length}) answered ${bytes.length} bytes, not the ${read.length} asked for`
|
|
4072
4074
|
);
|
|
4073
4075
|
}
|
|
@@ -4120,12 +4122,12 @@ var StreamClosed = class extends Error {
|
|
|
4120
4122
|
};
|
|
4121
4123
|
function within(bytes, start, from, to) {
|
|
4122
4124
|
if (from < start || to > start + bytes.length) {
|
|
4123
|
-
throw
|
|
4125
|
+
throw crbmIntegrity(".crbm extension block read short");
|
|
4124
4126
|
}
|
|
4125
4127
|
return bytes.subarray(from - start, to - start);
|
|
4126
4128
|
}
|
|
4127
4129
|
function extensionCorrupt(message) {
|
|
4128
|
-
throw
|
|
4130
|
+
throw crbmIntegrity(`.crbm extension block: ${message}`);
|
|
4129
4131
|
}
|
|
4130
4132
|
function extensionContents(sections, crypto) {
|
|
4131
4133
|
const view = new DataView(sections.buffer, sections.byteOffset, sections.byteLength);
|
|
@@ -4202,26 +4204,26 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
|
|
|
4202
4204
|
const card = cardRead.value;
|
|
4203
4205
|
pos = cardRead.next;
|
|
4204
4206
|
if (pos + CRC32C_BYTES > indexBytes.length) {
|
|
4205
|
-
throw
|
|
4207
|
+
throw crbmIntegrity(".crbm index truncated (missing payload CRC)");
|
|
4206
4208
|
}
|
|
4207
4209
|
const crc = (indexBytes[pos] | indexBytes[pos + 1] << 8 | indexBytes[pos + 2] << 16 | indexBytes[pos + 3] << 24) >>> 0;
|
|
4208
4210
|
pos += CRC32C_BYTES;
|
|
4209
4211
|
const chunkKey = prevKey + keyDelta;
|
|
4210
4212
|
const offset = prevEnd + offDelta;
|
|
4211
4213
|
if (i > 0 && keyDelta === 0) {
|
|
4212
|
-
throw
|
|
4214
|
+
throw crbmIntegrity(`.crbm index has a duplicate chunkKey ${chunkKey}`);
|
|
4213
4215
|
}
|
|
4214
|
-
if (chunkKey > 65535) throw
|
|
4216
|
+
if (chunkKey > 65535) throw crbmIntegrity(`.crbm chunkKey ${chunkKey} out of range`);
|
|
4215
4217
|
if (len < minPayloadBytes || len > maxPayloadBytes) {
|
|
4216
|
-
throw
|
|
4218
|
+
throw crbmIntegrity(
|
|
4217
4219
|
`.crbm chunk ${chunkKey} length ${len} invalid (most is ${maxPayloadBytes})`
|
|
4218
4220
|
);
|
|
4219
4221
|
}
|
|
4220
4222
|
if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
|
|
4221
|
-
throw
|
|
4223
|
+
throw crbmIntegrity(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
|
|
4222
4224
|
}
|
|
4223
4225
|
if (offset + len > payloadEnd) {
|
|
4224
|
-
throw
|
|
4226
|
+
throw crbmIntegrity(`.crbm chunk ${chunkKey} payload out of bounds`);
|
|
4225
4227
|
}
|
|
4226
4228
|
keys[count] = chunkKey;
|
|
4227
4229
|
cardinalityMinusOne[count] = card - 1;
|
|
@@ -6211,6 +6213,7 @@ var CrbmWriter = class {
|
|
|
6211
6213
|
this.metadata = metadataBytes(options.metadata, (message) => {
|
|
6212
6214
|
throw new ValidationError(message);
|
|
6213
6215
|
});
|
|
6216
|
+
this.writeIdentity = options.crypto === void 0 ? drawWriteIdentity(options.entropy) : void 0;
|
|
6214
6217
|
}
|
|
6215
6218
|
sink;
|
|
6216
6219
|
options;
|
|
@@ -6227,6 +6230,8 @@ var CrbmWriter = class {
|
|
|
6227
6230
|
/** Where the chunk payloads end: the extension block's start when there is metadata, else the index's. */
|
|
6228
6231
|
payloadEnd = 0;
|
|
6229
6232
|
footerCrc32c;
|
|
6233
|
+
/** A cleartext generation's write identity, drawn at construction; `undefined` for an encrypted one. */
|
|
6234
|
+
writeIdentity;
|
|
6230
6235
|
/**
|
|
6231
6236
|
* Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
|
|
6232
6237
|
* strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
|
|
@@ -6289,6 +6294,8 @@ var CrbmWriter = class {
|
|
|
6289
6294
|
indexRegion = sealed.ciphertext;
|
|
6290
6295
|
indexNonce = sealed.nonce;
|
|
6291
6296
|
indexTag = sealed.tag;
|
|
6297
|
+
} else {
|
|
6298
|
+
indexNonce = this.writeIdentity;
|
|
6292
6299
|
}
|
|
6293
6300
|
const indexOffset = this.offset;
|
|
6294
6301
|
await this.sink.write(indexRegion);
|
|
@@ -6402,8 +6409,35 @@ function framed(sealed) {
|
|
|
6402
6409
|
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
6403
6410
|
return out;
|
|
6404
6411
|
}
|
|
6412
|
+
function drawWriteIdentity(entropy) {
|
|
6413
|
+
if (typeof entropy !== "function") {
|
|
6414
|
+
throw new ValidationError(
|
|
6415
|
+
"a cleartext CrbmWriter needs an entropy source (`entropy`) for its write's identity"
|
|
6416
|
+
);
|
|
6417
|
+
}
|
|
6418
|
+
const drawn = entropy(AEAD_NONCE_BYTES);
|
|
6419
|
+
if (!(drawn instanceof Uint8Array)) {
|
|
6420
|
+
throw new ValidationError(
|
|
6421
|
+
`the entropy source must return a Uint8Array of ${AEAD_NONCE_BYTES} bytes for a write's identity; got ${Array.isArray(drawn) ? "an array" : typeof drawn}`
|
|
6422
|
+
);
|
|
6423
|
+
}
|
|
6424
|
+
if (drawn.length !== AEAD_NONCE_BYTES) {
|
|
6425
|
+
throw new ValidationError(
|
|
6426
|
+
`the entropy source must return ${AEAD_NONCE_BYTES} bytes for a write's identity; got ${drawn.length}`
|
|
6427
|
+
);
|
|
6428
|
+
}
|
|
6429
|
+
return Uint8Array.from(drawn);
|
|
6430
|
+
}
|
|
6405
6431
|
|
|
6406
6432
|
// src/core/crbm-write.ts
|
|
6433
|
+
function requireEntropy(entropy, verb) {
|
|
6434
|
+
if (typeof entropy !== "function") {
|
|
6435
|
+
throw new ValidationError(
|
|
6436
|
+
`${verb} needs an entropy source (\`entropy\`): the footer of every generation it writes carries 12 random bytes of the write's identity`
|
|
6437
|
+
);
|
|
6438
|
+
}
|
|
6439
|
+
return entropy;
|
|
6440
|
+
}
|
|
6407
6441
|
function* encodeEach(chunks) {
|
|
6408
6442
|
for (const { chunkKey, bitmap } of chunks) {
|
|
6409
6443
|
if (bitmap.isEmpty) continue;
|
|
@@ -6420,7 +6454,8 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
6420
6454
|
const writer = new CrbmWriter(sink, {
|
|
6421
6455
|
generation: key.generation,
|
|
6422
6456
|
crypto: options.crypto,
|
|
6423
|
-
metadata: options.metadata
|
|
6457
|
+
metadata: options.metadata,
|
|
6458
|
+
entropy: options.entropy
|
|
6424
6459
|
});
|
|
6425
6460
|
for (const chunk of chunks) {
|
|
6426
6461
|
await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
|
|
@@ -6446,7 +6481,7 @@ async function writeEncodedChunks(driver, key, chunks, options) {
|
|
|
6446
6481
|
tailBytes: identity.tailBytes
|
|
6447
6482
|
};
|
|
6448
6483
|
}
|
|
6449
|
-
async function writeCrbmGenerationStream(driver, key, chunks, options
|
|
6484
|
+
async function writeCrbmGenerationStream(driver, key, chunks, options) {
|
|
6450
6485
|
const chunkKeys = [];
|
|
6451
6486
|
let cardinality = 0;
|
|
6452
6487
|
const tick = yieldEvery(options.clock);
|
|
@@ -6455,7 +6490,8 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
|
6455
6490
|
const writer = new CrbmWriter(sink, {
|
|
6456
6491
|
generation: key.generation,
|
|
6457
6492
|
crypto: options.crypto,
|
|
6458
|
-
metadata: options.metadata
|
|
6493
|
+
metadata: options.metadata,
|
|
6494
|
+
entropy: options.entropy
|
|
6459
6495
|
});
|
|
6460
6496
|
for await (const { chunkKey, bitmap } of chunks) {
|
|
6461
6497
|
if (bitmap.isEmpty) continue;
|
|
@@ -6701,7 +6737,7 @@ async function publishGenerationKept(registry, key, options) {
|
|
|
6701
6737
|
}
|
|
6702
6738
|
if (unanswered !== void 0) throw unanswered;
|
|
6703
6739
|
throw new WriteConflictError(
|
|
6704
|
-
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
6740
|
+
`publishGeneration: contention setting currentGen for ${key.segment}${CONTENTION_NEXT_STEP}`
|
|
6705
6741
|
);
|
|
6706
6742
|
}
|
|
6707
6743
|
var REFUSED = { published: false };
|
|
@@ -6749,6 +6785,7 @@ async function bucketIds(ids, codec, clock) {
|
|
|
6749
6785
|
const byChunk = /* @__PURE__ */ new Map();
|
|
6750
6786
|
const pendingByChunk = /* @__PURE__ */ new Map();
|
|
6751
6787
|
let pending = 0;
|
|
6788
|
+
let flushed = 0;
|
|
6752
6789
|
const tickChunk = yieldEvery(clock);
|
|
6753
6790
|
const tickId = yieldEvery(clock, YIELD_EVERY_IDS);
|
|
6754
6791
|
const flushPending = async () => {
|
|
@@ -6761,6 +6798,7 @@ async function bucketIds(ids, codec, clock) {
|
|
|
6761
6798
|
const pause = tickChunk();
|
|
6762
6799
|
if (pause !== null) await pause;
|
|
6763
6800
|
}
|
|
6801
|
+
flushed += pending;
|
|
6764
6802
|
pending = 0;
|
|
6765
6803
|
};
|
|
6766
6804
|
const ingest = (id) => {
|
|
@@ -6773,18 +6811,27 @@ async function bucketIds(ids, codec, clock) {
|
|
|
6773
6811
|
bucket.push(remainder);
|
|
6774
6812
|
return ++pending >= BULK_FLUSH_IDS;
|
|
6775
6813
|
};
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6814
|
+
try {
|
|
6815
|
+
if (Symbol.asyncIterator in ids) {
|
|
6816
|
+
for await (const id of ids) {
|
|
6817
|
+
if (ingest(id)) await flushPending();
|
|
6818
|
+
const pause = tickId();
|
|
6819
|
+
if (pause !== null) await pause;
|
|
6820
|
+
}
|
|
6821
|
+
} else {
|
|
6822
|
+
for (const id of ids) {
|
|
6823
|
+
if (ingest(id)) await flushPending();
|
|
6824
|
+
const pause = tickId();
|
|
6825
|
+
if (pause !== null) await pause;
|
|
6826
|
+
}
|
|
6781
6827
|
}
|
|
6782
|
-
}
|
|
6783
|
-
|
|
6784
|
-
|
|
6785
|
-
|
|
6786
|
-
|
|
6828
|
+
} catch (err) {
|
|
6829
|
+
if (isBadIdError(err)) {
|
|
6830
|
+
const positioned = new ValidationError(`ids[${flushed + pending}]: ${err.message}`);
|
|
6831
|
+
positioned.cause = err;
|
|
6832
|
+
throw positioned;
|
|
6787
6833
|
}
|
|
6834
|
+
throw err;
|
|
6788
6835
|
}
|
|
6789
6836
|
await flushPending();
|
|
6790
6837
|
return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
|
|
@@ -6794,20 +6841,28 @@ function cleartextUnderRequiredEncryption(ref, currentGen) {
|
|
|
6794
6841
|
`requireEncryption: segment "${ref.segment}" already has generation ${currentGen} in cleartext, so this load cannot be encrypted \u2014 encryption is chosen when a segment is first loaded. Load into a new segment with the keystore wired, then drop this one.`
|
|
6795
6842
|
);
|
|
6796
6843
|
}
|
|
6797
|
-
|
|
6844
|
+
function requireKeystoreWhenEncryptionRequired(options) {
|
|
6798
6845
|
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
6799
6846
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
6800
6847
|
}
|
|
6848
|
+
}
|
|
6849
|
+
function refuseCleartextWhenEncryptionRequired(options, ref, currentGen) {
|
|
6850
|
+
if (options.requireEncryption === true) {
|
|
6851
|
+
throw cleartextUnderRequiredEncryption(ref, currentGen);
|
|
6852
|
+
}
|
|
6853
|
+
}
|
|
6854
|
+
async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
6855
|
+
requireKeystoreWhenEncryptionRequired(options);
|
|
6801
6856
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
6857
|
+
const entropy = requireEntropy(options.entropy, "a load");
|
|
6802
6858
|
let chunks;
|
|
6803
6859
|
if (ids instanceof ChunkLoadInput) {
|
|
6804
6860
|
chunks = encodeEach(rangeCheckedAtWrite(await collectChunks(ids, codec, options.clock)));
|
|
6805
6861
|
} else {
|
|
6806
6862
|
chunks = await encodedChunksOfInput(ids, codec, options.clock);
|
|
6807
6863
|
}
|
|
6808
|
-
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6864
|
+
if (options.keystore !== void 0 && options.registry === void 0)
|
|
6809
6865
|
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
6810
|
-
}
|
|
6811
6866
|
const passed = options.row;
|
|
6812
6867
|
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(ref) : null;
|
|
6813
6868
|
if (existing?.status === "destroyed") {
|
|
@@ -6830,8 +6885,7 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6830
6885
|
const aead = early !== void 0 && sameKeys(early.wrapped, existing.wrappedDeks) ? await early.aead : await options.keystore.openDek(existing.wrappedDeks);
|
|
6831
6886
|
crypto = { aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
6832
6887
|
} else if (existing !== null && existing.currentGen !== null) {
|
|
6833
|
-
|
|
6834
|
-
throw cleartextUnderRequiredEncryption(ref, existing.currentGen);
|
|
6888
|
+
refuseCleartextWhenEncryptionRequired(options, ref, existing.currentGen);
|
|
6835
6889
|
} else {
|
|
6836
6890
|
const minted = await options.keystore.createDek();
|
|
6837
6891
|
newWrapped = minted.wrapped;
|
|
@@ -6841,7 +6895,8 @@ async function bulkLoadAhead(driver, ref, ahead, ids, options = {}) {
|
|
|
6841
6895
|
const { size, sha256, chunkCount, cardinality, fingerprint, tailBytes } = await writeEncodedChunks(driver, key, chunks, {
|
|
6842
6896
|
crypto,
|
|
6843
6897
|
clock: options.clock,
|
|
6844
|
-
metadata: options.metadata
|
|
6898
|
+
metadata: options.metadata,
|
|
6899
|
+
entropy
|
|
6845
6900
|
});
|
|
6846
6901
|
const summary = ownSummary(
|
|
6847
6902
|
key,
|
|
@@ -7380,6 +7435,10 @@ async function currentCardinality(ref, deps, record, unwrapped) {
|
|
|
7380
7435
|
function loadSegment(ref, input, deps, options = {}) {
|
|
7381
7436
|
return runLoad(ref, (codec) => prepareLoadInput(input, codec), deps, options);
|
|
7382
7437
|
}
|
|
7438
|
+
function startUnwrap(row, keystore) {
|
|
7439
|
+
const wrappedKeys = row?.wrappedDeks;
|
|
7440
|
+
return row !== null && row.status !== "destroyed" && wrappedKeys !== void 0 && wrappedKeys.length > 0 && keystore !== void 0 ? { wrapped: wrappedKeys, aead: settled((async () => keystore.openDek(wrappedKeys))()) } : void 0;
|
|
7441
|
+
}
|
|
7383
7442
|
function loadSegmentChunks(ref, chunks, deps, options = {}) {
|
|
7384
7443
|
return runLoad(
|
|
7385
7444
|
ref,
|
|
@@ -7399,6 +7458,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7399
7458
|
validateUserRef(ref);
|
|
7400
7459
|
checkedAuditSink(options.audit, "load");
|
|
7401
7460
|
const codec = requireCodec(deps.codec, "loadSegment");
|
|
7461
|
+
const entropy = requireEntropy(deps.entropy, "a load");
|
|
7402
7462
|
const keep = options.keep ?? 1;
|
|
7403
7463
|
requireInteger(keep, { min: 0, what: "keep" });
|
|
7404
7464
|
const guard = checkedGuard(options.guard, "");
|
|
@@ -7411,9 +7471,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7411
7471
|
const row = await deps.registry.get(ref);
|
|
7412
7472
|
const fromToken = row?.token;
|
|
7413
7473
|
const fromGeneration = row?.currentGen ?? void 0;
|
|
7414
|
-
const
|
|
7415
|
-
const keystore = deps.keystore;
|
|
7416
|
-
const unwrapped = row !== null && row.status !== "destroyed" && wrappedKeys !== void 0 && wrappedKeys.length > 0 && keystore !== void 0 ? { wrapped: wrappedKeys, aead: settled((async () => keystore.openDek(wrappedKeys))()) } : void 0;
|
|
7474
|
+
const unwrapped = startUnwrap(row, deps.keystore);
|
|
7417
7475
|
const needsBefore = sizeNeeded(guard, options.allowEmpty);
|
|
7418
7476
|
const current = needsBefore ? await currentCardinality(ref, deps, row, unwrapped?.aead) : { cardinality: null, fromSummary: false };
|
|
7419
7477
|
const before = current.cardinality;
|
|
@@ -7432,6 +7490,7 @@ async function runLoad(ref, prepare, deps, options) {
|
|
|
7432
7490
|
requireEncryption: deps.requireEncryption,
|
|
7433
7491
|
codec: deps.codec,
|
|
7434
7492
|
clock: deps.clock,
|
|
7493
|
+
entropy,
|
|
7435
7494
|
metadata,
|
|
7436
7495
|
row
|
|
7437
7496
|
}
|
|
@@ -7770,16 +7829,16 @@ function collectFetch(node, demanded, operands, planned, fetch) {
|
|
|
7770
7829
|
}
|
|
7771
7830
|
for (const kid of node.kids) collectFetch(kid, here, operands, planned, fetch);
|
|
7772
7831
|
}
|
|
7773
|
-
function canHold(node,
|
|
7832
|
+
function canHold(node, holds2) {
|
|
7774
7833
|
switch (node.kind) {
|
|
7775
7834
|
case "leaf":
|
|
7776
|
-
return
|
|
7835
|
+
return holds2(node.operand);
|
|
7777
7836
|
case "and":
|
|
7778
|
-
return node.kids.every((kid) => canHold(kid,
|
|
7837
|
+
return node.kids.every((kid) => canHold(kid, holds2));
|
|
7779
7838
|
case "or":
|
|
7780
|
-
return node.kids.some((kid) => canHold(kid,
|
|
7839
|
+
return node.kids.some((kid) => canHold(kid, holds2));
|
|
7781
7840
|
case "andNot":
|
|
7782
|
-
return canHold(node.kids[0],
|
|
7841
|
+
return canHold(node.kids[0], holds2);
|
|
7783
7842
|
}
|
|
7784
7843
|
}
|
|
7785
7844
|
var KeyBits = class {
|
|
@@ -7857,11 +7916,12 @@ var ResidentLedger = class {
|
|
|
7857
7916
|
return true;
|
|
7858
7917
|
}
|
|
7859
7918
|
/**
|
|
7860
|
-
* Charge `bytes`, asking `makeRoom` to free some while they do not fit. `makeRoom`
|
|
7861
|
-
* {@link release}) and answers whether it freed anything; the charge is refused when
|
|
7919
|
+
* Charge `bytes`, asking `makeRoom` to free some while they do not fit. `makeRoom` is told how many bytes are wanted,
|
|
7920
|
+
* releases what it frees (through {@link release}) and answers whether it freed anything; the charge is refused when
|
|
7921
|
+
* it cannot.
|
|
7862
7922
|
*/
|
|
7863
7923
|
reserve(bytes, makeRoom) {
|
|
7864
|
-
while (!this.tryCharge(bytes)) if (!makeRoom()) return false;
|
|
7924
|
+
while (!this.tryCharge(bytes)) if (!makeRoom(bytes)) return false;
|
|
7865
7925
|
return true;
|
|
7866
7926
|
}
|
|
7867
7927
|
release(bytes) {
|
|
@@ -8523,8 +8583,9 @@ var GroupCost = class {
|
|
|
8523
8583
|
}
|
|
8524
8584
|
};
|
|
8525
8585
|
var planWorkOf = (o) => o.rootKeys.byteLength * (1 + 2 * o.compiled.operands.length);
|
|
8526
|
-
var budgetError = (limit, what) => new BudgetExceededError(
|
|
8527
|
-
`materializeMany: ${what} would hold more than maxBufferedBytes (${limit}) resident \u2014 raise maxBufferedBytes, or split the outputs across calls
|
|
8586
|
+
var budgetError = (limit, spent, what) => new BudgetExceededError(
|
|
8587
|
+
`materializeMany: ${what} would hold more than maxBufferedBytes (${limit}) resident \u2014 raise maxBufferedBytes, or split the outputs across calls`,
|
|
8588
|
+
{ option: "maxBufferedBytes", limit, spent }
|
|
8528
8589
|
);
|
|
8529
8590
|
async function runCombineMany(compiled, deps) {
|
|
8530
8591
|
const run = new Run(compiled, deps);
|
|
@@ -8648,8 +8709,8 @@ var Run = class {
|
|
|
8648
8709
|
codec: this.codec,
|
|
8649
8710
|
base: this.fedBase,
|
|
8650
8711
|
charge: (bytes) => {
|
|
8651
|
-
if (!this.ledger.reserve(bytes, () => this.reclaim())) {
|
|
8652
|
-
throw
|
|
8712
|
+
if (!this.ledger.reserve(bytes, (needed) => this.reclaim(needed))) {
|
|
8713
|
+
throw this.refuse(bytes, "the feed");
|
|
8653
8714
|
}
|
|
8654
8715
|
},
|
|
8655
8716
|
release: (bytes) => this.ledger.release(bytes),
|
|
@@ -8682,20 +8743,24 @@ var Run = class {
|
|
|
8682
8743
|
this.fail(o, error instanceof FeedStale ? this.staleError(o) : error);
|
|
8683
8744
|
}
|
|
8684
8745
|
}
|
|
8746
|
+
/** The refusal for `needed` more resident bytes than the ledger has room for: what it held plus what it was asked for. */
|
|
8747
|
+
refuse(needed, what) {
|
|
8748
|
+
return budgetError(this.req.maxBufferedBytes, this.ledger.used + needed, what);
|
|
8749
|
+
}
|
|
8685
8750
|
/**
|
|
8686
8751
|
* Make room in the ledger. The plans kept for later groups give way first: they are rebuilt when their group starts,
|
|
8687
8752
|
* and the outputs were judged without them. A fed output cannot be run again (the feed is read once), so a fed call
|
|
8688
8753
|
* gives up its fed outputs and the feed next, as soon as the ledger passes the budget; only then is a stored-only
|
|
8689
8754
|
* buffer evicted, to run again alone.
|
|
8690
8755
|
*/
|
|
8691
|
-
reclaim() {
|
|
8756
|
+
reclaim(needed) {
|
|
8692
8757
|
if (this.dropKeptPlans()) return true;
|
|
8693
8758
|
if (this.feed !== void 0) {
|
|
8694
8759
|
let fed = this.feed.heldBytes;
|
|
8695
8760
|
for (const o of this.running)
|
|
8696
8761
|
if (o.compiled.fed.length > 0 && o.status === "live") fed += o.charged;
|
|
8697
8762
|
if (fed > 0) {
|
|
8698
|
-
this.failFed(
|
|
8763
|
+
this.failFed(this.refuse(needed, "the feed and the fed outputs"));
|
|
8699
8764
|
return true;
|
|
8700
8765
|
}
|
|
8701
8766
|
}
|
|
@@ -8783,7 +8848,12 @@ var Run = class {
|
|
|
8783
8848
|
return;
|
|
8784
8849
|
}
|
|
8785
8850
|
throw new BudgetExceededError(
|
|
8786
|
-
`materializeMany: the plan alone (${what}) passes maxBufferedBytes (${this.req.maxBufferedBytes}) with ${this.planCharge} bytes already held for it, before any chunk is read \u2014 raise maxBufferedBytes, or split the outputs across calls
|
|
8851
|
+
`materializeMany: the plan alone (${what}) passes maxBufferedBytes (${this.req.maxBufferedBytes}) with ${this.planCharge} bytes already held for it, before any chunk is read \u2014 raise maxBufferedBytes, or split the outputs across calls`,
|
|
8852
|
+
{
|
|
8853
|
+
option: "maxBufferedBytes",
|
|
8854
|
+
limit: this.req.maxBufferedBytes,
|
|
8855
|
+
spent: this.ledger.used + bytes
|
|
8856
|
+
}
|
|
8787
8857
|
);
|
|
8788
8858
|
}
|
|
8789
8859
|
planOutputs() {
|
|
@@ -8861,7 +8931,8 @@ var Run = class {
|
|
|
8861
8931
|
this.fail(
|
|
8862
8932
|
o,
|
|
8863
8933
|
new BudgetExceededError(
|
|
8864
|
-
`materializeMany: output ${o.index} needs about ${alone} bytes resident with its operands, and the plan holds ${base}, over maxBufferedBytes (${this.req.maxBufferedBytes}) \u2014 raise maxBufferedBytes to at least ${alone + base}
|
|
8934
|
+
`materializeMany: output ${o.index} needs about ${alone} bytes resident with its operands, and the plan holds ${base}, over maxBufferedBytes (${this.req.maxBufferedBytes}) \u2014 raise maxBufferedBytes to at least ${alone + base}`,
|
|
8935
|
+
{ option: "maxBufferedBytes", limit: this.req.maxBufferedBytes, spent: alone + base }
|
|
8865
8936
|
)
|
|
8866
8937
|
);
|
|
8867
8938
|
continue;
|
|
@@ -8957,7 +9028,8 @@ var Run = class {
|
|
|
8957
9028
|
this.limit = given.maxRequests;
|
|
8958
9029
|
if (this.planned > this.limit) {
|
|
8959
9030
|
throw new BudgetExceededError(
|
|
8960
|
-
`materializeMany would read ${this.planned} chunks over ${groups.length} group(s), over the budget of ${this.limit} \u2014 raise \`budget.maxRequests\`, raise maxBufferedBytes so fewer groups re-read the operands, or set \`budget: false
|
|
9031
|
+
`materializeMany would read ${this.planned} chunks over ${groups.length} group(s), over the budget of ${this.limit} \u2014 raise \`budget.maxRequests\`, raise maxBufferedBytes so fewer groups re-read the operands, or set \`budget: false\``,
|
|
9032
|
+
{ option: "budget.maxRequests", limit: this.limit, spent: this.planned }
|
|
8961
9033
|
);
|
|
8962
9034
|
}
|
|
8963
9035
|
}
|
|
@@ -8990,7 +9062,8 @@ var Run = class {
|
|
|
8990
9062
|
const budget = this.limit;
|
|
8991
9063
|
if (budget !== null && this.chunkReads + plan.reads > budget) {
|
|
8992
9064
|
const error = new BudgetExceededError(
|
|
8993
|
-
`materializeMany: re-running outputs would read ${plan.reads} more chunks, past the budget of ${budget} (${this.chunkReads} read)
|
|
9065
|
+
`materializeMany: re-running outputs would read ${plan.reads} more chunks, past the budget of ${budget} (${this.chunkReads} read)`,
|
|
9066
|
+
{ option: "budget.maxRequests", limit: budget, spent: this.chunkReads + plan.reads }
|
|
8994
9067
|
);
|
|
8995
9068
|
for (const o of members) this.fail(o, error);
|
|
8996
9069
|
return;
|
|
@@ -9006,7 +9079,7 @@ var Run = class {
|
|
|
9006
9079
|
}
|
|
9007
9080
|
const fixed = plan.fixedBytes + plan.planBytes;
|
|
9008
9081
|
if (!this.ledger.tryCharge(fixed) && !(this.dropKeptPlans() && this.ledger.tryCharge(fixed))) {
|
|
9009
|
-
const error =
|
|
9082
|
+
const error = this.refuse(fixed, "the operand streams");
|
|
9010
9083
|
for (const o of members) this.fail(o, error);
|
|
9011
9084
|
return;
|
|
9012
9085
|
}
|
|
@@ -9084,7 +9157,7 @@ var Run = class {
|
|
|
9084
9157
|
for (const o of victims) if (o.status === "live") this.fail(o, error);
|
|
9085
9158
|
};
|
|
9086
9159
|
const w = this.window === "empty" ? null : this.window;
|
|
9087
|
-
const makeRoom = () => this.reclaim();
|
|
9160
|
+
const makeRoom = (needed) => this.reclaim(needed);
|
|
9088
9161
|
const feed = this.feed !== void 0 && this.feed.active ? this.feed : void 0;
|
|
9089
9162
|
const fedBase = this.fedBase;
|
|
9090
9163
|
let at = 0;
|
|
@@ -9115,8 +9188,8 @@ var Run = class {
|
|
|
9115
9188
|
if (!(error instanceof FeedClosed)) this.failFed(error);
|
|
9116
9189
|
}
|
|
9117
9190
|
}
|
|
9118
|
-
const
|
|
9119
|
-
const needs = (o) => o.compiled.fed.length === 0 || canHold(o.compiled.node,
|
|
9191
|
+
const holds2 = (operand) => operand >= fedBase ? fedHere?.has(operand) === true : hasKey(this.operands[operand].bounds.keys, key);
|
|
9192
|
+
const needs = (o) => o.compiled.fed.length === 0 || canHold(o.compiled.node, holds2);
|
|
9120
9193
|
try {
|
|
9121
9194
|
const exact = feed !== void 0 || members.some((o) => o.status !== "live");
|
|
9122
9195
|
const cursorsAt = [];
|
|
@@ -9181,10 +9254,7 @@ var Run = class {
|
|
|
9181
9254
|
try {
|
|
9182
9255
|
const resident = residentBytes(bytes.length);
|
|
9183
9256
|
if (!this.ledger.reserve(resident, makeRoom)) {
|
|
9184
|
-
throw
|
|
9185
|
-
this.req.maxBufferedBytes,
|
|
9186
|
-
`operand "${c.op.spec.name}" at chunk ${key}`
|
|
9187
|
-
);
|
|
9257
|
+
throw this.refuse(resident, `operand "${c.op.spec.name}" at chunk ${key}`);
|
|
9188
9258
|
}
|
|
9189
9259
|
held += resident;
|
|
9190
9260
|
decoded.set(
|
|
@@ -9277,7 +9347,7 @@ var Run = class {
|
|
|
9277
9347
|
const resident = residentBound(size);
|
|
9278
9348
|
kc.drop(held);
|
|
9279
9349
|
if (!this.ledger.reserve(resident, makeRoom)) {
|
|
9280
|
-
throw
|
|
9350
|
+
throw this.refuse(resident, `output ${o.index}`);
|
|
9281
9351
|
}
|
|
9282
9352
|
if (o.status !== "live") {
|
|
9283
9353
|
this.ledger.release(resident);
|
|
@@ -9357,7 +9427,7 @@ var Run = class {
|
|
|
9357
9427
|
}
|
|
9358
9428
|
if (!admitted) {
|
|
9359
9429
|
if (o.solo || o.compiled.fed.length > 0) {
|
|
9360
|
-
this.fail(o,
|
|
9430
|
+
this.fail(o, this.refuse(transient, `publishing output ${o.index}`));
|
|
9361
9431
|
} else {
|
|
9362
9432
|
this.dropBuffer(o);
|
|
9363
9433
|
o.status = "deferred";
|
|
@@ -9532,7 +9602,7 @@ var KeyContext = class {
|
|
|
9532
9602
|
/** Charge `bytes` for a bitmap this evaluation made. */
|
|
9533
9603
|
take(bytes) {
|
|
9534
9604
|
if (!this.ledger.reserve(bytes, this.makeRoom)) {
|
|
9535
|
-
throw budgetError(this.ledger.limit, "a chunk being evaluated");
|
|
9605
|
+
throw budgetError(this.ledger.limit, this.ledger.used + bytes, "a chunk being evaluated");
|
|
9536
9606
|
}
|
|
9537
9607
|
this.temp += bytes;
|
|
9538
9608
|
}
|
|
@@ -9757,7 +9827,13 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
9757
9827
|
const seen = await deps.registry.get(ref);
|
|
9758
9828
|
const settled2 = await churn.settle(record, seen, () => deps.registry.get(ref));
|
|
9759
9829
|
const now = settled2?.row ?? null;
|
|
9760
|
-
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now))
|
|
9830
|
+
if (settled2 === void 0 || now === null || !onlyLeasesDiffer(record, now)) {
|
|
9831
|
+
const lost = new WriteConflictError(
|
|
9832
|
+
"rollback: the segment's row changed while the rollback ran and the pointer was not moved; check store.generations(ref) and run it again"
|
|
9833
|
+
);
|
|
9834
|
+
lost.cause = err;
|
|
9835
|
+
throw lost;
|
|
9836
|
+
}
|
|
9761
9837
|
against = now.token;
|
|
9762
9838
|
}
|
|
9763
9839
|
}
|
|
@@ -9934,6 +10010,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9934
10010
|
checkedAuditSink(options.audit, "eraseIdFromSegment");
|
|
9935
10011
|
const { chunkKey, remainder } = splitId(id);
|
|
9936
10012
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
10013
|
+
requireEntropy(deps.entropy, "eraseIdFromSegment");
|
|
9937
10014
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
9938
10015
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
9939
10016
|
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
@@ -9945,510 +10022,560 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
9945
10022
|
const pointerless = !tombstoned && record.currentGen === null;
|
|
9946
10023
|
const from = record.currentGen ?? -1;
|
|
9947
10024
|
const fromToken = record.token;
|
|
9948
|
-
|
|
9949
|
-
let premiseToken = fromToken;
|
|
9950
|
-
let aead;
|
|
9951
|
-
if (tombstoned) {
|
|
9952
|
-
} else if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
9953
|
-
if (deps.keystore === void 0) {
|
|
9954
|
-
throw new KeyUnavailableError(
|
|
9955
|
-
`segment "${ref.segment}" is encrypted but eraseIdFromSegment has no keystore`
|
|
9956
|
-
);
|
|
9957
|
-
}
|
|
9958
|
-
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
9959
|
-
} else if (!pointerless && deps.requireEncryption === true) {
|
|
9960
|
-
throw new ValidationError(`requireEncryption: segment "${ref.segment}" is cleartext`);
|
|
9961
|
-
}
|
|
9962
|
-
const cryptoAt = (generation2) => aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation2, scope) };
|
|
10025
|
+
const aead = await resolveDek(ref, deps, record, { tombstoned, pointerless });
|
|
9963
10026
|
const fromKey = { ...base, generation: from };
|
|
9964
|
-
const
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
9977
|
-
|
|
9978
|
-
|
|
10027
|
+
const c = {
|
|
10028
|
+
ref,
|
|
10029
|
+
deps,
|
|
10030
|
+
seen,
|
|
10031
|
+
audit,
|
|
10032
|
+
chunkKey,
|
|
10033
|
+
remainder,
|
|
10034
|
+
codec,
|
|
10035
|
+
maxBytes,
|
|
10036
|
+
base,
|
|
10037
|
+
record,
|
|
10038
|
+
tombstoned,
|
|
10039
|
+
pointerless,
|
|
10040
|
+
from,
|
|
10041
|
+
fromToken,
|
|
10042
|
+
aead,
|
|
10043
|
+
fromKey,
|
|
10044
|
+
read,
|
|
10045
|
+
sealed: /* @__PURE__ */ new Set(),
|
|
10046
|
+
versions: /* @__PURE__ */ new Map(),
|
|
10047
|
+
premise: record,
|
|
10048
|
+
premiseToken: fromToken,
|
|
10049
|
+
fenceLost: 0,
|
|
10050
|
+
rewrote: void 0
|
|
9979
10051
|
};
|
|
9980
|
-
|
|
9981
|
-
|
|
9982
|
-
const
|
|
9983
|
-
|
|
9984
|
-
|
|
10052
|
+
if (tombstoned) return await underTombstone(c);
|
|
10053
|
+
if (pointerless) return await unpublished(c);
|
|
10054
|
+
const staged = await stage(c);
|
|
10055
|
+
if ("erased" in staged) {
|
|
10056
|
+
if (!staged.erased && staged.generation !== void 0 && c.rewrote !== void 0) {
|
|
10057
|
+
await discardRefused(c, c.rewrote);
|
|
10058
|
+
}
|
|
10059
|
+
return staged;
|
|
10060
|
+
}
|
|
10061
|
+
return await finishRewrite(c, staged);
|
|
10062
|
+
}
|
|
10063
|
+
function rowVerdict(c, row, alsoIgnore = []) {
|
|
10064
|
+
const { pointerless, from } = c;
|
|
10065
|
+
if (row === null) return "absent";
|
|
10066
|
+
if (row.status === "destroyed") return "destroyed";
|
|
10067
|
+
const same = row.token === c.premiseToken || onlyLeasesDiffer(c.premise, row, alsoIgnore);
|
|
10068
|
+
if (pointerless) return row.currentGen === null && same ? null : "superseded";
|
|
10069
|
+
if (row.currentGen === null) return "superseded";
|
|
10070
|
+
return row.currentGen === from && same ? null : "superseded";
|
|
10071
|
+
}
|
|
10072
|
+
function refusedResult(c, reason, generation) {
|
|
10073
|
+
const { base, from } = c;
|
|
10074
|
+
return { ...base, erased: false, reason, fromGeneration: from, generation, collected: [] };
|
|
10075
|
+
}
|
|
10076
|
+
function cryptoAt(c, generation) {
|
|
10077
|
+
const { ref, aead } = c;
|
|
10078
|
+
return aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
10079
|
+
}
|
|
10080
|
+
function found(c, generation, held) {
|
|
10081
|
+
return held === true && !c.sealed.has(generation);
|
|
10082
|
+
}
|
|
10083
|
+
async function holds(c, generation) {
|
|
10084
|
+
const { deps, chunkKey, remainder, codec, maxBytes, base, tombstoned, read, sealed, versions } = c;
|
|
10085
|
+
const key = { ...base, generation };
|
|
10086
|
+
const crypto = cryptoAt(c, generation);
|
|
10087
|
+
try {
|
|
10088
|
+
let reader;
|
|
10089
|
+
const index = { refused: false };
|
|
10090
|
+
let opened;
|
|
9985
10091
|
try {
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
() =>
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
);
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
() => objectIsEncrypted(deps.storage, key2, (version) => footer = version)
|
|
10003
|
-
);
|
|
10004
|
-
if (encrypted) {
|
|
10005
|
-
if (tombstoned) return false;
|
|
10006
|
-
sealed.add(generation2);
|
|
10007
|
-
versions.set(generation2, footer);
|
|
10008
|
-
return true;
|
|
10009
|
-
}
|
|
10010
|
-
throw err;
|
|
10011
|
-
}
|
|
10012
|
-
if (isIntegrityError(err) && index.refused) {
|
|
10013
|
-
sealed.add(generation2);
|
|
10014
|
-
versions.set(generation2, opened);
|
|
10092
|
+
reader = await read(
|
|
10093
|
+
() => openGenerationReaderNoting(
|
|
10094
|
+
deps.storage,
|
|
10095
|
+
key,
|
|
10096
|
+
crypto === void 0 ? void 0 : noticingIndex(crypto, index),
|
|
10097
|
+
(version) => opened = version
|
|
10098
|
+
)
|
|
10099
|
+
);
|
|
10100
|
+
} catch (err) {
|
|
10101
|
+
if (crypto === void 0) {
|
|
10102
|
+
let footer;
|
|
10103
|
+
const encrypted = !isTransientError(err) && await read(() => objectIsEncrypted(deps.storage, key, (version) => footer = version));
|
|
10104
|
+
if (encrypted) {
|
|
10105
|
+
if (tombstoned) return false;
|
|
10106
|
+
sealed.add(generation);
|
|
10107
|
+
versions.set(generation, footer);
|
|
10015
10108
|
return true;
|
|
10016
10109
|
}
|
|
10017
|
-
|
|
10018
|
-
throw err;
|
|
10019
|
-
}
|
|
10020
|
-
reader = await read(() => openGenerationReader(deps.storage, key2, void 0));
|
|
10110
|
+
throw err;
|
|
10021
10111
|
}
|
|
10022
|
-
|
|
10023
|
-
|
|
10024
|
-
|
|
10025
|
-
|
|
10026
|
-
}
|
|
10027
|
-
|
|
10028
|
-
return null;
|
|
10029
|
-
}
|
|
10112
|
+
if (isIntegrityError(err) && index.refused) {
|
|
10113
|
+
sealed.add(generation);
|
|
10114
|
+
versions.set(generation, opened);
|
|
10115
|
+
return true;
|
|
10116
|
+
}
|
|
10117
|
+
if (!isIntegrityError(err) || await read(() => objectIsEncrypted(deps.storage, key))) {
|
|
10030
10118
|
throw err;
|
|
10031
10119
|
}
|
|
10032
|
-
|
|
10120
|
+
reader = await read(() => openGenerationReader(deps.storage, key, void 0));
|
|
10121
|
+
}
|
|
10122
|
+
versions.set(generation, objectVersionOf(reader));
|
|
10123
|
+
let bytes;
|
|
10124
|
+
try {
|
|
10125
|
+
bytes = await read(() => reader.getChunk(chunkKey));
|
|
10033
10126
|
} catch (err) {
|
|
10034
|
-
if (
|
|
10127
|
+
if (!isTransientError(err) && !await read(() => sameObjectUnder(deps.storage, key, reader.fingerprint))) {
|
|
10128
|
+
return null;
|
|
10129
|
+
}
|
|
10035
10130
|
throw err;
|
|
10036
10131
|
}
|
|
10037
|
-
|
|
10038
|
-
|
|
10039
|
-
|
|
10040
|
-
|
|
10041
|
-
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10050
|
-
|
|
10051
|
-
|
|
10052
|
-
|
|
10053
|
-
|
|
10054
|
-
|
|
10132
|
+
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
10133
|
+
} catch (err) {
|
|
10134
|
+
if (isNotFoundError(err)) return null;
|
|
10135
|
+
throw err;
|
|
10136
|
+
}
|
|
10137
|
+
}
|
|
10138
|
+
async function holdsEach(c, generations, { stopAtHolder }) {
|
|
10139
|
+
let firstHolder = Infinity;
|
|
10140
|
+
const settled2 = await mapWithConcurrency(
|
|
10141
|
+
generations,
|
|
10142
|
+
HOLDS_CONCURRENCY,
|
|
10143
|
+
async (generation, index) => {
|
|
10144
|
+
if (stopAtHolder && index > firstHolder) {
|
|
10145
|
+
return { generation, held: null, fault: void 0 };
|
|
10146
|
+
}
|
|
10147
|
+
try {
|
|
10148
|
+
const held = await holds(c, generation);
|
|
10149
|
+
if (found(c, generation, held) && index < firstHolder) firstHolder = index;
|
|
10150
|
+
return { generation, held, fault: void 0 };
|
|
10151
|
+
} catch (fault) {
|
|
10152
|
+
return { generation, held: null, fault: { error: fault } };
|
|
10055
10153
|
}
|
|
10056
|
-
);
|
|
10057
|
-
const outcomes = [];
|
|
10058
|
-
for (const { generation: generation2, held, fault } of settled2) {
|
|
10059
|
-
if (fault !== void 0) throw fault.error;
|
|
10060
|
-
outcomes.push({ generation: generation2, held });
|
|
10061
|
-
if (stopAtHolder && found(generation2, held)) break;
|
|
10062
10154
|
}
|
|
10063
|
-
return outcomes;
|
|
10064
|
-
};
|
|
10065
|
-
const holderLeft = async (clean) => {
|
|
10066
|
-
const toRead = (await generationsOf(deps.storage, ref)).reverse().filter((generation2) => !clean.has(generation2));
|
|
10067
|
-
return (await holdsEach(toRead, { stopAtHolder: true })).find(
|
|
10068
|
-
(outcome) => outcome.held === true
|
|
10069
|
-
)?.generation;
|
|
10070
|
-
};
|
|
10071
|
-
const cannotRemove = (generation2) => new WriteConflictError(
|
|
10072
|
-
`erasure of segment ${ref.segment} could not remove generation ${generation2}, which ${sealed.has(generation2) ? "cannot be searched, so may hold the id," : "still holds the id"} and is still in the bucket; re-run`
|
|
10073
10155
|
);
|
|
10074
|
-
|
|
10075
|
-
const
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10090
|
-
|
|
10091
|
-
|
|
10092
|
-
|
|
10093
|
-
|
|
10094
|
-
|
|
10095
|
-
|
|
10096
|
-
|
|
10097
|
-
|
|
10098
|
-
|
|
10099
|
-
|
|
10100
|
-
|
|
10101
|
-
|
|
10102
|
-
|
|
10103
|
-
|
|
10104
|
-
|
|
10105
|
-
|
|
10106
|
-
const below = plan.sweeps && !pointerless && pointer !== null && pointer > from && plan.byName.every((h) => h <= pointer);
|
|
10107
|
-
if (!below) return refuse(pointerless || pointer !== null ? "superseded" : "no-generation");
|
|
10108
|
-
byName = [];
|
|
10109
|
-
moved = "superseded";
|
|
10110
|
-
pastHolders = true;
|
|
10111
|
-
}
|
|
10112
|
-
const ledger = usableKnownGens(row);
|
|
10113
|
-
const doomed = new Set(byName);
|
|
10114
|
-
if (plan.sweeps && ledger !== void 0 && pointer !== null) {
|
|
10115
|
-
const bound = Math.min(plan.below ?? from, pointer);
|
|
10116
|
-
for (const e of ledger) if (e.generation < bound) doomed.add(e.generation);
|
|
10117
|
-
if (byName.length === 0) {
|
|
10118
|
-
for (const h of plan.byName) if (h < pointer) doomed.add(h);
|
|
10119
|
-
}
|
|
10120
|
-
}
|
|
10121
|
-
if (pointer !== null) doomed.delete(pointer);
|
|
10122
|
-
const named = ledger !== void 0 && ledger.some((e) => doomed.has(e.generation));
|
|
10123
|
-
const renew = byName.length > 0;
|
|
10124
|
-
const renewedAfter = held && row.pointerId !== first.pointerId;
|
|
10125
|
-
if (!named && (!renew || renewedAfter)) {
|
|
10126
|
-
premise = row;
|
|
10127
|
-
premiseToken = row.token;
|
|
10128
|
-
return {
|
|
10129
|
-
moved,
|
|
10130
|
-
sweep: plan.sweeps,
|
|
10131
|
-
byName,
|
|
10132
|
-
ledger: ledger !== void 0,
|
|
10133
|
-
pruned: false,
|
|
10134
|
-
past: pastHolders
|
|
10135
|
-
};
|
|
10136
|
-
}
|
|
10137
|
-
const written = await renewal.write(row, {
|
|
10138
|
-
...renew ? renewPointer(row) : {},
|
|
10139
|
-
...named ? { knownGens: pruneKnown(row, doomed) } : {}
|
|
10140
|
-
});
|
|
10141
|
-
if ("row" in written) {
|
|
10142
|
-
premise = written.row;
|
|
10143
|
-
premiseToken = written.row.token;
|
|
10144
|
-
return {
|
|
10145
|
-
moved,
|
|
10146
|
-
sweep: plan.sweeps,
|
|
10147
|
-
byName,
|
|
10148
|
-
ledger: ledger !== void 0,
|
|
10149
|
-
pruned: named,
|
|
10150
|
-
past: pastHolders
|
|
10151
|
-
};
|
|
10152
|
-
}
|
|
10153
|
-
const now = written.lost;
|
|
10154
|
-
const settled2 = await churn.settle(row, now, () => deps.registry.get(ref));
|
|
10155
|
-
if (settled2 !== void 0) {
|
|
10156
|
-
row = settled2.row;
|
|
10157
|
-
continue;
|
|
10158
|
-
}
|
|
10159
|
-
fenceLost += 1;
|
|
10160
|
-
if (fenceLost >= FENCE_ATTEMPTS) return refuse("superseded");
|
|
10161
|
-
row = now;
|
|
10156
|
+
const outcomes = [];
|
|
10157
|
+
for (const { generation, held, fault } of settled2) {
|
|
10158
|
+
if (fault !== void 0) throw fault.error;
|
|
10159
|
+
outcomes.push({ generation, held });
|
|
10160
|
+
if (stopAtHolder && found(c, generation, held)) break;
|
|
10161
|
+
}
|
|
10162
|
+
return outcomes;
|
|
10163
|
+
}
|
|
10164
|
+
async function holderLeft(c, clean) {
|
|
10165
|
+
const { ref, deps } = c;
|
|
10166
|
+
const toRead = (await generationsOf(deps.storage, ref)).reverse().filter((generation) => !clean.has(generation));
|
|
10167
|
+
return (await holdsEach(c, toRead, { stopAtHolder: true })).find(
|
|
10168
|
+
(outcome) => outcome.held === true
|
|
10169
|
+
)?.generation;
|
|
10170
|
+
}
|
|
10171
|
+
function cannotRemove(c, generation) {
|
|
10172
|
+
const { ref, sealed } = c;
|
|
10173
|
+
return new WriteConflictError(
|
|
10174
|
+
`erasure of segment ${ref.segment} could not remove generation ${generation}, which ${sealed.has(generation) ? "cannot be searched, so may hold the id," : "still holds the id"} and is still in the bucket; re-run`
|
|
10175
|
+
);
|
|
10176
|
+
}
|
|
10177
|
+
function fenceRefused(moved) {
|
|
10178
|
+
return { moved, sweep: false, byName: [], ledger: true, pruned: false };
|
|
10179
|
+
}
|
|
10180
|
+
function doomedGenerations(c, plan, byName, ledger, pointer) {
|
|
10181
|
+
const { from } = c;
|
|
10182
|
+
const doomed = new Set(byName);
|
|
10183
|
+
if (plan.sweeps && ledger !== void 0 && pointer !== null) {
|
|
10184
|
+
const bound = Math.min(plan.below ?? from, pointer);
|
|
10185
|
+
for (const e of ledger) if (e.generation < bound) doomed.add(e.generation);
|
|
10186
|
+
if (byName.length === 0) {
|
|
10187
|
+
for (const h of plan.byName) if (h < pointer) doomed.add(h);
|
|
10162
10188
|
}
|
|
10163
|
-
}
|
|
10164
|
-
|
|
10165
|
-
|
|
10189
|
+
}
|
|
10190
|
+
if (pointer !== null) doomed.delete(pointer);
|
|
10191
|
+
return doomed;
|
|
10192
|
+
}
|
|
10193
|
+
async function fence(c, plan) {
|
|
10194
|
+
const { ref, deps, record, pointerless, from } = c;
|
|
10195
|
+
const renewal = rowRenewal(ref, deps);
|
|
10196
|
+
const churn = leaseChurn({ clock: deps.clock, rng: renewal.rng });
|
|
10197
|
+
const first = await deps.registry.get(ref);
|
|
10198
|
+
let row = first;
|
|
10199
|
+
for (; ; ) {
|
|
10200
|
+
if (row === null || row.status === "destroyed" || !pointerless && row.currentGen === null) {
|
|
10201
|
+
if (row !== null && !sameIncarnation(record, row)) return fenceRefused("superseded");
|
|
10202
|
+
const reason = row === null ? "absent" : row.status === "destroyed" ? "destroyed" : "superseded";
|
|
10203
|
+
return {
|
|
10204
|
+
moved: plan.byName.length > 0 ? reason : null,
|
|
10205
|
+
sweep: plan.sweeps,
|
|
10206
|
+
byName: [],
|
|
10207
|
+
ledger: true,
|
|
10208
|
+
pruned: false
|
|
10209
|
+
};
|
|
10210
|
+
}
|
|
10211
|
+
if (!sameIncarnation(record, row)) return fenceRefused("superseded");
|
|
10212
|
+
const held = row.token === record.token || onlyLeasesDiffer(record, row, ["pointerId", "knownGens"]);
|
|
10213
|
+
const pointer = row.currentGen;
|
|
10214
|
+
let byName = plan.byName;
|
|
10166
10215
|
let moved = null;
|
|
10167
|
-
let
|
|
10168
|
-
|
|
10169
|
-
|
|
10170
|
-
|
|
10171
|
-
|
|
10172
|
-
|
|
10173
|
-
|
|
10174
|
-
|
|
10175
|
-
|
|
10176
|
-
|
|
10177
|
-
|
|
10178
|
-
|
|
10179
|
-
|
|
10216
|
+
let pastHolders = false;
|
|
10217
|
+
if (plan.byName.length > 0 && !held) {
|
|
10218
|
+
const below = plan.sweeps && !pointerless && pointer !== null && pointer > from && plan.byName.every((h) => h <= pointer);
|
|
10219
|
+
if (!below) return fenceRefused("superseded");
|
|
10220
|
+
byName = [];
|
|
10221
|
+
moved = "superseded";
|
|
10222
|
+
pastHolders = true;
|
|
10223
|
+
}
|
|
10224
|
+
const ledger = usableKnownGens(row);
|
|
10225
|
+
const doomed = doomedGenerations(c, plan, byName, ledger, pointer);
|
|
10226
|
+
const named = ledger !== void 0 && ledger.some((e) => doomed.has(e.generation));
|
|
10227
|
+
const renew = byName.length > 0;
|
|
10228
|
+
const renewedAfter = held && row.pointerId !== first.pointerId;
|
|
10229
|
+
if (!named && (!renew || renewedAfter)) {
|
|
10230
|
+
c.premise = row;
|
|
10231
|
+
c.premiseToken = row.token;
|
|
10232
|
+
return {
|
|
10233
|
+
moved,
|
|
10234
|
+
sweep: plan.sweeps,
|
|
10235
|
+
byName,
|
|
10236
|
+
ledger: ledger !== void 0,
|
|
10237
|
+
pruned: false,
|
|
10238
|
+
past: pastHolders
|
|
10239
|
+
};
|
|
10180
10240
|
}
|
|
10181
|
-
|
|
10182
|
-
|
|
10183
|
-
|
|
10184
|
-
|
|
10185
|
-
|
|
10186
|
-
|
|
10187
|
-
|
|
10188
|
-
|
|
10189
|
-
|
|
10190
|
-
|
|
10191
|
-
|
|
10192
|
-
|
|
10193
|
-
|
|
10194
|
-
|
|
10195
|
-
}
|
|
10196
|
-
if (moved !== null) break;
|
|
10197
|
-
try {
|
|
10198
|
-
await deps.storage.delete({ ...base, generation: generation2 }, { ifVersion: versions.get(generation2) });
|
|
10199
|
-
} catch (err) {
|
|
10200
|
-
if (!isWriteConflictError(err)) throw err;
|
|
10201
|
-
moved = rowVerdict(await deps.registry.get(ref)) ?? "superseded";
|
|
10202
|
-
break;
|
|
10203
|
-
}
|
|
10204
|
-
deleted.push(generation2);
|
|
10205
|
-
watch.deleted(generation2);
|
|
10241
|
+
const written = await renewal.write(row, {
|
|
10242
|
+
...renew ? renewPointer(row) : {},
|
|
10243
|
+
...named ? { knownGens: pruneKnown(row, doomed) } : {}
|
|
10244
|
+
});
|
|
10245
|
+
if ("row" in written) {
|
|
10246
|
+
c.premise = written.row;
|
|
10247
|
+
c.premiseToken = written.row.token;
|
|
10248
|
+
return {
|
|
10249
|
+
moved,
|
|
10250
|
+
sweep: plan.sweeps,
|
|
10251
|
+
byName,
|
|
10252
|
+
ledger: ledger !== void 0,
|
|
10253
|
+
pruned: named,
|
|
10254
|
+
past: pastHolders
|
|
10255
|
+
};
|
|
10206
10256
|
}
|
|
10207
|
-
const
|
|
10257
|
+
const now = written.lost;
|
|
10258
|
+
const settled2 = await churn.settle(row, now, () => deps.registry.get(ref));
|
|
10259
|
+
if (settled2 !== void 0) {
|
|
10260
|
+
row = settled2.row;
|
|
10261
|
+
continue;
|
|
10262
|
+
}
|
|
10263
|
+
c.fenceLost += 1;
|
|
10264
|
+
if (c.fenceLost >= FENCE_ATTEMPTS) return fenceRefused("superseded");
|
|
10265
|
+
row = now;
|
|
10266
|
+
}
|
|
10267
|
+
}
|
|
10268
|
+
async function collectAgain(c, start, plan) {
|
|
10269
|
+
const { ref, deps } = c;
|
|
10270
|
+
const deleted = [];
|
|
10271
|
+
let moved = null;
|
|
10272
|
+
let missing;
|
|
10273
|
+
let spared = start.spared;
|
|
10274
|
+
for (let passes = 0; passes < FENCE_ATTEMPTS && missing === void 0 && (!start.ledgered && passes === 0 || spared.length > 0); passes++) {
|
|
10275
|
+
const again = await fence(c, { byName: plan.byName, sweeps: true, below: plan.below });
|
|
10276
|
+
if (again.moved !== null && again.past !== true) {
|
|
10277
|
+
moved = again.moved;
|
|
10278
|
+
break;
|
|
10279
|
+
}
|
|
10280
|
+
if (!again.pruned) break;
|
|
10281
|
+
const more = await collectBelowPointer(ref, deps, { keep: 0, skipKnown: true });
|
|
10282
|
+
deleted.push(...more.deleted);
|
|
10283
|
+
missing = more.missing;
|
|
10284
|
+
spared = more.knownSpared ?? [];
|
|
10285
|
+
}
|
|
10286
|
+
return missing === void 0 ? { deleted, moved } : { deleted, moved, missing };
|
|
10287
|
+
}
|
|
10288
|
+
async function deleteFenced(c, holders, fenced) {
|
|
10289
|
+
const { ref, deps, base, versions } = c;
|
|
10290
|
+
const deleted = [];
|
|
10291
|
+
const watch = watchDeletes(deps, ref);
|
|
10292
|
+
let moved = fenced;
|
|
10293
|
+
for (const generation of holders) {
|
|
10294
|
+
if (moved !== null) break;
|
|
10295
|
+
const row = await deps.registry.get(ref);
|
|
10296
|
+
const missing2 = await watch.check(c.premise, row);
|
|
10208
10297
|
if (missing2 !== void 0) return { deleted, moved: "superseded", missing: missing2 };
|
|
10209
|
-
|
|
10210
|
-
|
|
10211
|
-
|
|
10212
|
-
|
|
10298
|
+
moved = rowVerdict(c, row, ["pointerId", "knownGens"]);
|
|
10299
|
+
if (moved === null && usableKnownGens(row)?.some((e) => e.generation === generation)) {
|
|
10300
|
+
moved = "superseded";
|
|
10301
|
+
}
|
|
10302
|
+
if (moved !== null) break;
|
|
10303
|
+
try {
|
|
10304
|
+
await deps.storage.delete({ ...base, generation }, { ifVersion: versions.get(generation) });
|
|
10305
|
+
} catch (err) {
|
|
10306
|
+
if (!isWriteConflictError(err)) throw err;
|
|
10307
|
+
moved = rowVerdict(c, await deps.registry.get(ref)) ?? "superseded";
|
|
10308
|
+
break;
|
|
10309
|
+
}
|
|
10310
|
+
deleted.push(generation);
|
|
10311
|
+
watch.deleted(generation);
|
|
10312
|
+
}
|
|
10313
|
+
const missing = await watch.settle(c.premise);
|
|
10314
|
+
if (missing !== void 0) return { deleted, moved: "superseded", missing };
|
|
10315
|
+
return { deleted, moved };
|
|
10316
|
+
}
|
|
10317
|
+
function notePointerOnDeleted(c, missing) {
|
|
10318
|
+
const { ref, seen } = c;
|
|
10319
|
+
seen.missing = pointerOnDeletedError(ref, { kind: "erasure" }, missing);
|
|
10320
|
+
}
|
|
10321
|
+
function collectedAll(c, fromGeneration, collected) {
|
|
10322
|
+
const { seen, base, record } = c;
|
|
10323
|
+
seen.collectedOn = record.token;
|
|
10324
|
+
seen.found = true;
|
|
10325
|
+
return {
|
|
10326
|
+
...base,
|
|
10327
|
+
erased: true,
|
|
10328
|
+
fromGeneration,
|
|
10329
|
+
collected: [...collected].sort(ascending)
|
|
10213
10330
|
};
|
|
10214
|
-
|
|
10215
|
-
|
|
10216
|
-
|
|
10331
|
+
}
|
|
10332
|
+
function sealedOnly(c, answer, collected) {
|
|
10333
|
+
const { seen, record } = c;
|
|
10334
|
+
if (collected.length > 0) seen.collectedOn = record.token;
|
|
10335
|
+
return { ...answer, collected: [...collected].sort(ascending) };
|
|
10336
|
+
}
|
|
10337
|
+
async function unpublished(c) {
|
|
10338
|
+
const { ref, deps, base, sealed } = c;
|
|
10339
|
+
const newestFirst = (await generationsOf(deps.storage, ref)).reverse();
|
|
10340
|
+
const holders = [];
|
|
10341
|
+
const clean = /* @__PURE__ */ new Set();
|
|
10342
|
+
for (const { generation, held } of await holdsEach(c, newestFirst, { stopAtHolder: false })) {
|
|
10343
|
+
if (held === true) holders.push(generation);
|
|
10344
|
+
else if (held === false) clean.add(generation);
|
|
10345
|
+
}
|
|
10346
|
+
const none = { ...base, erased: false, reason: "no-generation", collected: [] };
|
|
10347
|
+
if (holders.length === 0) return none;
|
|
10348
|
+
const newest = holders.find((generation) => !sealed.has(generation));
|
|
10349
|
+
const fenced = await fence(c, { byName: holders, sweeps: false });
|
|
10350
|
+
const { deleted, moved, missing } = await deleteFenced(c, fenced.byName, fenced.moved);
|
|
10351
|
+
const collected = [...deleted].sort(ascending);
|
|
10352
|
+
const left = await holderLeft(c, clean);
|
|
10353
|
+
if (missing !== void 0) notePointerOnDeleted(c, missing);
|
|
10354
|
+
if (left === void 0) {
|
|
10355
|
+
return newest === void 0 ? sealedOnly(c, none, collected) : collectedAll(c, newest, collected);
|
|
10356
|
+
}
|
|
10357
|
+
if (moved !== null) {
|
|
10217
10358
|
return {
|
|
10218
10359
|
...base,
|
|
10219
|
-
erased:
|
|
10220
|
-
|
|
10221
|
-
|
|
10360
|
+
erased: false,
|
|
10361
|
+
reason: moved,
|
|
10362
|
+
...newest === void 0 ? {} : { fromGeneration: newest },
|
|
10363
|
+
collected
|
|
10222
10364
|
};
|
|
10365
|
+
}
|
|
10366
|
+
throw cannotRemove(c, left);
|
|
10367
|
+
}
|
|
10368
|
+
async function underTombstone(c) {
|
|
10369
|
+
const { ref, deps, base } = c;
|
|
10370
|
+
const newestFirst = (await generationsOf(deps.storage, ref)).reverse();
|
|
10371
|
+
const holder = (await holdsEach(c, newestFirst, { stopAtHolder: true })).find(
|
|
10372
|
+
(h) => h.held === true
|
|
10373
|
+
);
|
|
10374
|
+
if (holder === void 0) return { ...base, erased: false, reason: "destroyed", collected: [] };
|
|
10375
|
+
const collected = await gcOrphanGenerations(ref, deps, { keep: 0 });
|
|
10376
|
+
const left = await holderLeft(c, /* @__PURE__ */ new Set());
|
|
10377
|
+
if (left !== void 0) throw cannotRemove(c, left);
|
|
10378
|
+
return collectedAll(c, holder.generation, collected);
|
|
10379
|
+
}
|
|
10380
|
+
async function notInCurrent(c) {
|
|
10381
|
+
const { ref, deps, base, from, sealed } = c;
|
|
10382
|
+
const notMember = {
|
|
10383
|
+
...base,
|
|
10384
|
+
erased: false,
|
|
10385
|
+
reason: "not-member",
|
|
10386
|
+
fromGeneration: from,
|
|
10387
|
+
collected: []
|
|
10223
10388
|
};
|
|
10224
|
-
const
|
|
10225
|
-
|
|
10226
|
-
|
|
10227
|
-
|
|
10228
|
-
const
|
|
10229
|
-
|
|
10230
|
-
|
|
10231
|
-
|
|
10232
|
-
|
|
10233
|
-
|
|
10234
|
-
|
|
10235
|
-
|
|
10236
|
-
|
|
10237
|
-
|
|
10238
|
-
|
|
10239
|
-
|
|
10240
|
-
const {
|
|
10241
|
-
|
|
10242
|
-
|
|
10243
|
-
|
|
10244
|
-
|
|
10245
|
-
|
|
10246
|
-
|
|
10247
|
-
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
const
|
|
10261
|
-
|
|
10389
|
+
const newestFirst = (await generationsOf(deps.storage, ref)).filter((g) => g !== from).reverse();
|
|
10390
|
+
if (newestFirst.length === 0) return notMember;
|
|
10391
|
+
const clean = /* @__PURE__ */ new Set([from]);
|
|
10392
|
+
const holdersAbove = [];
|
|
10393
|
+
for (const { generation, held } of await holdsEach(
|
|
10394
|
+
c,
|
|
10395
|
+
newestFirst.filter((g) => g > from),
|
|
10396
|
+
{ stopAtHolder: false }
|
|
10397
|
+
)) {
|
|
10398
|
+
if (held === true) holdersAbove.push(generation);
|
|
10399
|
+
else if (held === false) clean.add(generation);
|
|
10400
|
+
}
|
|
10401
|
+
const foundAbove = holdersAbove.find((generation) => !sealed.has(generation));
|
|
10402
|
+
let holderBelow;
|
|
10403
|
+
const sealedBelow = [];
|
|
10404
|
+
if (foundAbove === void 0) {
|
|
10405
|
+
for (const { generation, held } of await holdsEach(
|
|
10406
|
+
c,
|
|
10407
|
+
newestFirst.filter((g) => g < from),
|
|
10408
|
+
{ stopAtHolder: true }
|
|
10409
|
+
)) {
|
|
10410
|
+
if (found(c, generation, held)) holderBelow = generation;
|
|
10411
|
+
else if (held === true) sealedBelow.push(generation);
|
|
10412
|
+
else if (held === false) clean.add(generation);
|
|
10413
|
+
}
|
|
10414
|
+
}
|
|
10415
|
+
const newest = foundAbove ?? holderBelow;
|
|
10416
|
+
const fenced = newest === void 0 ? [...holdersAbove, ...sealedBelow] : holdersAbove;
|
|
10417
|
+
if (newest === void 0 && fenced.length === 0) return notMember;
|
|
10418
|
+
const sweeping = newest !== void 0;
|
|
10419
|
+
let moved = null;
|
|
10420
|
+
let byName = fenced;
|
|
10421
|
+
let sweep = sweeping;
|
|
10422
|
+
let ledgered = true;
|
|
10423
|
+
let past = false;
|
|
10424
|
+
if (sweeping || fenced.length > 0) {
|
|
10425
|
+
const made = await fence(c, { byName: fenced, sweeps: sweeping });
|
|
10426
|
+
moved = made.moved;
|
|
10427
|
+
byName = made.byName;
|
|
10428
|
+
sweep = made.sweep;
|
|
10429
|
+
ledgered = made.ledger;
|
|
10430
|
+
past = made.past === true;
|
|
10431
|
+
}
|
|
10432
|
+
const swept = sweep ? await collectBelowPointer(ref, deps, { keep: 0, skipKnown: true }) : { deleted: [] };
|
|
10433
|
+
const collected = [...swept.deleted];
|
|
10434
|
+
let missing = swept.missing;
|
|
10435
|
+
if (sweep && missing === void 0 && (moved === null || past)) {
|
|
10436
|
+
const again = await collectAgain(
|
|
10437
|
+
c,
|
|
10438
|
+
{ spared: swept.knownSpared ?? [], ledgered },
|
|
10439
|
+
{ byName: past ? fenced : [] }
|
|
10262
10440
|
);
|
|
10263
|
-
|
|
10264
|
-
|
|
10265
|
-
|
|
10266
|
-
|
|
10267
|
-
|
|
10268
|
-
}
|
|
10269
|
-
|
|
10270
|
-
const
|
|
10441
|
+
collected.push(...again.deleted);
|
|
10442
|
+
missing = again.missing;
|
|
10443
|
+
if (again.moved !== null && again.moved !== "absent") {
|
|
10444
|
+
moved = again.moved;
|
|
10445
|
+
}
|
|
10446
|
+
}
|
|
10447
|
+
if (byName.length > 0 && missing === void 0) {
|
|
10448
|
+
const deleted = await deleteFenced(c, byName, moved);
|
|
10449
|
+
collected.push(...deleted.deleted);
|
|
10450
|
+
moved = deleted.moved;
|
|
10451
|
+
missing = deleted.missing;
|
|
10452
|
+
}
|
|
10453
|
+
const left = await holderLeft(c, clean);
|
|
10454
|
+
if (missing !== void 0) notePointerOnDeleted(c, missing);
|
|
10455
|
+
if (left === void 0) {
|
|
10456
|
+
if (newest !== void 0) return collectedAll(c, newest, collected);
|
|
10457
|
+
return sealedOnly(c, notMember, collected);
|
|
10458
|
+
}
|
|
10459
|
+
if (moved !== null || missing !== void 0) {
|
|
10460
|
+
return {
|
|
10271
10461
|
...base,
|
|
10272
10462
|
erased: false,
|
|
10273
|
-
reason: "
|
|
10274
|
-
fromGeneration: from,
|
|
10275
|
-
collected
|
|
10463
|
+
reason: moved ?? "superseded",
|
|
10464
|
+
fromGeneration: newest ?? from,
|
|
10465
|
+
collected
|
|
10276
10466
|
};
|
|
10277
|
-
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
|
|
10282
|
-
|
|
10283
|
-
|
|
10284
|
-
|
|
10285
|
-
|
|
10286
|
-
|
|
10287
|
-
|
|
10288
|
-
|
|
10289
|
-
|
|
10290
|
-
|
|
10291
|
-
|
|
10292
|
-
|
|
10293
|
-
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
}
|
|
10300
|
-
|
|
10301
|
-
const
|
|
10302
|
-
|
|
10303
|
-
|
|
10304
|
-
|
|
10305
|
-
|
|
10306
|
-
|
|
10307
|
-
|
|
10308
|
-
|
|
10309
|
-
|
|
10310
|
-
|
|
10311
|
-
|
|
10312
|
-
|
|
10313
|
-
|
|
10314
|
-
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10318
|
-
|
|
10319
|
-
|
|
10320
|
-
let missing2 = swept2.missing;
|
|
10321
|
-
if (sweep && missing2 === void 0 && (moved === null || past)) {
|
|
10322
|
-
const again = await collectAgain(
|
|
10323
|
-
{ spared: swept2.knownSpared ?? [], ledgered },
|
|
10324
|
-
{ byName: past ? fenced : [] }
|
|
10325
|
-
);
|
|
10326
|
-
collected2.push(...again.deleted);
|
|
10327
|
-
missing2 = again.missing;
|
|
10328
|
-
if (again.moved !== null && again.moved !== "absent" && again.moved !== "no-generation") {
|
|
10329
|
-
moved = again.moved;
|
|
10467
|
+
}
|
|
10468
|
+
throw cannotRemove(c, left);
|
|
10469
|
+
}
|
|
10470
|
+
async function stage(c) {
|
|
10471
|
+
const {
|
|
10472
|
+
ref,
|
|
10473
|
+
deps,
|
|
10474
|
+
chunkKey,
|
|
10475
|
+
remainder,
|
|
10476
|
+
codec,
|
|
10477
|
+
maxBytes,
|
|
10478
|
+
base,
|
|
10479
|
+
record,
|
|
10480
|
+
from,
|
|
10481
|
+
aead,
|
|
10482
|
+
fromKey,
|
|
10483
|
+
read
|
|
10484
|
+
} = c;
|
|
10485
|
+
let written;
|
|
10486
|
+
const described = usableSummary(ref, record, aead);
|
|
10487
|
+
try {
|
|
10488
|
+
const reader = await read(
|
|
10489
|
+
() => openGenerationReader(deps.storage, fromKey, cryptoAt(c, from), {}, described?.fingerprint)
|
|
10490
|
+
);
|
|
10491
|
+
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
10492
|
+
if (bytes === null) return await notInCurrent(c);
|
|
10493
|
+
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
10494
|
+
assertRemaindersInRange(target, chunkKey);
|
|
10495
|
+
if (!target.has(remainder)) return await notInCurrent(c);
|
|
10496
|
+
target.remove(remainder);
|
|
10497
|
+
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
10498
|
+
const generation = await nextGeneration(ref, deps);
|
|
10499
|
+
const key = { ...base, generation };
|
|
10500
|
+
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
10501
|
+
const tally = await writeCrbmGenerationStream(
|
|
10502
|
+
deps.storage,
|
|
10503
|
+
key,
|
|
10504
|
+
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
10505
|
+
{
|
|
10506
|
+
crypto: cryptoAt(c, generation),
|
|
10507
|
+
clock: deps.clock,
|
|
10508
|
+
metadata,
|
|
10509
|
+
entropy: deps.entropy
|
|
10330
10510
|
}
|
|
10331
|
-
|
|
10332
|
-
|
|
10333
|
-
|
|
10334
|
-
|
|
10335
|
-
|
|
10336
|
-
|
|
10337
|
-
|
|
10338
|
-
|
|
10339
|
-
|
|
10340
|
-
|
|
10341
|
-
|
|
10342
|
-
|
|
10343
|
-
|
|
10344
|
-
|
|
10345
|
-
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
|
|
10353
|
-
|
|
10354
|
-
|
|
10355
|
-
let rewrote;
|
|
10356
|
-
const stage = async () => {
|
|
10357
|
-
let written;
|
|
10358
|
-
const described = usableSummary(ref, record, aead);
|
|
10511
|
+
);
|
|
10512
|
+
written = generation;
|
|
10513
|
+
c.rewrote = { generation, fingerprint: tally.fingerprint };
|
|
10514
|
+
const beforeVerify = await deps.registry.get(ref);
|
|
10515
|
+
const early = rowVerdict(c, beforeVerify, ["knownGens"]);
|
|
10516
|
+
if (early !== null) return refusedResult(c, early, written);
|
|
10517
|
+
await read(
|
|
10518
|
+
() => verifyGeneration(deps.storage, key, { ...tally, metadata }, cryptoAt(c, generation))
|
|
10519
|
+
);
|
|
10520
|
+
const summary = ownSummary(
|
|
10521
|
+
ref,
|
|
10522
|
+
generation,
|
|
10523
|
+
{
|
|
10524
|
+
cardinality: tally.cardinality,
|
|
10525
|
+
metadata,
|
|
10526
|
+
fingerprint: tally.fingerprint,
|
|
10527
|
+
tailBytes: tally.tailBytes
|
|
10528
|
+
},
|
|
10529
|
+
aead
|
|
10530
|
+
);
|
|
10531
|
+
return { generation, key, fingerprint: tally.fingerprint, summary };
|
|
10532
|
+
} catch (err) {
|
|
10533
|
+
if (!isNotFoundError(err)) throw err;
|
|
10534
|
+
let now;
|
|
10359
10535
|
try {
|
|
10360
|
-
|
|
10361
|
-
|
|
10362
|
-
);
|
|
10363
|
-
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
10364
|
-
if (bytes === null) return await notInCurrent();
|
|
10365
|
-
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
10366
|
-
assertRemaindersInRange(target, chunkKey);
|
|
10367
|
-
if (!target.has(remainder)) return await notInCurrent();
|
|
10368
|
-
target.remove(remainder);
|
|
10369
|
-
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
10370
|
-
const generation2 = await nextGeneration(ref, deps);
|
|
10371
|
-
const key2 = { ...base, generation: generation2 };
|
|
10372
|
-
const metadata = metadataToCarry(reader.metadata, aead === void 0 ? void 0 : described);
|
|
10373
|
-
const tally = await writeCrbmGenerationStream(
|
|
10374
|
-
deps.storage,
|
|
10375
|
-
key2,
|
|
10376
|
-
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
10377
|
-
{ crypto: cryptoAt(generation2), clock: deps.clock, metadata }
|
|
10378
|
-
);
|
|
10379
|
-
written = generation2;
|
|
10380
|
-
rewrote = { generation: generation2, fingerprint: tally.fingerprint };
|
|
10381
|
-
const beforeVerify = await deps.registry.get(ref);
|
|
10382
|
-
const early = rowVerdict(beforeVerify, ["knownGens"]);
|
|
10383
|
-
if (early !== null) return refused(early, written);
|
|
10384
|
-
await read(
|
|
10385
|
-
() => verifyGeneration(deps.storage, key2, { ...tally, metadata }, cryptoAt(generation2))
|
|
10386
|
-
);
|
|
10387
|
-
const summary2 = ownSummary(
|
|
10388
|
-
ref,
|
|
10389
|
-
generation2,
|
|
10390
|
-
{
|
|
10391
|
-
cardinality: tally.cardinality,
|
|
10392
|
-
metadata,
|
|
10393
|
-
fingerprint: tally.fingerprint,
|
|
10394
|
-
tailBytes: tally.tailBytes
|
|
10395
|
-
},
|
|
10396
|
-
aead
|
|
10397
|
-
);
|
|
10398
|
-
return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
|
|
10399
|
-
} catch (err) {
|
|
10400
|
-
if (!isNotFoundError(err)) throw err;
|
|
10401
|
-
let now;
|
|
10402
|
-
try {
|
|
10403
|
-
now = await deps.registry.get(ref);
|
|
10404
|
-
} catch {
|
|
10405
|
-
throw err;
|
|
10406
|
-
}
|
|
10407
|
-
const verdict = rowVerdict(now);
|
|
10408
|
-
if (verdict !== null) return refused(verdict, written);
|
|
10536
|
+
now = await deps.registry.get(ref);
|
|
10537
|
+
} catch {
|
|
10409
10538
|
throw err;
|
|
10410
10539
|
}
|
|
10411
|
-
|
|
10412
|
-
|
|
10413
|
-
|
|
10414
|
-
|
|
10415
|
-
|
|
10416
|
-
|
|
10417
|
-
|
|
10418
|
-
|
|
10419
|
-
|
|
10420
|
-
|
|
10421
|
-
|
|
10422
|
-
|
|
10423
|
-
|
|
10424
|
-
|
|
10425
|
-
|
|
10426
|
-
|
|
10427
|
-
|
|
10428
|
-
|
|
10429
|
-
|
|
10430
|
-
|
|
10431
|
-
|
|
10432
|
-
|
|
10433
|
-
|
|
10540
|
+
const verdict = rowVerdict(c, now);
|
|
10541
|
+
if (verdict !== null) return refusedResult(c, verdict, written);
|
|
10542
|
+
throw err;
|
|
10543
|
+
}
|
|
10544
|
+
}
|
|
10545
|
+
async function discardRefused(c, own) {
|
|
10546
|
+
const { ref, deps, base, from } = c;
|
|
10547
|
+
const key = { ...base, generation: own.generation };
|
|
10548
|
+
const renewal = rowRenewal(ref, deps);
|
|
10549
|
+
const churn = leaseChurn({ clock: deps.clock, rng: renewal.rng });
|
|
10550
|
+
let row = await deps.registry.get(ref);
|
|
10551
|
+
for (let attempts = 1; ; attempts++) {
|
|
10552
|
+
if (row !== null && row.status === "destroyed") {
|
|
10553
|
+
await deps.storage.delete(key);
|
|
10554
|
+
return row;
|
|
10555
|
+
}
|
|
10556
|
+
if (row === null || row.status !== "active" || row.currentGen === null) return row;
|
|
10557
|
+
if (row.currentGen <= from || row.currentGen >= own.generation) return row;
|
|
10558
|
+
const proven = await provenObject(deps.storage, key, own.fingerprint);
|
|
10559
|
+
if (proven === null) return row;
|
|
10560
|
+
const renewed = await renewal.renew(row);
|
|
10561
|
+
if ("row" in renewed) {
|
|
10562
|
+
try {
|
|
10563
|
+
await deps.storage.delete(key, { ifVersion: proven.version });
|
|
10564
|
+
} catch (err) {
|
|
10565
|
+
if (!isWriteConflictError(err)) throw err;
|
|
10434
10566
|
}
|
|
10435
|
-
|
|
10436
|
-
"pointerId"
|
|
10437
|
-
]);
|
|
10438
|
-
if (settled2 === void 0 && attempts >= DISCARD_ATTEMPTS) return renewed.lost;
|
|
10439
|
-
if (settled2 !== void 0) attempts -= 1;
|
|
10440
|
-
row = settled2 === void 0 ? renewed.lost : settled2.row;
|
|
10441
|
-
}
|
|
10442
|
-
};
|
|
10443
|
-
if (tombstoned) return await underTombstone();
|
|
10444
|
-
if (pointerless) return await unpublished();
|
|
10445
|
-
const staged = await stage();
|
|
10446
|
-
if ("erased" in staged) {
|
|
10447
|
-
if (!staged.erased && staged.generation !== void 0 && rewrote !== void 0) {
|
|
10448
|
-
await discardRefused(rewrote);
|
|
10567
|
+
return row;
|
|
10449
10568
|
}
|
|
10450
|
-
|
|
10569
|
+
const settled2 = await churn.settle(row, renewed.lost, () => deps.registry.get(ref), [
|
|
10570
|
+
"pointerId"
|
|
10571
|
+
]);
|
|
10572
|
+
if (settled2 === void 0 && attempts >= DISCARD_ATTEMPTS) return renewed.lost;
|
|
10573
|
+
if (settled2 !== void 0) attempts -= 1;
|
|
10574
|
+
row = settled2 === void 0 ? renewed.lost : settled2.row;
|
|
10451
10575
|
}
|
|
10576
|
+
}
|
|
10577
|
+
async function finishRewrite(c, staged) {
|
|
10578
|
+
const { ref, deps, audit, base, record, from, fromToken } = c;
|
|
10452
10579
|
const { generation, key, fingerprint, summary } = staged;
|
|
10453
10580
|
const published = await publishGeneration(deps.registry, key, {
|
|
10454
10581
|
expectFrom: from,
|
|
@@ -10465,18 +10592,18 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
10465
10592
|
clearLeases: true,
|
|
10466
10593
|
// A write that ends without an answer is settled by reading the row: a registry that reports its writes' tokens
|
|
10467
10594
|
// settles it by the row's `pointerId`; any other takes a pointer at this number for this rewrite's only over an
|
|
10468
|
-
// object whose footer matches what it wrote,
|
|
10469
|
-
//
|
|
10595
|
+
// object whose footer matches what it wrote, which carries this rewrite's own write identity, so another write's
|
|
10596
|
+
// object cannot pass for it.
|
|
10470
10597
|
holdsOwnObject: () => provesOwnObject(deps.storage, key, fingerprint),
|
|
10471
10598
|
clock: deps.clock,
|
|
10472
10599
|
rng: deps.rng ?? deps.readRetry?.rng
|
|
10473
10600
|
});
|
|
10474
10601
|
if (!published) {
|
|
10475
|
-
const now = await discardRefused({ generation, fingerprint });
|
|
10602
|
+
const now = await discardRefused(c, { generation, fingerprint });
|
|
10476
10603
|
return {
|
|
10477
10604
|
...base,
|
|
10478
10605
|
erased: false,
|
|
10479
|
-
reason: rowVerdict(now) ?? "superseded",
|
|
10606
|
+
reason: rowVerdict(c, now) ?? "superseded",
|
|
10480
10607
|
fromGeneration: from,
|
|
10481
10608
|
generation,
|
|
10482
10609
|
collected: []
|
|
@@ -10498,6 +10625,7 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
10498
10625
|
let missing;
|
|
10499
10626
|
if (missing === void 0 && (swept.knownSpared?.length ?? 0) > 0) {
|
|
10500
10627
|
const again = await collectAgain(
|
|
10628
|
+
c,
|
|
10501
10629
|
{ spared: swept.knownSpared ?? [], ledgered: true },
|
|
10502
10630
|
{ byName: [], below: generation }
|
|
10503
10631
|
);
|
|
@@ -10507,10 +10635,25 @@ async function eraseOnce(ref, id, deps, options, seen) {
|
|
|
10507
10635
|
if (missing !== void 0) {
|
|
10508
10636
|
throw pointerOnDeletedError(ref, { kind: "erasure" }, missing);
|
|
10509
10637
|
}
|
|
10510
|
-
const left = await holderLeft(/* @__PURE__ */ new Set([generation]));
|
|
10511
|
-
if (left !== void 0) throw cannotRemove(left);
|
|
10638
|
+
const left = await holderLeft(c, /* @__PURE__ */ new Set([generation]));
|
|
10639
|
+
if (left !== void 0) throw cannotRemove(c, left);
|
|
10512
10640
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
10513
10641
|
}
|
|
10642
|
+
async function resolveDek(ref, deps, record, row) {
|
|
10643
|
+
if (row.tombstoned) return void 0;
|
|
10644
|
+
if (record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0) {
|
|
10645
|
+
if (deps.keystore === void 0) {
|
|
10646
|
+
throw new KeyUnavailableError(
|
|
10647
|
+
`segment "${ref.segment}" is encrypted but eraseIdFromSegment has no keystore`
|
|
10648
|
+
);
|
|
10649
|
+
}
|
|
10650
|
+
return await deps.keystore.openDek(record.wrappedDeks);
|
|
10651
|
+
}
|
|
10652
|
+
if (!row.pointerless && deps.requireEncryption === true) {
|
|
10653
|
+
throw new ValidationError(`requireEncryption: segment "${ref.segment}" is cleartext`);
|
|
10654
|
+
}
|
|
10655
|
+
return void 0;
|
|
10656
|
+
}
|
|
10514
10657
|
function noticingIndex(crypto, index) {
|
|
10515
10658
|
const indexAad = crypto.aadFor("index");
|
|
10516
10659
|
return {
|
|
@@ -10828,7 +10971,7 @@ async function setSegmentRetention(ref, deps, policy) {
|
|
|
10828
10971
|
{
|
|
10829
10972
|
attempts: RETENTION_CAS_ATTEMPTS,
|
|
10830
10973
|
contention: () => new WriteConflictError(
|
|
10831
|
-
`setRetention: contention writing the retention policy for segment "${ref.segment}"`
|
|
10974
|
+
`setRetention: contention writing the retention policy for segment "${ref.segment}"${CONTENTION_NEXT_STEP}`
|
|
10832
10975
|
)
|
|
10833
10976
|
},
|
|
10834
10977
|
async (record) => {
|
|
@@ -10866,7 +11009,7 @@ async function clearSegmentRetention(ref, deps) {
|
|
|
10866
11009
|
{
|
|
10867
11010
|
attempts: RETENTION_CAS_ATTEMPTS,
|
|
10868
11011
|
contention: () => new WriteConflictError(
|
|
10869
|
-
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"`
|
|
11012
|
+
`clearRetention: contention clearing the retention policy for segment "${ref.segment}"${CONTENTION_NEXT_STEP}`
|
|
10870
11013
|
)
|
|
10871
11014
|
},
|
|
10872
11015
|
async (record) => {
|
|
@@ -10929,7 +11072,8 @@ async function eraseNamespace(namespace, deps, options) {
|
|
|
10929
11072
|
}
|
|
10930
11073
|
const rows = await drainRegistry(deps.registry, {
|
|
10931
11074
|
namespace,
|
|
10932
|
-
|
|
11075
|
+
// Only an absent value means the default: a `null` or a malformed one is refused, as a malformed `budget` is.
|
|
11076
|
+
maxScanSegments: options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments,
|
|
10933
11077
|
op: "eraseNamespace",
|
|
10934
11078
|
narrowable: false
|
|
10935
11079
|
// already one namespace: the only remedy is a higher ceiling
|
|
@@ -11140,7 +11284,7 @@ async function rowsFromDueIndex(registry, options) {
|
|
|
11140
11284
|
if (ref === null) continue;
|
|
11141
11285
|
if (isReservedNamespace(ref.namespace)) continue;
|
|
11142
11286
|
if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
|
|
11143
|
-
const
|
|
11287
|
+
const found2 = {
|
|
11144
11288
|
ref: { namespace: pointer.namespace, segment: pointer.segment },
|
|
11145
11289
|
target: ref,
|
|
11146
11290
|
token: pointer.token
|
|
@@ -11148,26 +11292,29 @@ async function rowsFromDueIndex(registry, options) {
|
|
|
11148
11292
|
const key = segmentKey(ref);
|
|
11149
11293
|
const known = pointers.get(key);
|
|
11150
11294
|
if (known !== void 0) {
|
|
11151
|
-
known.push(
|
|
11295
|
+
known.push(found2);
|
|
11152
11296
|
continue;
|
|
11153
11297
|
}
|
|
11154
11298
|
if (gone.has(key)) {
|
|
11155
|
-
if (litter.length < options.maxScanSegments) litter.push(
|
|
11299
|
+
if (litter.length < options.maxScanSegments) litter.push(found2);
|
|
11156
11300
|
continue;
|
|
11157
11301
|
}
|
|
11158
11302
|
if (rows.length >= options.maxScanSegments) {
|
|
11159
11303
|
throw new BudgetExceededError(
|
|
11160
|
-
`retireExpired: the due index yielded more than ${options.maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Raise \`maxScanSegments\`, narrow with \`namespace\`, or reduce \`lookbackBuckets
|
|
11304
|
+
`retireExpired: the due index yielded more than ${options.maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Raise \`maxScanSegments\`, narrow with \`namespace\`, or reduce \`lookbackBuckets\`.`,
|
|
11305
|
+
// The scan refuses at the first pointer past the bound, before reading it, so `spent` is that pointer
|
|
11306
|
+
// counted with the rows held. A stale pointer counts too, so it can pass the live rows by one.
|
|
11307
|
+
{ option: "maxScanSegments", limit: options.maxScanSegments, spent: rows.length + 1 }
|
|
11161
11308
|
);
|
|
11162
11309
|
}
|
|
11163
11310
|
const live = await registry.get(ref);
|
|
11164
11311
|
if (live === null) {
|
|
11165
11312
|
if (gone.size < options.maxScanSegments) gone.add(key);
|
|
11166
|
-
if (litter.length < options.maxScanSegments) litter.push(
|
|
11313
|
+
if (litter.length < options.maxScanSegments) litter.push(found2);
|
|
11167
11314
|
continue;
|
|
11168
11315
|
}
|
|
11169
11316
|
rows.push(live);
|
|
11170
|
-
pointers.set(key, [
|
|
11317
|
+
pointers.set(key, [found2]);
|
|
11171
11318
|
}
|
|
11172
11319
|
}
|
|
11173
11320
|
return { rows, pointers, litter };
|
|
@@ -11193,11 +11340,11 @@ async function rowsFromFleet(registry, options) {
|
|
|
11193
11340
|
const litter = [];
|
|
11194
11341
|
if (held.length === 0) return { rows, pointers, litter };
|
|
11195
11342
|
const present = new Set(rows.map((rec) => segmentKey(rec)));
|
|
11196
|
-
for (const
|
|
11197
|
-
const key = segmentKey(
|
|
11198
|
-
if (!present.has(key)) litter.push(
|
|
11199
|
-
else if (pointers.has(key)) pointers.get(key).push(
|
|
11200
|
-
else pointers.set(key, [
|
|
11343
|
+
for (const found2 of held) {
|
|
11344
|
+
const key = segmentKey(found2.target);
|
|
11345
|
+
if (!present.has(key)) litter.push(found2);
|
|
11346
|
+
else if (pointers.has(key)) pointers.get(key).push(found2);
|
|
11347
|
+
else pointers.set(key, [found2]);
|
|
11201
11348
|
}
|
|
11202
11349
|
return { rows, pointers, litter };
|
|
11203
11350
|
}
|
|
@@ -11216,8 +11363,8 @@ async function filePurgePointer(registry, ref, purgeAt) {
|
|
|
11216
11363
|
} catch {
|
|
11217
11364
|
}
|
|
11218
11365
|
}
|
|
11219
|
-
async function forgetFoundPointers(registry,
|
|
11220
|
-
for (const pointer of
|
|
11366
|
+
async function forgetFoundPointers(registry, found2, onFault) {
|
|
11367
|
+
for (const pointer of found2) {
|
|
11221
11368
|
try {
|
|
11222
11369
|
await registry.delete(pointer.ref, pointer.token);
|
|
11223
11370
|
} catch (err) {
|
|
@@ -11225,10 +11372,10 @@ async function forgetFoundPointers(registry, found, onFault) {
|
|
|
11225
11372
|
}
|
|
11226
11373
|
}
|
|
11227
11374
|
}
|
|
11228
|
-
async function forgetPurgePointers(registry, ref, purgeAt,
|
|
11375
|
+
async function forgetPurgePointers(registry, ref, purgeAt, found2, onFault) {
|
|
11229
11376
|
if (!canIndex(ref)) return;
|
|
11230
11377
|
const filed = dueIndexRef(dueBucket(purgeAt), ref);
|
|
11231
|
-
const all =
|
|
11378
|
+
const all = found2.some((p) => p.ref.namespace === filed.namespace) ? found2 : [...found2, { ref: filed, target: ref }];
|
|
11232
11379
|
for (const pointer of all) {
|
|
11233
11380
|
try {
|
|
11234
11381
|
await registry.delete(pointer.ref, pointer.token);
|
|
@@ -11264,7 +11411,7 @@ function resolveRetireOptions(options) {
|
|
|
11264
11411
|
const purgeTombstones = options.purgeTombstones ?? true;
|
|
11265
11412
|
const grace = options.tombstoneGraceMs ?? DEFAULT_TOMBSTONE_GRACE_MS;
|
|
11266
11413
|
requireFiniteNumber(grace, { min: 0, what: "retireExpired: `tombstoneGraceMs`" });
|
|
11267
|
-
const maxScanSegments = options.maxScanSegments
|
|
11414
|
+
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11268
11415
|
requirePositiveInteger(maxScanSegments, "retireExpired: `maxScanSegments`");
|
|
11269
11416
|
return {
|
|
11270
11417
|
audit: options.audit,
|
|
@@ -11327,13 +11474,13 @@ async function purgeTombstone(rec, ctx) {
|
|
|
11327
11474
|
const base = { segment: rec.segment, namespace: rec.namespace };
|
|
11328
11475
|
const stamp = retirementStamp(rec.retention);
|
|
11329
11476
|
if (stamp === null) {
|
|
11330
|
-
const
|
|
11331
|
-
if (!dryRun && removesRows &&
|
|
11477
|
+
const found2 = pointers.get(segmentKey(ref)) ?? [];
|
|
11478
|
+
if (!dryRun && removesRows && found2.length > 0 && tally.attemptedForeign < limit) {
|
|
11332
11479
|
tally.attemptedForeign += 1;
|
|
11333
11480
|
try {
|
|
11334
11481
|
const live = await deps.registry.get(ref);
|
|
11335
11482
|
if (live?.status === "destroyed" && retirementStamp(live.retention) === null) {
|
|
11336
|
-
await forgetFoundPointers(deps.registry,
|
|
11483
|
+
await forgetFoundPointers(deps.registry, found2, tally.noteFault);
|
|
11337
11484
|
}
|
|
11338
11485
|
} catch (err) {
|
|
11339
11486
|
tally.noteFault(err);
|
|
@@ -11555,7 +11702,7 @@ var errorEntry = (rec, error) => ({
|
|
|
11555
11702
|
namespace: rec.namespace,
|
|
11556
11703
|
error: error instanceof Error ? error.message : String(error)
|
|
11557
11704
|
});
|
|
11558
|
-
async function checkSummary(ref, live, deps,
|
|
11705
|
+
async function checkSummary(ref, live, deps, found2) {
|
|
11559
11706
|
const generation = live.currentGen;
|
|
11560
11707
|
const keyed = live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0;
|
|
11561
11708
|
let aead;
|
|
@@ -11589,7 +11736,7 @@ async function checkSummary(ref, live, deps, found) {
|
|
|
11589
11736
|
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
11590
11737
|
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) },
|
|
11591
11738
|
described,
|
|
11592
|
-
|
|
11739
|
+
found2
|
|
11593
11740
|
);
|
|
11594
11741
|
if (reader !== void 0 && summaryAgrees(described, {
|
|
11595
11742
|
cardinality: reader.count(),
|
|
@@ -11671,11 +11818,11 @@ async function checkCurrentObject(ref, live, deps, summaries) {
|
|
|
11671
11818
|
};
|
|
11672
11819
|
const checksSummary = summaries && live.summary !== void 0;
|
|
11673
11820
|
if (checksSummary && summaryOpensObject(ref, live)) {
|
|
11674
|
-
let
|
|
11821
|
+
let found2 = false;
|
|
11675
11822
|
try {
|
|
11676
|
-
return await checkSummary(ref, live, deps, () =>
|
|
11823
|
+
return await checkSummary(ref, live, deps, () => found2 = true);
|
|
11677
11824
|
} catch (err) {
|
|
11678
|
-
if (
|
|
11825
|
+
if (found2) throw err;
|
|
11679
11826
|
if (!await listed(deps.storage, key)) return missing;
|
|
11680
11827
|
throw err;
|
|
11681
11828
|
}
|
|
@@ -11693,14 +11840,11 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
11693
11840
|
}
|
|
11694
11841
|
const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
|
|
11695
11842
|
requirePositiveInteger(concurrency, "concurrency");
|
|
11696
|
-
const maxScanSegments = options.maxScanSegments
|
|
11843
|
+
const maxScanSegments = options.maxScanSegments === void 0 ? DEFAULT_MAX_SCAN_SEGMENTS : options.maxScanSegments;
|
|
11697
11844
|
const recs = await drainRegistry(deps.registry, {
|
|
11698
11845
|
namespace: options.namespace,
|
|
11699
11846
|
maxScanSegments,
|
|
11700
|
-
op: "checkConsistency"
|
|
11701
|
-
// Two callers reach this refusal, and only one of them can raise the ceiling: `store.checkConsistency` takes
|
|
11702
|
-
// no `maxScanSegments`, so the advice names the call that does.
|
|
11703
|
-
raise: "raise `maxScanSegments` on `runConsistencyCheck` (`store.checkConsistency` takes no ceiling, so run `runConsistencyCheck({ storage: backend.storage, registry: backend.registry }, { maxScanSegments })` over the same backend instead)"
|
|
11847
|
+
op: "checkConsistency"
|
|
11704
11848
|
});
|
|
11705
11849
|
const summaries = options.summaries === true;
|
|
11706
11850
|
const generations = options.generations === true;
|
|
@@ -11886,22 +12030,10 @@ export {
|
|
|
11886
12030
|
dropSegment,
|
|
11887
12031
|
eraseIdFromSegment,
|
|
11888
12032
|
eraseNamespace,
|
|
12033
|
+
errorKind,
|
|
11889
12034
|
excludingReservedRows,
|
|
11890
12035
|
getSegmentRetention,
|
|
11891
|
-
isBudgetExceededError,
|
|
11892
|
-
isCapabilityError,
|
|
11893
|
-
isCloudRoaringError,
|
|
11894
|
-
isIntegrityError,
|
|
11895
|
-
isKeyUnavailableError,
|
|
11896
|
-
isLeaseExpiredError,
|
|
11897
|
-
isLeaseLimitError,
|
|
11898
|
-
isNotFoundError,
|
|
11899
|
-
isStaleOperandError,
|
|
11900
12036
|
isStorageBackend,
|
|
11901
|
-
isTransientError,
|
|
11902
|
-
isUnsupportedError,
|
|
11903
|
-
isValidationError,
|
|
11904
|
-
isWriteConflictError,
|
|
11905
12037
|
judgeLoad,
|
|
11906
12038
|
listGenerations,
|
|
11907
12039
|
listSegments,
|