@cloudbitmaps/core 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-5OMC5U7U.js → chunk-5W3XS2IU.js} +292 -22
- package/dist/chunk-5W3XS2IU.js.map +7 -0
- package/dist/core/audit.d.ts +6 -0
- package/dist/core/codec.d.ts +33 -1
- package/dist/core/consistency.d.ts +21 -2
- package/dist/core/cost.d.ts +91 -10
- package/dist/core/crbm/format.d.ts +31 -1
- package/dist/core/crbm/reader.d.ts +44 -2
- package/dist/core/crbm/writer.d.ts +32 -3
- package/dist/core/crbm-storage-source.d.ts +203 -22
- package/dist/core/crypto.d.ts +11 -7
- package/dist/core/determinism.d.ts +7 -0
- package/dist/core/engine.d.ts +17 -5
- package/dist/core/erase-id.d.ts +20 -0
- package/dist/core/erasure.d.ts +8 -4
- package/dist/core/generation-gc.d.ts +102 -4
- package/dist/core/load-input.d.ts +55 -0
- package/dist/core/load.d.ts +65 -10
- package/dist/core/metadata.d.ts +42 -0
- package/dist/core/pinned-storage-source.d.ts +2 -1
- package/dist/core/ports.d.ts +119 -13
- package/dist/core/registry-scan.d.ts +6 -0
- package/dist/core/retention-sweep.d.ts +47 -2
- package/dist/core/retry.d.ts +14 -0
- package/dist/core/rollback.d.ts +34 -4
- package/dist/core/summary.d.ts +61 -0
- package/dist/core/token.d.ts +36 -0
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.js +39 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/entropy.d.ts +5 -0
- package/dist/drivers/_shared/object-registry.d.ts +36 -6
- package/dist/drivers/_shared/registry.d.ts +47 -19
- package/dist/drivers/localfs/registry.d.ts +10 -0
- package/dist/drivers/memory.d.ts +9 -3
- package/dist/drivers/retry/retrying-chunk-source.d.ts +2 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1794 -461
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +6 -5
- package/package.json +1 -1
- package/dist/chunk-5OMC5U7U.js.map +0 -7
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
DEFAULT_NAMESPACE,
|
|
6
6
|
IntegrityError,
|
|
7
7
|
KeyUnavailableError,
|
|
8
|
+
MAX_METADATA_BYTES,
|
|
8
9
|
MAX_NAME_LENGTH,
|
|
9
10
|
NotFoundError,
|
|
10
11
|
RESERVED_NAMESPACE_PREFIX,
|
|
@@ -13,11 +14,18 @@ import {
|
|
|
13
14
|
ValidationError,
|
|
14
15
|
WriteConflictError,
|
|
15
16
|
applyRegistryPatch,
|
|
17
|
+
assertRegistryCanWrite,
|
|
16
18
|
brandAsBackend,
|
|
19
|
+
canonicalMetadataJson,
|
|
20
|
+
copiedMetadata,
|
|
17
21
|
decodeNameFromPath,
|
|
22
|
+
drawIncarnation,
|
|
23
|
+
drawWrite,
|
|
18
24
|
encodeNameForKey,
|
|
19
25
|
encodeNameForPath,
|
|
20
26
|
encodedNameLength,
|
|
27
|
+
entropyIsAvailable,
|
|
28
|
+
incarnationOf,
|
|
21
29
|
isCloudRoaringError,
|
|
22
30
|
isIntegrityError,
|
|
23
31
|
isNotFoundError,
|
|
@@ -27,18 +35,23 @@ import {
|
|
|
27
35
|
isValidationError,
|
|
28
36
|
isWriteConflictError,
|
|
29
37
|
mapWithConcurrency,
|
|
38
|
+
metadataBytes,
|
|
39
|
+
metadataFromBytes,
|
|
30
40
|
namespacePathPart,
|
|
41
|
+
newIncarnationToken,
|
|
42
|
+
nextRegistryToken,
|
|
31
43
|
parseRegistryEnvelope,
|
|
32
44
|
recordFromNew,
|
|
33
|
-
|
|
45
|
+
sameIncarnation,
|
|
34
46
|
serializeRegistryEnvelope,
|
|
35
47
|
validateChunkRef,
|
|
36
48
|
validateNewRegistryRecord,
|
|
37
49
|
validateRegistryPatch,
|
|
38
50
|
validateSegmentRef,
|
|
39
51
|
validateUserNamespace,
|
|
40
|
-
validateUserRef
|
|
41
|
-
|
|
52
|
+
validateUserRef,
|
|
53
|
+
webCryptoEntropy
|
|
54
|
+
} from "./chunk-5W3XS2IU.js";
|
|
42
55
|
|
|
43
56
|
// src/core/bit-route.ts
|
|
44
57
|
var U32_MAX = 4294967295;
|
|
@@ -313,15 +326,19 @@ var SegmentEngine = class {
|
|
|
313
326
|
return chunk ? chunk.has(remainder) : false;
|
|
314
327
|
}
|
|
315
328
|
/**
|
|
316
|
-
* Cardinality. When the Storage source can
|
|
317
|
-
*
|
|
318
|
-
*
|
|
329
|
+
* Cardinality. When the Storage source can describe the current generation (the `.crbm` source can), the count is
|
|
330
|
+
* that description's: the registry row's summary when the row has one it can use, which is one request when cold
|
|
331
|
+
* and no read of the object, and otherwise the sum of the object's index. Either way, **zero payload reads**. A
|
|
332
|
+
* source that can only serve per-chunk cardinality sums those, and one with neither (the in-memory source) falls back
|
|
333
|
+
* to fetching every chunk.
|
|
319
334
|
*
|
|
320
|
-
* The index sum
|
|
335
|
+
* The row's summary and the index sum are trusted, not confirmed: the source holds the summary against the object
|
|
336
|
+
* whenever it opens it, and the reader checked the index for internal consistency when it opened
|
|
321
337
|
* the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
|
|
322
338
|
* combines decode the payloads.
|
|
323
339
|
*/
|
|
324
340
|
async count(seg) {
|
|
341
|
+
if (this.storage.summary) return (await this.storage.summary(seg))?.cardinality ?? 0;
|
|
325
342
|
const cardinalities = this.storage.cardinalities ? await this.storage.cardinalities(seg) : null;
|
|
326
343
|
if (cardinalities) {
|
|
327
344
|
let total2 = 0;
|
|
@@ -339,6 +356,17 @@ var SegmentEngine = class {
|
|
|
339
356
|
for (let i = 0; i < chunkKeys.length; i++) total += (await window.take())?.size ?? 0;
|
|
340
357
|
return total;
|
|
341
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* What the segment's current generation is: its number, its id count and its metadata, from one resolution, so the
|
|
361
|
+
* three cannot straddle a publish. A source with no summary answers from `count()` and the generation it resolves.
|
|
362
|
+
*/
|
|
363
|
+
async stat(seg) {
|
|
364
|
+
if (this.storage.summary) {
|
|
365
|
+
return await this.storage.summary(seg) ?? { generation: null, cardinality: 0 };
|
|
366
|
+
}
|
|
367
|
+
const generation = this.storage.currentGeneration ? await this.storage.currentGeneration(seg) : null;
|
|
368
|
+
return { generation, cardinality: await this.count(seg) };
|
|
369
|
+
}
|
|
342
370
|
/** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
343
371
|
get supportsStorageSize() {
|
|
344
372
|
return typeof this.storage.sizeOf === "function";
|
|
@@ -871,6 +899,111 @@ var BoundedLru = class {
|
|
|
871
899
|
}
|
|
872
900
|
};
|
|
873
901
|
|
|
902
|
+
// src/core/generation-gc.ts
|
|
903
|
+
async function nextGeneration(ref, deps) {
|
|
904
|
+
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
905
|
+
}
|
|
906
|
+
async function aboveEverything(ref, deps, record) {
|
|
907
|
+
let highest = record?.currentGen ?? -1;
|
|
908
|
+
for await (const key of deps.storage.list(ref)) {
|
|
909
|
+
if (key.generation > highest) highest = key.generation;
|
|
910
|
+
}
|
|
911
|
+
return highest + 1;
|
|
912
|
+
}
|
|
913
|
+
async function nextLoadGeneration(ref, deps, row) {
|
|
914
|
+
const generation = (row?.currentGen ?? -1) + 1;
|
|
915
|
+
try {
|
|
916
|
+
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
917
|
+
} catch (err) {
|
|
918
|
+
if (isNotFoundError(err)) return { generation, checked: true };
|
|
919
|
+
}
|
|
920
|
+
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
921
|
+
}
|
|
922
|
+
var LIST_COLLECTION_CADENCE = 16;
|
|
923
|
+
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
924
|
+
const keep = options.keep ?? 1;
|
|
925
|
+
if (!Number.isInteger(keep) || keep < 0) {
|
|
926
|
+
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
927
|
+
}
|
|
928
|
+
const record = await deps.registry.get(ref);
|
|
929
|
+
if (record === null) return [];
|
|
930
|
+
const current = record.currentGen;
|
|
931
|
+
const seen = /* @__PURE__ */ new Set();
|
|
932
|
+
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
933
|
+
const gens = [...seen];
|
|
934
|
+
const after = await deps.registry.get(ref);
|
|
935
|
+
if (after === null)
|
|
936
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
937
|
+
if (record.status === "destroyed" && after.token !== record.token) {
|
|
938
|
+
throw new WriteConflictError(
|
|
939
|
+
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
943
|
+
const stillCollectable = async () => {
|
|
944
|
+
const still = await deps.registry.get(ref);
|
|
945
|
+
if (still === null) {
|
|
946
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
947
|
+
}
|
|
948
|
+
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
949
|
+
if (!ok) {
|
|
950
|
+
throw new WriteConflictError(
|
|
951
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
};
|
|
955
|
+
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
956
|
+
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
957
|
+
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
958
|
+
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
959
|
+
[]
|
|
960
|
+
) : (
|
|
961
|
+
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
962
|
+
gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
|
|
963
|
+
);
|
|
964
|
+
for (const generation of toDelete) {
|
|
965
|
+
await stillCollectable();
|
|
966
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
967
|
+
}
|
|
968
|
+
return toDelete;
|
|
969
|
+
}
|
|
970
|
+
async function collectAfterLoad(ref, deps, options) {
|
|
971
|
+
const { generation, keep, byName, proveCurrent } = options;
|
|
972
|
+
if (keep >= generation) return [];
|
|
973
|
+
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
974
|
+
if (byName && keep <= 1 && !periodic) {
|
|
975
|
+
if (keep === 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
976
|
+
return gcOrphanGenerations(ref, deps, { keep });
|
|
977
|
+
}
|
|
978
|
+
return collectByName(ref, deps, { generation, keep });
|
|
979
|
+
}
|
|
980
|
+
return gcOrphanGenerations(ref, deps, { keep });
|
|
981
|
+
}
|
|
982
|
+
async function objectIsThere(storage, key) {
|
|
983
|
+
try {
|
|
984
|
+
await storage.getTail(key, 0);
|
|
985
|
+
return true;
|
|
986
|
+
} catch {
|
|
987
|
+
return false;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
async function collectByName(ref, deps, options) {
|
|
991
|
+
const { generation, keep } = options;
|
|
992
|
+
if (!Number.isInteger(keep) || keep < 0 || keep > 1) {
|
|
993
|
+
throw new ValidationError(`collecting by name needs a keep of 0 or 1; got ${String(keep)}`);
|
|
994
|
+
}
|
|
995
|
+
const evicted = generation - keep - 1;
|
|
996
|
+
if (evicted < 0) return [];
|
|
997
|
+
const still = await deps.registry.get(ref);
|
|
998
|
+
if (still === null || still.currentGen === null || still.currentGen < generation) return [];
|
|
999
|
+
await deps.storage.delete({
|
|
1000
|
+
namespace: ref.namespace,
|
|
1001
|
+
segment: ref.segment,
|
|
1002
|
+
generation: evicted
|
|
1003
|
+
});
|
|
1004
|
+
return [evicted];
|
|
1005
|
+
}
|
|
1006
|
+
|
|
874
1007
|
// src/core/reader-defaults.ts
|
|
875
1008
|
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
876
1009
|
var REFRESH_RETRY_MS = 500;
|
|
@@ -918,9 +1051,20 @@ var AWS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
|
918
1051
|
var HOURS_PER_MONTH = 730;
|
|
919
1052
|
var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
|
|
920
1053
|
var GIB = 1024 ** 3;
|
|
921
|
-
var
|
|
922
|
-
var
|
|
923
|
-
var
|
|
1054
|
+
var STORE_LOAD_PUT_CLASS = 1 + 1 / LIST_COLLECTION_CADENCE;
|
|
1055
|
+
var STORE_LOAD_POINTER_READS = 3 + 2 / LIST_COLLECTION_CADENCE;
|
|
1056
|
+
var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
|
|
1057
|
+
var RETENTION_SWEEP_REQUESTS = deepFreeze({
|
|
1058
|
+
conditionalDelete: {
|
|
1059
|
+
retirement: { reads: 9, writes: 3, deletes: 1 },
|
|
1060
|
+
purge: { reads: 4, writes: 0, deletes: 2 }
|
|
1061
|
+
},
|
|
1062
|
+
tombstoning: {
|
|
1063
|
+
retirement: { reads: 8, writes: 3, deletes: 0 },
|
|
1064
|
+
purge: { reads: 3, writes: 1, deletes: 0 }
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
var shown = (n) => String(Number(n.toFixed(2)));
|
|
924
1068
|
function requireFiniteNonNeg(n, field) {
|
|
925
1069
|
if (n === void 0 || !Number.isFinite(n) || n < 0) {
|
|
926
1070
|
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
@@ -1057,8 +1201,8 @@ function bytesWords(bytes) {
|
|
|
1057
1201
|
const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
|
|
1058
1202
|
let value = bytes;
|
|
1059
1203
|
let unit = 0;
|
|
1060
|
-
const
|
|
1061
|
-
while (unit < units.length - 1 &&
|
|
1204
|
+
const shown2 = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
|
|
1205
|
+
while (unit < units.length - 1 && shown2(value, unit) >= 1024) {
|
|
1062
1206
|
value /= 1024;
|
|
1063
1207
|
unit += 1;
|
|
1064
1208
|
}
|
|
@@ -1085,6 +1229,10 @@ function buildReport(input) {
|
|
|
1085
1229
|
requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
|
|
1086
1230
|
requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
|
|
1087
1231
|
requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
|
|
1232
|
+
const pointerRead = requireFiniteNonNeg(
|
|
1233
|
+
storage.requestsPerPointerRead ?? 1,
|
|
1234
|
+
"pricing.storage.requestsPerPointerRead"
|
|
1235
|
+
);
|
|
1088
1236
|
const sizedRead = requireFiniteNonNeg(
|
|
1089
1237
|
storage.requestsPerSizedRead ?? 1,
|
|
1090
1238
|
"pricing.storage.requestsPerSizedRead"
|
|
@@ -1131,6 +1279,15 @@ function buildReport(input) {
|
|
|
1131
1279
|
input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
|
|
1132
1280
|
"genTtlMs"
|
|
1133
1281
|
);
|
|
1282
|
+
const retirementsPerMonth = requireFiniteNonNeg(
|
|
1283
|
+
input.workload.retirementsPerMonth ?? 0,
|
|
1284
|
+
"retirementsPerMonth"
|
|
1285
|
+
);
|
|
1286
|
+
const purgesPerMonth = requireFiniteNonNeg(input.workload.purgesPerMonth ?? 0, "purgesPerMonth");
|
|
1287
|
+
const conditionalDelete = input.workload.conditionalDelete ?? true;
|
|
1288
|
+
if (typeof conditionalDelete !== "boolean") {
|
|
1289
|
+
throw new ValidationError(`conditionalDelete must be a boolean; got ${conditionalDelete}`);
|
|
1290
|
+
}
|
|
1134
1291
|
const storageGetUSD = storage.getPerMillion / 1e6;
|
|
1135
1292
|
const putUSD = storage.putPerMillion / 1e6;
|
|
1136
1293
|
const missFraction = 1 - cacheHitRate;
|
|
@@ -1139,16 +1296,26 @@ function buildReport(input) {
|
|
|
1139
1296
|
const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
|
|
1140
1297
|
const storageUSD = storedGiB * storage.storagePerGiBMonth;
|
|
1141
1298
|
const readsUSD = readMisses * storageGetUSD;
|
|
1142
|
-
const
|
|
1299
|
+
const getsPerColdOperand = pointerRead + sizedRead;
|
|
1300
|
+
const intersectGets = chunksPerIntersect + getsPerColdOperand * operandsPerIntersect;
|
|
1143
1301
|
const intersectsUSD = intersects * intersectGets * storageGetUSD;
|
|
1144
|
-
const loadGets =
|
|
1302
|
+
const loadGets = STORE_LOAD_POINTER_READS * pointerRead + STORE_LOAD_EXISTENCE_CHECKS;
|
|
1145
1303
|
const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
|
|
1146
|
-
const refreshUSD = refreshes *
|
|
1147
|
-
const
|
|
1304
|
+
const refreshUSD = refreshes * pointerRead * storageGetUSD;
|
|
1305
|
+
const sweep = RETENTION_SWEEP_REQUESTS[conditionalDelete ? "conditionalDelete" : "tombstoning"];
|
|
1306
|
+
const retentionUSD = retirementsPerMonth * (sweep.retirement.reads * storageGetUSD + sweep.retirement.writes * putUSD) + purgesPerMonth * (sweep.purge.reads * storageGetUSD + sweep.purge.writes * putUSD);
|
|
1307
|
+
const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD + retentionUSD;
|
|
1148
1308
|
const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
|
|
1149
1309
|
const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
|
|
1150
1310
|
const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
|
|
1151
|
-
const dominant = Math.max(
|
|
1311
|
+
const dominant = Math.max(
|
|
1312
|
+
readsUSD,
|
|
1313
|
+
intersectsUSD,
|
|
1314
|
+
storageUSD,
|
|
1315
|
+
loadsUSD,
|
|
1316
|
+
refreshUSD,
|
|
1317
|
+
retentionUSD
|
|
1318
|
+
);
|
|
1152
1319
|
let driver = "storage";
|
|
1153
1320
|
if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
|
|
1154
1321
|
else if (dominant === intersectsUSD && intersectsUSD > 0) {
|
|
@@ -1156,6 +1323,8 @@ function buildReport(input) {
|
|
|
1156
1323
|
} else if (dominant === loadsUSD && loadsUSD > 0)
|
|
1157
1324
|
driver = "loads (object, listing and pointer requests)";
|
|
1158
1325
|
else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
|
|
1326
|
+
else if (dominant === retentionUSD && retentionUSD > 0)
|
|
1327
|
+
driver = "the retention sweep (registry reads and writes)";
|
|
1159
1328
|
const sized = baseline.sized;
|
|
1160
1329
|
const noBytes = Math.round(storageBytes) === 0;
|
|
1161
1330
|
const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
|
|
@@ -1165,12 +1334,15 @@ function buildReport(input) {
|
|
|
1165
1334
|
const notes = [
|
|
1166
1335
|
"Same-region egress treated as free; internet egress is not modeled.",
|
|
1167
1336
|
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
1168
|
-
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${STORE_LOAD_PUT_CLASS} PUT-class and ${loadGets} GETs that store.load() adds (
|
|
1337
|
+
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${shown(STORE_LOAD_PUT_CLASS)} PUT-class and ${shown(loadGets)} GETs that store.load() adds (the pointer, a check that the next generation number is free, a check that the current generation is there, and a listing every ${LIST_COLLECTION_CADENCE}th load).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
|
|
1169
1338
|
...intersects > 0 ? [
|
|
1170
|
-
`Intersections priced cold: ${intersectGets} GETs each, ${
|
|
1339
|
+
`Intersections priced cold: ${intersectGets} GETs each, ${getsPerColdOperand} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk read(s); cacheHitRate does not apply.`
|
|
1171
1340
|
] : [],
|
|
1172
1341
|
hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
|
|
1173
1342
|
hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
|
|
1343
|
+
...retirementsPerMonth > 0 || purgesPerMonth > 0 ? [
|
|
1344
|
+
`Retention sweep modeled: ${retirementsPerMonth}/mo retirements at ${sweep.retirement.reads} reads, ${sweep.retirement.writes} writes and ${sweep.retirement.deletes} delete(s) each, ${purgesPerMonth}/mo purges at ${sweep.purge.reads} reads, ${sweep.purge.writes} writes and ${sweep.purge.deletes} delete(s) each; reads are priced at the GET rate and writes at the PUT rate, deletes at nothing, as S3 bills none` + (conditionalDelete ? "." : "; a registry that only tombstones is read again, two reads per purged segment, by every later full sweep, which is not priced.")
|
|
1345
|
+
] : [],
|
|
1174
1346
|
...input.extraNotes ?? [],
|
|
1175
1347
|
// Last, so the notes a report already carried keep their places.
|
|
1176
1348
|
redisNote
|
|
@@ -1182,7 +1354,8 @@ function buildReport(input) {
|
|
|
1182
1354
|
intersects: intersectsUSD,
|
|
1183
1355
|
storage: storageUSD,
|
|
1184
1356
|
loads: loadsUSD,
|
|
1185
|
-
pointerRefresh: refreshUSD
|
|
1357
|
+
pointerRefresh: refreshUSD,
|
|
1358
|
+
retention: retentionUSD
|
|
1186
1359
|
},
|
|
1187
1360
|
total
|
|
1188
1361
|
},
|
|
@@ -1291,7 +1464,10 @@ async function drainRegistry(registry, options) {
|
|
|
1291
1464
|
const advice = options.narrowable === false ? `${raise[0].toUpperCase()}${raise.slice(1)} if the namespace really is that large and the memory is ` : `Narrow it with \`namespace\`, or ${raise} if the fleet really is that large and the memory is `;
|
|
1292
1465
|
const rows = [];
|
|
1293
1466
|
for await (const rec of registry.list(options.namespace)) {
|
|
1294
|
-
if (options.namespace === void 0 && isReservedRow(rec))
|
|
1467
|
+
if (options.namespace === void 0 && isReservedRow(rec)) {
|
|
1468
|
+
options.onReserved?.(rec);
|
|
1469
|
+
continue;
|
|
1470
|
+
}
|
|
1295
1471
|
if (rows.length >= maxScanSegments) {
|
|
1296
1472
|
throw new BudgetExceededError(
|
|
1297
1473
|
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident).`
|
|
@@ -1463,16 +1639,19 @@ var BufferReader = class {
|
|
|
1463
1639
|
var MemoryRegistryDriver = class {
|
|
1464
1640
|
rows = /* @__PURE__ */ new Map();
|
|
1465
1641
|
now;
|
|
1642
|
+
entropy;
|
|
1466
1643
|
seq = 0;
|
|
1467
1644
|
constructor(options = {}) {
|
|
1468
1645
|
this.now = options.now ?? (() => Date.now());
|
|
1646
|
+
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
1469
1647
|
}
|
|
1470
|
-
nextToken() {
|
|
1648
|
+
nextToken(incarnation) {
|
|
1471
1649
|
this.seq += 1;
|
|
1472
|
-
return
|
|
1650
|
+
return `${incarnation}.${this.seq}.${drawWrite(this.entropy)}`;
|
|
1473
1651
|
}
|
|
1652
|
+
/** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
|
|
1474
1653
|
capabilities() {
|
|
1475
|
-
return { strongRead: true };
|
|
1654
|
+
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
1476
1655
|
}
|
|
1477
1656
|
async get(ref) {
|
|
1478
1657
|
validateSegmentRef(ref);
|
|
@@ -1481,25 +1660,25 @@ var MemoryRegistryDriver = class {
|
|
|
1481
1660
|
}
|
|
1482
1661
|
async create(ref, record) {
|
|
1483
1662
|
validateSegmentRef(ref);
|
|
1484
|
-
validateNewRegistryRecord(record);
|
|
1663
|
+
const checked = validateNewRegistryRecord(record);
|
|
1485
1664
|
const key = segmentKey(ref);
|
|
1486
1665
|
if (this.rows.has(key)) {
|
|
1487
1666
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
1488
1667
|
}
|
|
1489
|
-
const token = this.nextToken();
|
|
1490
|
-
this.rows.set(key, structuredClone(recordFromNew(ref,
|
|
1668
|
+
const token = this.nextToken(drawIncarnation(this.entropy));
|
|
1669
|
+
this.rows.set(key, structuredClone(recordFromNew(ref, checked, this.now(), token)));
|
|
1491
1670
|
return { token };
|
|
1492
1671
|
}
|
|
1493
1672
|
async compareAndSwap(ref, expected, patch) {
|
|
1494
1673
|
validateSegmentRef(ref);
|
|
1495
|
-
validateRegistryPatch(patch);
|
|
1674
|
+
const checked = validateRegistryPatch(patch);
|
|
1496
1675
|
const key = segmentKey(ref);
|
|
1497
1676
|
const existing = this.rows.get(key);
|
|
1498
1677
|
if (!existing || existing.token !== expected) {
|
|
1499
1678
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1500
1679
|
}
|
|
1501
|
-
const token = this.nextToken();
|
|
1502
|
-
this.rows.set(key, structuredClone(applyRegistryPatch(existing,
|
|
1680
|
+
const token = this.nextToken(incarnationOf(existing.token) ?? drawIncarnation(this.entropy));
|
|
1681
|
+
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
1503
1682
|
return { token };
|
|
1504
1683
|
}
|
|
1505
1684
|
async *list(namespace) {
|
|
@@ -1816,43 +1995,48 @@ var LocalFsRegistryDriver = class {
|
|
|
1816
1995
|
constructor(root, options = {}) {
|
|
1817
1996
|
this.root = root;
|
|
1818
1997
|
this.now = options.now ?? (() => Date.now());
|
|
1998
|
+
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
1819
1999
|
}
|
|
1820
2000
|
root;
|
|
1821
2001
|
now;
|
|
2002
|
+
entropy;
|
|
2003
|
+
/**
|
|
2004
|
+
* `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
|
|
2005
|
+
* the root in this process takes, so no write can land between the check and the unlink.
|
|
2006
|
+
*/
|
|
1822
2007
|
capabilities() {
|
|
1823
|
-
return { strongRead: true };
|
|
2008
|
+
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
1824
2009
|
}
|
|
1825
2010
|
async get(ref) {
|
|
1826
2011
|
const env = await this.readRow(registryRowPath(this.root, ref));
|
|
1827
2012
|
return env && !env.deleted ? env.record : null;
|
|
1828
2013
|
}
|
|
1829
2014
|
async create(ref, record) {
|
|
1830
|
-
validateNewRegistryRecord(record);
|
|
2015
|
+
const checked = validateNewRegistryRecord(record);
|
|
1831
2016
|
const path = registryRowPath(this.root, ref);
|
|
1832
2017
|
return this.withRowLock(path, async () => {
|
|
1833
2018
|
const current = await this.readRow(path);
|
|
1834
2019
|
if (current !== null && !current.deleted) {
|
|
1835
2020
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
1836
2021
|
}
|
|
1837
|
-
const
|
|
1838
|
-
|
|
1839
|
-
await this.writeRow(path, false, recordFromNew(ref, record, this.now(), token));
|
|
2022
|
+
const token = newIncarnationToken(this.entropy, current?.record);
|
|
2023
|
+
await this.writeRow(path, false, recordFromNew(ref, checked, this.now(), token));
|
|
1840
2024
|
return { token };
|
|
1841
2025
|
});
|
|
1842
2026
|
}
|
|
1843
2027
|
async compareAndSwap(ref, expected, patch) {
|
|
1844
|
-
validateRegistryPatch(patch);
|
|
2028
|
+
const checked = validateRegistryPatch(patch);
|
|
1845
2029
|
const path = registryRowPath(this.root, ref);
|
|
1846
2030
|
return this.withRowLock(path, async () => {
|
|
1847
2031
|
const current = await this.readRow(path);
|
|
1848
2032
|
if (current === null || current.deleted || current.record.token !== expected) {
|
|
1849
2033
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1850
2034
|
}
|
|
1851
|
-
const token =
|
|
2035
|
+
const token = nextRegistryToken(current.record, this.entropy);
|
|
1852
2036
|
await this.writeRow(
|
|
1853
2037
|
path,
|
|
1854
2038
|
false,
|
|
1855
|
-
applyRegistryPatch(current.record,
|
|
2039
|
+
applyRegistryPatch(current.record, checked, this.now(), token)
|
|
1856
2040
|
);
|
|
1857
2041
|
return { token };
|
|
1858
2042
|
});
|
|
@@ -1886,10 +2070,23 @@ var LocalFsRegistryDriver = class {
|
|
|
1886
2070
|
} else if (current === null || current.deleted) {
|
|
1887
2071
|
return;
|
|
1888
2072
|
}
|
|
1889
|
-
|
|
2073
|
+
if (incarnationOf(current.record.token) !== void 0) {
|
|
2074
|
+
await this.unlinkRow(path);
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
const token = nextRegistryToken(current.record, this.entropy);
|
|
1890
2078
|
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
1891
2079
|
});
|
|
1892
2080
|
}
|
|
2081
|
+
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
2082
|
+
async unlinkRow(path) {
|
|
2083
|
+
try {
|
|
2084
|
+
await unlink2(path);
|
|
2085
|
+
} catch (err) {
|
|
2086
|
+
if (!isCode(err, "ENOENT")) throw mapFsError(err);
|
|
2087
|
+
}
|
|
2088
|
+
await fsyncDir(dirname2(path));
|
|
2089
|
+
}
|
|
1893
2090
|
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
1894
2091
|
async namespaceDirs(namespace) {
|
|
1895
2092
|
if (namespace !== void 0) return [namespace];
|
|
@@ -2045,6 +2242,8 @@ function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
|
2045
2242
|
var AAD_VERSION = 1;
|
|
2046
2243
|
var SCOPE_CHUNK = 0;
|
|
2047
2244
|
var SCOPE_INDEX = 1;
|
|
2245
|
+
var SCOPE_METADATA = 2;
|
|
2246
|
+
var SCOPE_SUMMARY = 3;
|
|
2048
2247
|
function aadFor(ref, generation, scope) {
|
|
2049
2248
|
const enc = new TextEncoder();
|
|
2050
2249
|
const ns = enc.encode(ref.namespace ?? "");
|
|
@@ -2064,12 +2263,82 @@ function aadFor(ref, generation, scope) {
|
|
|
2064
2263
|
o += seg.length;
|
|
2065
2264
|
view.setBigUint64(o, BigInt(generation), true);
|
|
2066
2265
|
o += 8;
|
|
2067
|
-
out[o] = scope === "index" ? SCOPE_INDEX : SCOPE_CHUNK;
|
|
2266
|
+
out[o] = scope === "index" ? SCOPE_INDEX : scope === "metadata" ? SCOPE_METADATA : scope === "summary" ? SCOPE_SUMMARY : SCOPE_CHUNK;
|
|
2068
2267
|
o += 1;
|
|
2069
|
-
view.setUint32(o, scope === "
|
|
2268
|
+
view.setUint32(o, typeof scope === "number" ? scope : 0, true);
|
|
2070
2269
|
return out;
|
|
2071
2270
|
}
|
|
2072
2271
|
|
|
2272
|
+
// src/core/crbm/format.ts
|
|
2273
|
+
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
2274
|
+
var VERSION_MAJOR = 1;
|
|
2275
|
+
var VERSION_MINOR = 0;
|
|
2276
|
+
var PREAMBLE_BYTES = 8;
|
|
2277
|
+
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
2278
|
+
var FOOTER_BYTES = 104;
|
|
2279
|
+
var FOOTER = {
|
|
2280
|
+
indexOffset: 0,
|
|
2281
|
+
// u64
|
|
2282
|
+
indexLength: 8,
|
|
2283
|
+
// u64
|
|
2284
|
+
indexCrc32c: 16,
|
|
2285
|
+
// u32
|
|
2286
|
+
flags: 20,
|
|
2287
|
+
// u32
|
|
2288
|
+
payloadCodecId: 24,
|
|
2289
|
+
// u16 — which codec wrote the chunk payloads
|
|
2290
|
+
elementWidth: 26,
|
|
2291
|
+
// u8
|
|
2292
|
+
containerCodec: 27,
|
|
2293
|
+
// u8
|
|
2294
|
+
versionMajor: 28,
|
|
2295
|
+
// u8
|
|
2296
|
+
versionMinor: 29,
|
|
2297
|
+
// u8
|
|
2298
|
+
reserved2: 30,
|
|
2299
|
+
// u16
|
|
2300
|
+
generation: 32,
|
|
2301
|
+
// u64
|
|
2302
|
+
indexNonce: 40,
|
|
2303
|
+
// 12 B
|
|
2304
|
+
indexTag: 52,
|
|
2305
|
+
// 16 B
|
|
2306
|
+
keyId: 68,
|
|
2307
|
+
// 16 B
|
|
2308
|
+
chunkCount: 84,
|
|
2309
|
+
// u32
|
|
2310
|
+
totalCardinality: 88,
|
|
2311
|
+
// u64
|
|
2312
|
+
footerCrc32c: 96,
|
|
2313
|
+
// u32 — covers footer bytes [0, 96)
|
|
2314
|
+
endMagic: 100
|
|
2315
|
+
// char[4]
|
|
2316
|
+
};
|
|
2317
|
+
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
2318
|
+
var FLAG_ENCRYPTED = 1 << 0;
|
|
2319
|
+
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
2320
|
+
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
2321
|
+
var FLAG_EXTENSION = 1 << 3;
|
|
2322
|
+
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
|
|
2323
|
+
var ELEMENT_WIDTH_32 = 32;
|
|
2324
|
+
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
2325
|
+
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
2326
|
+
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
2327
|
+
]);
|
|
2328
|
+
var CONTAINER_CODEC_NONE = 0;
|
|
2329
|
+
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
2330
|
+
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
2331
|
+
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
2332
|
+
var MAX_CHUNK_CARDINALITY = 65536;
|
|
2333
|
+
var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
|
|
2334
|
+
var EXT_TRAILER_BYTES = 12;
|
|
2335
|
+
var EXT_SECTION_HEADER_BYTES = 5;
|
|
2336
|
+
var MAX_EXT_BYTES = 4 * 1024;
|
|
2337
|
+
var EXT_SECTION_METADATA = 1;
|
|
2338
|
+
var CRC32C_BYTES = 4;
|
|
2339
|
+
var AEAD_NONCE_BYTES = 12;
|
|
2340
|
+
var AEAD_TAG_BYTES = 16;
|
|
2341
|
+
|
|
2073
2342
|
// src/core/crbm/crc32c.ts
|
|
2074
2343
|
var TABLES = (() => {
|
|
2075
2344
|
const POLY = 2197175160;
|
|
@@ -2148,71 +2417,9 @@ function readVarint(bytes, offset) {
|
|
|
2148
2417
|
throw new IntegrityError("varint exceeds u32 range");
|
|
2149
2418
|
}
|
|
2150
2419
|
|
|
2151
|
-
// src/core/crbm/format.ts
|
|
2152
|
-
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
2153
|
-
var VERSION_MAJOR = 1;
|
|
2154
|
-
var VERSION_MINOR = 0;
|
|
2155
|
-
var PREAMBLE_BYTES = 8;
|
|
2156
|
-
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
2157
|
-
var FOOTER_BYTES = 104;
|
|
2158
|
-
var FOOTER = {
|
|
2159
|
-
indexOffset: 0,
|
|
2160
|
-
// u64
|
|
2161
|
-
indexLength: 8,
|
|
2162
|
-
// u64
|
|
2163
|
-
indexCrc32c: 16,
|
|
2164
|
-
// u32
|
|
2165
|
-
flags: 20,
|
|
2166
|
-
// u32
|
|
2167
|
-
payloadCodecId: 24,
|
|
2168
|
-
// u16 — which codec wrote the chunk payloads
|
|
2169
|
-
elementWidth: 26,
|
|
2170
|
-
// u8
|
|
2171
|
-
containerCodec: 27,
|
|
2172
|
-
// u8
|
|
2173
|
-
versionMajor: 28,
|
|
2174
|
-
// u8
|
|
2175
|
-
versionMinor: 29,
|
|
2176
|
-
// u8
|
|
2177
|
-
reserved2: 30,
|
|
2178
|
-
// u16
|
|
2179
|
-
generation: 32,
|
|
2180
|
-
// u64
|
|
2181
|
-
indexNonce: 40,
|
|
2182
|
-
// 12 B
|
|
2183
|
-
indexTag: 52,
|
|
2184
|
-
// 16 B
|
|
2185
|
-
keyId: 68,
|
|
2186
|
-
// 16 B
|
|
2187
|
-
chunkCount: 84,
|
|
2188
|
-
// u32
|
|
2189
|
-
totalCardinality: 88,
|
|
2190
|
-
// u64
|
|
2191
|
-
footerCrc32c: 96,
|
|
2192
|
-
// u32 — covers footer bytes [0, 96)
|
|
2193
|
-
endMagic: 100
|
|
2194
|
-
// char[4]
|
|
2195
|
-
};
|
|
2196
|
-
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
2197
|
-
var FLAG_ENCRYPTED = 1 << 0;
|
|
2198
|
-
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
2199
|
-
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
2200
|
-
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN;
|
|
2201
|
-
var ELEMENT_WIDTH_32 = 32;
|
|
2202
|
-
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
2203
|
-
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
2204
|
-
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
2205
|
-
]);
|
|
2206
|
-
var CONTAINER_CODEC_NONE = 0;
|
|
2207
|
-
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
2208
|
-
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
2209
|
-
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
2210
|
-
var MAX_CHUNK_CARDINALITY = 65536;
|
|
2211
|
-
var CRC32C_BYTES = 4;
|
|
2212
|
-
var AEAD_NONCE_BYTES = 12;
|
|
2213
|
-
var AEAD_TAG_BYTES = 16;
|
|
2214
|
-
|
|
2215
2420
|
// src/core/crbm/reader.ts
|
|
2421
|
+
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
2422
|
+
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
2216
2423
|
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
2217
2424
|
function indexCapacity(indexLength) {
|
|
2218
2425
|
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
@@ -2243,6 +2450,11 @@ function checkedFooter(tail, size) {
|
|
|
2243
2450
|
}
|
|
2244
2451
|
return { footer, fview, storedFooterCrc };
|
|
2245
2452
|
}
|
|
2453
|
+
async function footerSaysEncrypted(blob) {
|
|
2454
|
+
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2455
|
+
const { fview } = checkedFooter(tail, size);
|
|
2456
|
+
return (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0;
|
|
2457
|
+
}
|
|
2246
2458
|
function readU64(view, offset, field) {
|
|
2247
2459
|
const big = view.getBigUint64(offset, true);
|
|
2248
2460
|
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
@@ -2251,7 +2463,7 @@ function readU64(view, offset, field) {
|
|
|
2251
2463
|
return Number(big);
|
|
2252
2464
|
}
|
|
2253
2465
|
var CrbmReader = class _CrbmReader {
|
|
2254
|
-
constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc) {
|
|
2466
|
+
constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc, metadata, metadataWeight) {
|
|
2255
2467
|
this.blob = blob;
|
|
2256
2468
|
this.objectSize = objectSize;
|
|
2257
2469
|
this.generation = generation;
|
|
@@ -2261,6 +2473,8 @@ var CrbmReader = class _CrbmReader {
|
|
|
2261
2473
|
this.servedFromTail = servedFromTail;
|
|
2262
2474
|
this.crypto = crypto;
|
|
2263
2475
|
this.footerCrc = footerCrc;
|
|
2476
|
+
this.metadata = metadata;
|
|
2477
|
+
this.metadataWeight = metadataWeight;
|
|
2264
2478
|
}
|
|
2265
2479
|
blob;
|
|
2266
2480
|
objectSize;
|
|
@@ -2271,6 +2485,8 @@ var CrbmReader = class _CrbmReader {
|
|
|
2271
2485
|
servedFromTail;
|
|
2272
2486
|
crypto;
|
|
2273
2487
|
footerCrc;
|
|
2488
|
+
metadata;
|
|
2489
|
+
metadataWeight;
|
|
2274
2490
|
/**
|
|
2275
2491
|
* Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
|
|
2276
2492
|
* objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
|
|
@@ -2292,6 +2508,15 @@ var CrbmReader = class _CrbmReader {
|
|
|
2292
2508
|
const { keys, cardinalityMinusOne, lengths, crcs, offsets } = this.index;
|
|
2293
2509
|
return keys.byteLength + cardinalityMinusOne.byteLength + lengths.byteLength + crcs.byteLength + offsets.byteLength;
|
|
2294
2510
|
}
|
|
2511
|
+
/**
|
|
2512
|
+
* What this reader holds, the weight the storage reader cache bounds on: its parsed index
|
|
2513
|
+
* ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
|
|
2514
|
+
* of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
|
|
2515
|
+
* how the metadata is weighed may change in any release.
|
|
2516
|
+
*/
|
|
2517
|
+
get retainedBytes() {
|
|
2518
|
+
return this.retainedIndexBytes + this.metadataWeight;
|
|
2519
|
+
}
|
|
2295
2520
|
/** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
|
|
2296
2521
|
cardinalities() {
|
|
2297
2522
|
const { keys, cardinalityMinusOne } = this.index;
|
|
@@ -2358,6 +2583,13 @@ var CrbmReader = class _CrbmReader {
|
|
|
2358
2583
|
const chunkCount = fview.getUint32(FOOTER.chunkCount, true);
|
|
2359
2584
|
const totalCardinality = readU64(fview, FOOTER.totalCardinality, "total_cardinality");
|
|
2360
2585
|
const generation = readU64(fview, FOOTER.generation, "generation");
|
|
2586
|
+
if (!encrypted && options.crypto !== void 0) {
|
|
2587
|
+
throw new IntegrityError(
|
|
2588
|
+
`.crbm generation ${generation} is not encrypted, but it was opened with a key: a cleartext object where an encrypted one belongs is forged, corrupt, or a write that never published (a crash, before the segment's key was made); a load's collection, dropSegment or deleting the object removes it`
|
|
2589
|
+
);
|
|
2590
|
+
}
|
|
2591
|
+
const versionMinor = footer[FOOTER.versionMinor];
|
|
2592
|
+
const hasExtension = (flags & FLAG_EXTENSION) !== 0;
|
|
2361
2593
|
if (indexOffset < PAYLOAD_START || indexOffset + indexLength > size - FOOTER_BYTES) {
|
|
2362
2594
|
throw new IntegrityError(
|
|
2363
2595
|
`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`
|
|
@@ -2368,20 +2600,27 @@ var CrbmReader = class _CrbmReader {
|
|
|
2368
2600
|
}
|
|
2369
2601
|
const tailCoversFrom = size - tail.length;
|
|
2370
2602
|
if (tailCoversFrom === 0) {
|
|
2371
|
-
if (!magicMatches(tail, 0) || tail[4] !== versionMajor) {
|
|
2603
|
+
if (!magicMatches(tail, 0) || tail[4] !== versionMajor || tail[5] !== versionMinor) {
|
|
2372
2604
|
throw new IntegrityError(".crbm preamble magic/version mismatch with footer");
|
|
2373
2605
|
}
|
|
2374
2606
|
}
|
|
2375
|
-
|
|
2607
|
+
const lowest = hasExtension ? Math.max(PAYLOAD_START, indexOffset - EXT_TRAILER_BYTES - MAX_EXT_BYTES) : indexOffset;
|
|
2608
|
+
let region;
|
|
2609
|
+
let regionStart;
|
|
2376
2610
|
let servedFromTail;
|
|
2377
2611
|
if (indexOffset >= tailCoversFrom) {
|
|
2378
|
-
|
|
2379
|
-
|
|
2612
|
+
region = tail;
|
|
2613
|
+
regionStart = tailCoversFrom;
|
|
2380
2614
|
servedFromTail = true;
|
|
2381
2615
|
} else {
|
|
2382
|
-
|
|
2616
|
+
region = await blob.getRange(lowest, indexOffset + indexLength - lowest);
|
|
2617
|
+
regionStart = lowest;
|
|
2383
2618
|
servedFromTail = false;
|
|
2384
2619
|
}
|
|
2620
|
+
const indexBytes = region.subarray(
|
|
2621
|
+
indexOffset - regionStart,
|
|
2622
|
+
indexOffset - regionStart + indexLength
|
|
2623
|
+
);
|
|
2385
2624
|
if (crc32c(indexBytes) !== indexCrc) {
|
|
2386
2625
|
throw new IntegrityError(".crbm index CRC mismatch");
|
|
2387
2626
|
}
|
|
@@ -2393,9 +2632,56 @@ var CrbmReader = class _CrbmReader {
|
|
|
2393
2632
|
},
|
|
2394
2633
|
options.crypto.aadFor("index")
|
|
2395
2634
|
) : indexBytes;
|
|
2635
|
+
let payloadEnd = indexOffset;
|
|
2636
|
+
let metadata;
|
|
2637
|
+
let metadataLength = 0;
|
|
2638
|
+
if (hasExtension) {
|
|
2639
|
+
const trailerStart = indexOffset - EXT_TRAILER_BYTES;
|
|
2640
|
+
if (trailerStart < PAYLOAD_START) {
|
|
2641
|
+
throw new IntegrityError(".crbm flags an extension block, but has no room for one");
|
|
2642
|
+
}
|
|
2643
|
+
let window = { bytes: region, start: regionStart };
|
|
2644
|
+
if (trailerStart < regionStart) {
|
|
2645
|
+
window = { bytes: await blob.getRange(lowest, indexOffset - lowest), start: lowest };
|
|
2646
|
+
servedFromTail = false;
|
|
2647
|
+
}
|
|
2648
|
+
const trailer = within(window.bytes, window.start, trailerStart, indexOffset);
|
|
2649
|
+
const tview = new DataView(trailer.buffer, trailer.byteOffset, trailer.byteLength);
|
|
2650
|
+
if (!EXT_MAGIC.every((b, i) => trailer[8 + i] === b)) {
|
|
2651
|
+
throw new IntegrityError(".crbm extension block trailer magic mismatch");
|
|
2652
|
+
}
|
|
2653
|
+
const sectionsLength = tview.getUint32(0, true);
|
|
2654
|
+
if (sectionsLength > MAX_EXT_BYTES) {
|
|
2655
|
+
throw new IntegrityError(
|
|
2656
|
+
`.crbm extension block ${sectionsLength}B exceeds cap ${MAX_EXT_BYTES}B`
|
|
2657
|
+
);
|
|
2658
|
+
}
|
|
2659
|
+
const extStart = trailerStart - sectionsLength;
|
|
2660
|
+
if (extStart < PAYLOAD_START) {
|
|
2661
|
+
throw new IntegrityError(`.crbm extension block of ${sectionsLength}B out of bounds`);
|
|
2662
|
+
}
|
|
2663
|
+
let covered;
|
|
2664
|
+
if (extStart >= window.start) {
|
|
2665
|
+
covered = within(window.bytes, window.start, extStart, trailerStart + 4);
|
|
2666
|
+
} else {
|
|
2667
|
+
covered = await blob.getRange(extStart, sectionsLength + 4);
|
|
2668
|
+
servedFromTail = false;
|
|
2669
|
+
if (covered.length !== sectionsLength + 4) {
|
|
2670
|
+
throw new IntegrityError(".crbm extension block read short");
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
if (crc32c(covered) !== tview.getUint32(4, true)) {
|
|
2674
|
+
throw new IntegrityError(".crbm extension block CRC mismatch");
|
|
2675
|
+
}
|
|
2676
|
+
({ metadata, metadataLength } = extensionContents(
|
|
2677
|
+
covered.subarray(0, sectionsLength),
|
|
2678
|
+
options.crypto
|
|
2679
|
+
));
|
|
2680
|
+
payloadEnd = extStart;
|
|
2681
|
+
}
|
|
2396
2682
|
const index = parseIndex(
|
|
2397
2683
|
indexForParse,
|
|
2398
|
-
|
|
2684
|
+
payloadEnd,
|
|
2399
2685
|
maxPayloadBytes,
|
|
2400
2686
|
encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
|
|
2401
2687
|
);
|
|
@@ -2426,7 +2712,9 @@ var CrbmReader = class _CrbmReader {
|
|
|
2426
2712
|
index,
|
|
2427
2713
|
servedFromTail,
|
|
2428
2714
|
options.crypto,
|
|
2429
|
-
storedFooterCrc
|
|
2715
|
+
storedFooterCrc,
|
|
2716
|
+
metadata,
|
|
2717
|
+
metadata === void 0 ? 0 : RETAINED_BYTES_PER_METADATA_BYTE * metadataLength + RETAINED_BYTES_PER_METADATA_KEY * Object.keys(metadata).length
|
|
2430
2718
|
);
|
|
2431
2719
|
}
|
|
2432
2720
|
/** Chunk keys present in this generation, ascending. */
|
|
@@ -2486,16 +2774,77 @@ var CrbmReader = class _CrbmReader {
|
|
|
2486
2774
|
);
|
|
2487
2775
|
}
|
|
2488
2776
|
};
|
|
2489
|
-
function
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2777
|
+
function within(bytes, start, from, to) {
|
|
2778
|
+
if (from < start || to > start + bytes.length) {
|
|
2779
|
+
throw new IntegrityError(".crbm extension block read short");
|
|
2780
|
+
}
|
|
2781
|
+
return bytes.subarray(from - start, to - start);
|
|
2782
|
+
}
|
|
2783
|
+
function extensionCorrupt(message) {
|
|
2784
|
+
throw new IntegrityError(`.crbm extension block: ${message}`);
|
|
2785
|
+
}
|
|
2786
|
+
function extensionContents(sections, crypto) {
|
|
2787
|
+
const view = new DataView(sections.buffer, sections.byteOffset, sections.byteLength);
|
|
2788
|
+
let metadata;
|
|
2789
|
+
let metadataLength = 0;
|
|
2790
|
+
let lastType = 0;
|
|
2497
2791
|
let pos = 0;
|
|
2498
|
-
|
|
2792
|
+
while (pos < sections.length) {
|
|
2793
|
+
if (sections.length - pos < EXT_SECTION_HEADER_BYTES)
|
|
2794
|
+
extensionCorrupt("a section header is cut off");
|
|
2795
|
+
const type = sections[pos];
|
|
2796
|
+
const length = view.getUint32(pos + 1, true);
|
|
2797
|
+
pos += EXT_SECTION_HEADER_BYTES;
|
|
2798
|
+
if (type <= lastType) {
|
|
2799
|
+
extensionCorrupt(
|
|
2800
|
+
type === 0 ? "section type 0 is not a section type" : `section type ${type} after ${lastType}`
|
|
2801
|
+
);
|
|
2802
|
+
}
|
|
2803
|
+
if (length > sections.length - pos) extensionCorrupt(`section ${type} runs past the block`);
|
|
2804
|
+
const body = sections.subarray(pos, pos + length);
|
|
2805
|
+
pos += length;
|
|
2806
|
+
lastType = type;
|
|
2807
|
+
if (type === EXT_SECTION_METADATA)
|
|
2808
|
+
({ metadata, metadataLength } = metadataSection(body, crypto));
|
|
2809
|
+
}
|
|
2810
|
+
return { metadata, metadataLength };
|
|
2811
|
+
}
|
|
2812
|
+
function metadataSection(body, crypto) {
|
|
2813
|
+
if (crypto === void 0) {
|
|
2814
|
+
return { metadata: metadataFromBytes(body, extensionCorrupt), metadataLength: body.length };
|
|
2815
|
+
}
|
|
2816
|
+
const framing = AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
|
|
2817
|
+
if (body.length <= framing || body.length > framing + MAX_METADATA_BYTES) {
|
|
2818
|
+
extensionCorrupt(`sealed metadata of ${body.length}B is not a nonce, up to 1 KiB and a tag`);
|
|
2819
|
+
}
|
|
2820
|
+
let plain;
|
|
2821
|
+
try {
|
|
2822
|
+
plain = crypto.aead.open(
|
|
2823
|
+
{
|
|
2824
|
+
nonce: body.subarray(0, AEAD_NONCE_BYTES),
|
|
2825
|
+
ciphertext: body.subarray(AEAD_NONCE_BYTES, body.length - AEAD_TAG_BYTES),
|
|
2826
|
+
tag: body.subarray(body.length - AEAD_TAG_BYTES)
|
|
2827
|
+
},
|
|
2828
|
+
crypto.aadFor("metadata")
|
|
2829
|
+
);
|
|
2830
|
+
} catch (err) {
|
|
2831
|
+
if (!isIntegrityError(err)) throw err;
|
|
2832
|
+
extensionCorrupt(
|
|
2833
|
+
"the sealed metadata does not open: tampered bytes, a block moved from another object, or a CrbmCrypto whose aadFor does not map the 'metadata' scope"
|
|
2834
|
+
);
|
|
2835
|
+
}
|
|
2836
|
+
return { metadata: metadataFromBytes(plain, extensionCorrupt), metadataLength: plain.length };
|
|
2837
|
+
}
|
|
2838
|
+
function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1) {
|
|
2839
|
+
const capacity = indexCapacity(indexBytes.length);
|
|
2840
|
+
const keys = new Uint16Array(capacity);
|
|
2841
|
+
const cardinalityMinusOne = new Uint16Array(capacity);
|
|
2842
|
+
const lengths = new Uint32Array(capacity);
|
|
2843
|
+
const crcs = new Uint32Array(capacity);
|
|
2844
|
+
const offsets = new Float64Array(capacity);
|
|
2845
|
+
let count = 0;
|
|
2846
|
+
let pos = 0;
|
|
2847
|
+
let prevKey = 0;
|
|
2499
2848
|
let prevEnd = PAYLOAD_START;
|
|
2500
2849
|
let cardinalitySum = 0;
|
|
2501
2850
|
for (let i = 0; pos < indexBytes.length; i++) {
|
|
@@ -2561,6 +2910,9 @@ var CrbmWriter = class {
|
|
|
2561
2910
|
`generation must be a non-negative integer; got ${options.generation}`
|
|
2562
2911
|
);
|
|
2563
2912
|
}
|
|
2913
|
+
this.metadata = metadataBytes(options.metadata, (message) => {
|
|
2914
|
+
throw new ValidationError(message);
|
|
2915
|
+
});
|
|
2564
2916
|
}
|
|
2565
2917
|
sink;
|
|
2566
2918
|
options;
|
|
@@ -2570,6 +2922,11 @@ var CrbmWriter = class {
|
|
|
2570
2922
|
totalCardinality = 0;
|
|
2571
2923
|
preambleWritten = false;
|
|
2572
2924
|
finished = false;
|
|
2925
|
+
/** The metadata's canonical JSON, copied at construction; `undefined` leaves the footer's extension flag clear. */
|
|
2926
|
+
metadata;
|
|
2927
|
+
/** Bytes written so far, the footer included once {@link finish} has run. */
|
|
2928
|
+
written = 0;
|
|
2929
|
+
footerCrc32c;
|
|
2573
2930
|
/**
|
|
2574
2931
|
* Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
|
|
2575
2932
|
* strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
|
|
@@ -2591,7 +2948,8 @@ var CrbmWriter = class {
|
|
|
2591
2948
|
}
|
|
2592
2949
|
if (payload.length === 0) throw new ValidationError("chunk payload must be non-empty");
|
|
2593
2950
|
await this.ensurePreamble();
|
|
2594
|
-
const
|
|
2951
|
+
const crypto = this.options.crypto;
|
|
2952
|
+
const stored = crypto ? framed(crypto.aead.seal(payload, crypto.aadFor(chunkKey))) : payload;
|
|
2595
2953
|
await this.sink.write(stored);
|
|
2596
2954
|
this.entries.push({
|
|
2597
2955
|
chunkKey,
|
|
@@ -2604,20 +2962,18 @@ var CrbmWriter = class {
|
|
|
2604
2962
|
this.totalCardinality += cardinality;
|
|
2605
2963
|
this.lastKey = chunkKey;
|
|
2606
2964
|
}
|
|
2607
|
-
/**
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
|
|
2612
|
-
out.set(sealed.nonce, 0);
|
|
2613
|
-
out.set(sealed.ciphertext, sealed.nonce.length);
|
|
2614
|
-
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
2615
|
-
return out;
|
|
2616
|
-
}
|
|
2617
|
-
/** Write the index region + the fixed footer. After this the object is complete and immutable. */
|
|
2965
|
+
/**
|
|
2966
|
+
* Write the extension block (when there is metadata), the index region and the fixed footer. After this the object
|
|
2967
|
+
* is complete and immutable.
|
|
2968
|
+
*/
|
|
2618
2969
|
async finish() {
|
|
2619
2970
|
if (this.finished) throw new ValidationError("CrbmWriter already finished");
|
|
2620
2971
|
await this.ensurePreamble();
|
|
2972
|
+
if (this.metadata !== void 0) {
|
|
2973
|
+
const block = this.extensionBlock(this.metadata);
|
|
2974
|
+
await this.sink.write(block);
|
|
2975
|
+
this.offset += block.length;
|
|
2976
|
+
}
|
|
2621
2977
|
const indexPlain = this.encodeIndex();
|
|
2622
2978
|
let indexRegion = indexPlain;
|
|
2623
2979
|
let indexNonce;
|
|
@@ -2633,8 +2989,22 @@ var CrbmWriter = class {
|
|
|
2633
2989
|
this.offset += indexRegion.length;
|
|
2634
2990
|
const footer = this.buildFooter(indexOffset, indexRegion, indexNonce, indexTag);
|
|
2635
2991
|
await this.sink.write(footer);
|
|
2992
|
+
this.written = this.offset + footer.length;
|
|
2993
|
+
this.footerCrc32c = new DataView(footer.buffer, footer.byteOffset).getUint32(
|
|
2994
|
+
FOOTER.footerCrc32c,
|
|
2995
|
+
true
|
|
2996
|
+
);
|
|
2636
2997
|
this.finished = true;
|
|
2637
2998
|
}
|
|
2999
|
+
/**
|
|
3000
|
+
* The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
|
|
3001
|
+
* has to tell its own object from another stored later under the same key compares these with the object's
|
|
3002
|
+
* footer. Throws before {@link finish}.
|
|
3003
|
+
*/
|
|
3004
|
+
get identity() {
|
|
3005
|
+
if (this.footerCrc32c === void 0) throw new ValidationError("CrbmWriter not finished");
|
|
3006
|
+
return { size: this.written, footerCrc: this.footerCrc32c };
|
|
3007
|
+
}
|
|
2638
3008
|
async ensurePreamble() {
|
|
2639
3009
|
if (this.preambleWritten) return;
|
|
2640
3010
|
const preamble = new Uint8Array(PREAMBLE_BYTES);
|
|
@@ -2644,6 +3014,25 @@ var CrbmWriter = class {
|
|
|
2644
3014
|
await this.sink.write(preamble);
|
|
2645
3015
|
this.preambleWritten = true;
|
|
2646
3016
|
}
|
|
3017
|
+
/**
|
|
3018
|
+
* The extension block: one metadata section (`u8 type ‖ u32 length ‖ bytes`), then the trailer
|
|
3019
|
+
* (`u32 sectionsLength ‖ u32 crc32c(sections ‖ sectionsLength) ‖ "CRBX"`). Encrypted, the section holds
|
|
3020
|
+
* `nonce ‖ ciphertext ‖ tag`, sealed with the metadata scope of the associated data.
|
|
3021
|
+
*/
|
|
3022
|
+
extensionBlock(metadata) {
|
|
3023
|
+
const crypto = this.options.crypto;
|
|
3024
|
+
const body = crypto ? framed(crypto.aead.seal(metadata, crypto.aadFor("metadata"))) : metadata;
|
|
3025
|
+
const sectionsLength = EXT_SECTION_HEADER_BYTES + body.length;
|
|
3026
|
+
const block = new Uint8Array(sectionsLength + EXT_TRAILER_BYTES);
|
|
3027
|
+
const view = new DataView(block.buffer);
|
|
3028
|
+
block[0] = EXT_SECTION_METADATA;
|
|
3029
|
+
view.setUint32(1, body.length, true);
|
|
3030
|
+
block.set(body, EXT_SECTION_HEADER_BYTES);
|
|
3031
|
+
view.setUint32(sectionsLength, sectionsLength, true);
|
|
3032
|
+
view.setUint32(sectionsLength + 4, crc32c(block.subarray(0, sectionsLength + 4)), true);
|
|
3033
|
+
block.set(EXT_MAGIC, sectionsLength + 8);
|
|
3034
|
+
return block;
|
|
3035
|
+
}
|
|
2647
3036
|
encodeIndex() {
|
|
2648
3037
|
const out = [];
|
|
2649
3038
|
let prevKey = 0;
|
|
@@ -2671,7 +3060,11 @@ var CrbmWriter = class {
|
|
|
2671
3060
|
view.setBigUint64(FOOTER.indexOffset, BigInt(indexOffset), true);
|
|
2672
3061
|
view.setBigUint64(FOOTER.indexLength, BigInt(indexRegion.length), true);
|
|
2673
3062
|
view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
|
|
2674
|
-
view.setUint32(
|
|
3063
|
+
view.setUint32(
|
|
3064
|
+
FOOTER.flags,
|
|
3065
|
+
FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0) | (this.metadata === void 0 ? 0 : FLAG_EXTENSION),
|
|
3066
|
+
true
|
|
3067
|
+
);
|
|
2675
3068
|
view.setUint16(
|
|
2676
3069
|
FOOTER.payloadCodecId,
|
|
2677
3070
|
this.options.payloadCodecId ?? PAYLOAD_CODEC_ROARING_PORTABLE,
|
|
@@ -2691,6 +3084,226 @@ var CrbmWriter = class {
|
|
|
2691
3084
|
return footer;
|
|
2692
3085
|
}
|
|
2693
3086
|
};
|
|
3087
|
+
function framed(sealed) {
|
|
3088
|
+
const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
|
|
3089
|
+
out.set(sealed.nonce, 0);
|
|
3090
|
+
out.set(sealed.ciphertext, sealed.nonce.length);
|
|
3091
|
+
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
3092
|
+
return out;
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
// src/core/load-input.ts
|
|
3096
|
+
var MAX_SERIALIZED_LOAD_BYTES = 4 + 8192 + 8 * 65536 + 65536 * 8192;
|
|
3097
|
+
var DecodedLoadInput = class {
|
|
3098
|
+
constructor(bitmap) {
|
|
3099
|
+
this.bitmap = bitmap;
|
|
3100
|
+
}
|
|
3101
|
+
bitmap;
|
|
3102
|
+
};
|
|
3103
|
+
function tagOf(value) {
|
|
3104
|
+
return Object.prototype.toString.call(value).slice(8, -1);
|
|
3105
|
+
}
|
|
3106
|
+
var VIEW = Object.getPrototypeOf(Uint8Array.prototype);
|
|
3107
|
+
var viewGetter = (name) => Object.getOwnPropertyDescriptor(VIEW, name)?.get;
|
|
3108
|
+
var bufferOf = viewGetter("buffer");
|
|
3109
|
+
var byteOffsetOf = viewGetter("byteOffset");
|
|
3110
|
+
var byteLengthOf = viewGetter("byteLength");
|
|
3111
|
+
function plainView(bytes, what) {
|
|
3112
|
+
let buffer;
|
|
3113
|
+
let offset;
|
|
3114
|
+
let length;
|
|
3115
|
+
try {
|
|
3116
|
+
buffer = bufferOf.call(bytes);
|
|
3117
|
+
offset = byteOffsetOf.call(bytes);
|
|
3118
|
+
length = byteLengthOf.call(bytes);
|
|
3119
|
+
} catch {
|
|
3120
|
+
throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
|
|
3121
|
+
}
|
|
3122
|
+
return length === 0 ? new Uint8Array(0) : new Uint8Array(buffer, offset, length);
|
|
3123
|
+
}
|
|
3124
|
+
var sharedByteLength = typeof SharedArrayBuffer === "function" ? Object.getOwnPropertyDescriptor(SharedArrayBuffer.prototype, "byteLength")?.get : void 0;
|
|
3125
|
+
function isShared(buffer) {
|
|
3126
|
+
if (sharedByteLength === void 0) return false;
|
|
3127
|
+
try {
|
|
3128
|
+
sharedByteLength.call(buffer);
|
|
3129
|
+
return true;
|
|
3130
|
+
} catch {
|
|
3131
|
+
return false;
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
function overCap(what, length) {
|
|
3135
|
+
return new ValidationError(
|
|
3136
|
+
`${what} is ${length} bytes, more than any canonical 32-bit bitmap serializes to (${MAX_SERIALIZED_LOAD_BYTES}). Call runOptimize() on the bitmap before serializing it.`
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
3139
|
+
function prepareLoadInput(input, codec) {
|
|
3140
|
+
if (typeof input !== "object" || input === null) {
|
|
3141
|
+
throw new ValidationError(
|
|
3142
|
+
`a load takes ids, { serialized } or { bitmap }; got ${String(input)}`
|
|
3143
|
+
);
|
|
3144
|
+
}
|
|
3145
|
+
const tag = tagOf(input);
|
|
3146
|
+
if (tag === "Uint8Array" || tag === "Uint8ClampedArray") {
|
|
3147
|
+
throw new ValidationError(
|
|
3148
|
+
`a load's ids cannot be a ${tag} or Buffer: each byte would be loaded as an id. Pass portable Roaring bytes as { serialized }, and ids as a Uint32Array or an array of numbers.`
|
|
3149
|
+
);
|
|
3150
|
+
}
|
|
3151
|
+
if (Symbol.iterator in input || Symbol.asyncIterator in input) return input;
|
|
3152
|
+
const keys = Object.keys(input);
|
|
3153
|
+
if (keys.length === 1 && keys[0] === "serialized") {
|
|
3154
|
+
return decode(input.serialized, "{ serialized }", codec);
|
|
3155
|
+
}
|
|
3156
|
+
if (keys.length === 1 && keys[0] === "bitmap") {
|
|
3157
|
+
const bitmap = input.bitmap;
|
|
3158
|
+
if (typeof bitmap !== "object" || bitmap === null || typeof bitmap.serialize !== "function") {
|
|
3159
|
+
throw new ValidationError(`{ bitmap } needs an object with serialize('portable')`);
|
|
3160
|
+
}
|
|
3161
|
+
const what = "{ bitmap }'s serialize('portable')";
|
|
3162
|
+
if (typeof bitmap.getSerializationSizeInBytes === "function") {
|
|
3163
|
+
const size = bitmap.getSerializationSizeInBytes("portable");
|
|
3164
|
+
if (typeof size === "number" && size > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, size);
|
|
3165
|
+
}
|
|
3166
|
+
return decode(bitmap.serialize("portable"), what, codec);
|
|
3167
|
+
}
|
|
3168
|
+
throw new ValidationError(
|
|
3169
|
+
`a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
function decode(bytes, what, codec) {
|
|
3173
|
+
if (typeof bytes !== "object" || bytes === null || tagOf(bytes) !== "Uint8Array") {
|
|
3174
|
+
throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
|
|
3175
|
+
}
|
|
3176
|
+
const view = plainView(bytes, what);
|
|
3177
|
+
if (view.byteLength > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, view.byteLength);
|
|
3178
|
+
const own = isShared(view.buffer) ? new Uint8Array(view) : view;
|
|
3179
|
+
try {
|
|
3180
|
+
return new DecodedLoadInput(
|
|
3181
|
+
codec.safeDeserialize(own, MAX_SERIALIZED_LOAD_BYTES, { whole: true })
|
|
3182
|
+
);
|
|
3183
|
+
} catch (err) {
|
|
3184
|
+
if (!isIntegrityError(err)) throw err;
|
|
3185
|
+
throw new ValidationError(`${what} is not a valid bitmap: ${err.message}`);
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
// src/core/summary.ts
|
|
3190
|
+
var NONCE_BYTES = 12;
|
|
3191
|
+
var TAG_BYTES = 16;
|
|
3192
|
+
var COUNT_BYTES = 8;
|
|
3193
|
+
var MAX_CARDINALITY = 2 ** 32;
|
|
3194
|
+
var BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
3195
|
+
function toBase64(bytes) {
|
|
3196
|
+
let out = "";
|
|
3197
|
+
for (let i = 0; i < bytes.length; i += 3) {
|
|
3198
|
+
const a = bytes[i];
|
|
3199
|
+
const b = bytes[i + 1];
|
|
3200
|
+
const c = bytes[i + 2];
|
|
3201
|
+
out += BASE64[a >> 2] + BASE64[(a & 3) << 4 | (b ?? 0) >> 4];
|
|
3202
|
+
out += b === void 0 ? "=" : BASE64[(b & 15) << 2 | (c ?? 0) >> 6];
|
|
3203
|
+
out += c === void 0 ? "=" : BASE64[c & 63];
|
|
3204
|
+
}
|
|
3205
|
+
return out;
|
|
3206
|
+
}
|
|
3207
|
+
function fromBase64(text) {
|
|
3208
|
+
if (text.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(text)) return void 0;
|
|
3209
|
+
const pad = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
3210
|
+
const out = new Uint8Array(text.length / 4 * 3 - pad);
|
|
3211
|
+
let o = 0;
|
|
3212
|
+
for (let i = 0; i < text.length; i += 4) {
|
|
3213
|
+
const n = BASE64.indexOf(text[i]) << 18 | BASE64.indexOf(text[i + 1]) << 12 | Math.max(BASE64.indexOf(text[i + 2]), 0) << 6 | Math.max(BASE64.indexOf(text[i + 3]), 0);
|
|
3214
|
+
if (o < out.length) out[o++] = n >> 16 & 255;
|
|
3215
|
+
if (o < out.length) out[o++] = n >> 8 & 255;
|
|
3216
|
+
if (o < out.length) out[o++] = n & 255;
|
|
3217
|
+
}
|
|
3218
|
+
return out;
|
|
3219
|
+
}
|
|
3220
|
+
function clearSummary(generation, cardinality, metadata) {
|
|
3221
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata };
|
|
3222
|
+
}
|
|
3223
|
+
function sealSummary(aead, ref, generation, cardinality, metadata) {
|
|
3224
|
+
const meta = metadataBytes(metadata, (message) => {
|
|
3225
|
+
throw new IntegrityError(`summary metadata: ${message}`);
|
|
3226
|
+
});
|
|
3227
|
+
const plain = new Uint8Array(COUNT_BYTES + (meta?.length ?? 0));
|
|
3228
|
+
new DataView(plain.buffer).setBigUint64(0, BigInt(cardinality), true);
|
|
3229
|
+
if (meta !== void 0) plain.set(meta, COUNT_BYTES);
|
|
3230
|
+
const { nonce, ciphertext, tag } = aead.seal(plain, aadFor(ref, generation, "summary"));
|
|
3231
|
+
const framed2 = new Uint8Array(nonce.length + ciphertext.length + tag.length);
|
|
3232
|
+
framed2.set(nonce, 0);
|
|
3233
|
+
framed2.set(ciphertext, nonce.length);
|
|
3234
|
+
framed2.set(tag, nonce.length + ciphertext.length);
|
|
3235
|
+
return { generation, sealed: toBase64(framed2) };
|
|
3236
|
+
}
|
|
3237
|
+
function openSummary(aead, ref, summary) {
|
|
3238
|
+
const fail = (message) => {
|
|
3239
|
+
throw new IntegrityError(`registry record summary: ${message}`);
|
|
3240
|
+
};
|
|
3241
|
+
const framed2 = fromBase64(summary.sealed);
|
|
3242
|
+
const framing = NONCE_BYTES + TAG_BYTES;
|
|
3243
|
+
if (framed2 === void 0 || framed2.length < framing + COUNT_BYTES || framed2.length > framing + COUNT_BYTES + MAX_METADATA_BYTES) {
|
|
3244
|
+
fail("the sealed summary is not a nonce, a count, up to 1 KiB of metadata and a tag");
|
|
3245
|
+
}
|
|
3246
|
+
const bytes = framed2;
|
|
3247
|
+
let plain;
|
|
3248
|
+
try {
|
|
3249
|
+
plain = aead.open(
|
|
3250
|
+
{
|
|
3251
|
+
nonce: bytes.subarray(0, NONCE_BYTES),
|
|
3252
|
+
ciphertext: bytes.subarray(NONCE_BYTES, bytes.length - TAG_BYTES),
|
|
3253
|
+
tag: bytes.subarray(bytes.length - TAG_BYTES)
|
|
3254
|
+
},
|
|
3255
|
+
aadFor(ref, summary.generation, "summary")
|
|
3256
|
+
);
|
|
3257
|
+
} catch (err) {
|
|
3258
|
+
if (!isIntegrityError(err)) throw err;
|
|
3259
|
+
return fail(
|
|
3260
|
+
"the sealed summary does not open: tampered bytes, or a summary moved from another generation"
|
|
3261
|
+
);
|
|
3262
|
+
}
|
|
3263
|
+
if (plain.length < COUNT_BYTES) fail("the sealed summary is shorter than its count");
|
|
3264
|
+
const count = new DataView(plain.buffer, plain.byteOffset, plain.byteLength).getBigUint64(
|
|
3265
|
+
0,
|
|
3266
|
+
true
|
|
3267
|
+
);
|
|
3268
|
+
if (count > BigInt(MAX_CARDINALITY)) fail(`cardinality ${count} is outside 0 to 2^32`);
|
|
3269
|
+
const metadata = plain.length === COUNT_BYTES ? void 0 : metadataFromBytes(plain.subarray(COUNT_BYTES), fail);
|
|
3270
|
+
return { cardinality: Number(count), metadata };
|
|
3271
|
+
}
|
|
3272
|
+
function usableSummary(ref, row, aead) {
|
|
3273
|
+
const summary = row.summary;
|
|
3274
|
+
if (summary === void 0 || row.status !== "active" || summary.generation !== row.currentGen) {
|
|
3275
|
+
return void 0;
|
|
3276
|
+
}
|
|
3277
|
+
const keyed = row.wrappedDeks !== void 0 && row.wrappedDeks.length > 0;
|
|
3278
|
+
if ("sealed" in summary) {
|
|
3279
|
+
if (!keyed || aead === void 0) return void 0;
|
|
3280
|
+
try {
|
|
3281
|
+
return openSummary(aead, ref, summary);
|
|
3282
|
+
} catch (err) {
|
|
3283
|
+
if (isIntegrityError(err)) return void 0;
|
|
3284
|
+
throw err;
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
if (keyed) return void 0;
|
|
3288
|
+
return { cardinality: summary.cardinality, metadata: summary.metadata };
|
|
3289
|
+
}
|
|
3290
|
+
function canonicalOrNone(metadata) {
|
|
3291
|
+
if (metadata === void 0) return void 0;
|
|
3292
|
+
const json = canonicalMetadataJson(metadata, (message) => {
|
|
3293
|
+
throw new IntegrityError(`metadata: ${message}`);
|
|
3294
|
+
});
|
|
3295
|
+
return json === "{}" ? void 0 : json;
|
|
3296
|
+
}
|
|
3297
|
+
function summaryAgrees(summary, object) {
|
|
3298
|
+
return summary.cardinality === object.cardinality && canonicalOrNone(summary.metadata) === canonicalOrNone(object.metadata);
|
|
3299
|
+
}
|
|
3300
|
+
function metadataToCarry(object, authenticated) {
|
|
3301
|
+
const empty = (metadata) => metadata === void 0 || Object.keys(metadata).length === 0;
|
|
3302
|
+
return empty(object) && authenticated !== void 0 && !empty(authenticated.metadata) ? authenticated.metadata : object;
|
|
3303
|
+
}
|
|
3304
|
+
function summaryOf(ref, generation, description, aead) {
|
|
3305
|
+
return aead === void 0 ? clearSummary(generation, description.cardinality, description.metadata) : sealSummary(aead, ref, generation, description.cardinality, description.metadata);
|
|
3306
|
+
}
|
|
2694
3307
|
|
|
2695
3308
|
// src/core/crbm-storage-source.ts
|
|
2696
3309
|
function storageBlobReader(driver, key) {
|
|
@@ -2709,6 +3322,88 @@ var notThePinned = (ref, generation) => new NotFoundError(
|
|
|
2709
3322
|
function versionOf(generation, lineage) {
|
|
2710
3323
|
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
2711
3324
|
}
|
|
3325
|
+
function summaryOfGeneration(generation, description) {
|
|
3326
|
+
const { cardinality, metadata } = description;
|
|
3327
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
3328
|
+
}
|
|
3329
|
+
var sameKeys = (a, b) => a !== void 0 && JSON.stringify(a) === JSON.stringify(b);
|
|
3330
|
+
function once(fn) {
|
|
3331
|
+
let pending;
|
|
3332
|
+
return () => {
|
|
3333
|
+
if (pending === void 0) {
|
|
3334
|
+
const mine = fn();
|
|
3335
|
+
pending = mine;
|
|
3336
|
+
mine.catch(() => {
|
|
3337
|
+
if (pending === mine) pending = void 0;
|
|
3338
|
+
});
|
|
3339
|
+
}
|
|
3340
|
+
return pending;
|
|
3341
|
+
};
|
|
3342
|
+
}
|
|
3343
|
+
var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
|
|
3344
|
+
function describe(reader) {
|
|
3345
|
+
let cardinality = 0;
|
|
3346
|
+
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
3347
|
+
return { cardinality, metadata: reader.metadata };
|
|
3348
|
+
}
|
|
3349
|
+
var Snapshot = class _Snapshot {
|
|
3350
|
+
constructor(target, open4, prior) {
|
|
3351
|
+
this.target = target;
|
|
3352
|
+
this.open = open4;
|
|
3353
|
+
this.prior = prior;
|
|
3354
|
+
}
|
|
3355
|
+
target;
|
|
3356
|
+
open;
|
|
3357
|
+
prior;
|
|
3358
|
+
/** When the snapshot's refresh clock started. A refresh that failed transiently moves it back, so it lapses sooner. */
|
|
3359
|
+
installedAtMs = 0;
|
|
3360
|
+
/** Set by the cache, to hear of a reader as it is opened. */
|
|
3361
|
+
onReader;
|
|
3362
|
+
opened;
|
|
3363
|
+
static live(target, open4, prior) {
|
|
3364
|
+
return new _Snapshot(target, open4, prior);
|
|
3365
|
+
}
|
|
3366
|
+
static eager(reader) {
|
|
3367
|
+
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
3368
|
+
snap.opened = reader;
|
|
3369
|
+
return snap;
|
|
3370
|
+
}
|
|
3371
|
+
/** The reader, opened by the first call. */
|
|
3372
|
+
get reader() {
|
|
3373
|
+
if (this.opened === void 0) {
|
|
3374
|
+
const target = this.target;
|
|
3375
|
+
this.opened = target.then(async (live) => {
|
|
3376
|
+
if (live === null) return null;
|
|
3377
|
+
const kept = await this.reuse(live);
|
|
3378
|
+
this.prior = void 0;
|
|
3379
|
+
return kept ?? this.open(live);
|
|
3380
|
+
});
|
|
3381
|
+
this.onReader?.(this.opened);
|
|
3382
|
+
}
|
|
3383
|
+
return this.opened;
|
|
3384
|
+
}
|
|
3385
|
+
/** The reader if one has been asked for, without asking for one. */
|
|
3386
|
+
get openedReader() {
|
|
3387
|
+
return this.opened;
|
|
3388
|
+
}
|
|
3389
|
+
/** The prior snapshot's reader, if it is of what this snapshot resolved. */
|
|
3390
|
+
async reuse(live) {
|
|
3391
|
+
const prior = this.prior;
|
|
3392
|
+
const reader = prior === void 0 ? null : await prior.catch(() => null);
|
|
3393
|
+
return reader !== null && readerIsOf(reader, live) ? reader : null;
|
|
3394
|
+
}
|
|
3395
|
+
/** Keep the prior snapshot's reader when the refresh found its generation, so a count does not drop it. */
|
|
3396
|
+
async adopt(live) {
|
|
3397
|
+
if (this.opened !== void 0 || this.prior === void 0) return;
|
|
3398
|
+
const reader = await this.reuse(live);
|
|
3399
|
+
this.prior = void 0;
|
|
3400
|
+
if (reader !== null && this.opened === void 0) {
|
|
3401
|
+
this.opened = Promise.resolve(reader);
|
|
3402
|
+
this.onReader?.(this.opened);
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
};
|
|
3406
|
+
var SNAPSHOT_BASE_BYTES = 256;
|
|
2712
3407
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
2713
3408
|
var YIELD_EVERY_IDS = 1 << 14;
|
|
2714
3409
|
var CrbmStorageChunkSource = class {
|
|
@@ -2746,6 +3441,10 @@ var CrbmStorageChunkSource = class {
|
|
|
2746
3441
|
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
2747
3442
|
clock: clock ?? { now: () => 0 }
|
|
2748
3443
|
});
|
|
3444
|
+
this.distrusted = new BoundedLru({
|
|
3445
|
+
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
3446
|
+
clock: { now: () => 0 }
|
|
3447
|
+
});
|
|
2749
3448
|
this.replacedPins = new BoundedLru({
|
|
2750
3449
|
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
2751
3450
|
clock: { now: () => 0 }
|
|
@@ -2787,23 +3486,46 @@ var CrbmStorageChunkSource = class {
|
|
|
2787
3486
|
* check again.
|
|
2788
3487
|
*/
|
|
2789
3488
|
invalidations = 0;
|
|
3489
|
+
/**
|
|
3490
|
+
* Generations whose row summary this process found to disagree with their object, by segment and generation. A
|
|
3491
|
+
* summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
|
|
3492
|
+
* long-lived process cannot grow it without limit; one that falls out is trusted again until its next open finds
|
|
3493
|
+
* the disagreement again.
|
|
3494
|
+
*/
|
|
3495
|
+
distrusted;
|
|
2790
3496
|
registry;
|
|
2791
3497
|
keystore;
|
|
2792
3498
|
requireEncryption;
|
|
2793
3499
|
readerOptions;
|
|
2794
3500
|
clock;
|
|
2795
3501
|
currentGenTtlMs;
|
|
2796
|
-
/**
|
|
2797
|
-
|
|
3502
|
+
/**
|
|
3503
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
|
|
3504
|
+
* target, and its reader once a read has asked for one.
|
|
3505
|
+
*/
|
|
3506
|
+
liveSnapshot(ref) {
|
|
2798
3507
|
const key = segmentKey(ref);
|
|
2799
3508
|
const existing = this.snapshots.get(key);
|
|
2800
|
-
if (existing === void 0)
|
|
2801
|
-
|
|
3509
|
+
if (existing === void 0) {
|
|
3510
|
+
return this.install(
|
|
3511
|
+
key,
|
|
3512
|
+
Snapshot.live(this.resolveLive(ref), (live) => this.openLive(ref, live))
|
|
3513
|
+
);
|
|
3514
|
+
}
|
|
3515
|
+
if (!this.expired(existing.installedAtMs)) return existing;
|
|
2802
3516
|
const snap = this.install(
|
|
2803
3517
|
key,
|
|
2804
|
-
|
|
3518
|
+
Snapshot.live(
|
|
3519
|
+
this.refreshedTarget(ref, existing, () => this.retrySoon(snap)),
|
|
3520
|
+
(live) => this.openLive(ref, live),
|
|
3521
|
+
existing.openedReader
|
|
3522
|
+
)
|
|
2805
3523
|
);
|
|
2806
|
-
return snap
|
|
3524
|
+
return snap;
|
|
3525
|
+
}
|
|
3526
|
+
/** The current resolved reader for a segment, refreshing on the TTL. */
|
|
3527
|
+
resolvedReader(ref) {
|
|
3528
|
+
return this.liveSnapshot(ref).reader;
|
|
2807
3529
|
}
|
|
2808
3530
|
/**
|
|
2809
3531
|
* A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
|
|
@@ -2813,45 +3535,56 @@ var CrbmStorageChunkSource = class {
|
|
|
2813
3535
|
retrySoon(snap) {
|
|
2814
3536
|
snap.installedAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
2815
3537
|
}
|
|
2816
|
-
/**
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
3538
|
+
/**
|
|
3539
|
+
* TTL elapsed: cheaply re-resolve the pointer. The row read is always fresh, so what the new snapshot says of the
|
|
3540
|
+
* generation (its summary included) is what the row says now, never the prior snapshot's. The reader is reopened
|
|
3541
|
+
* only if the generation or the row changed, and only when a read asks for it.
|
|
3542
|
+
*/
|
|
3543
|
+
async refreshedTarget(ref, existing, onTransientFault) {
|
|
3544
|
+
const prior = await existing.target.catch(() => null);
|
|
2820
3545
|
try {
|
|
2821
|
-
|
|
3546
|
+
return await this.resolveLive(ref, prior);
|
|
2822
3547
|
} catch (err) {
|
|
2823
3548
|
if (!isTransientError(err)) throw err;
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
if (target === void 0) {
|
|
2827
|
-
if (current === null) return this.resolveLatest(ref);
|
|
3549
|
+
const alive = prior !== null;
|
|
3550
|
+
if (!alive) return this.resolveLive(ref);
|
|
2828
3551
|
onTransientFault();
|
|
2829
|
-
return
|
|
3552
|
+
return prior;
|
|
2830
3553
|
}
|
|
2831
|
-
if (target === null) return null;
|
|
2832
|
-
if (current !== null && target.generation === current.generation && target.lineage === current.lineage) {
|
|
2833
|
-
return current;
|
|
2834
|
-
}
|
|
2835
|
-
return this.openForTarget(ref, target);
|
|
2836
3554
|
}
|
|
2837
3555
|
/**
|
|
2838
|
-
* Memoize a
|
|
2839
|
-
*
|
|
2840
|
-
*
|
|
3556
|
+
* Memoize a snapshot as the segment's at `now()`, forgetting it later if it resolves to null / throws
|
|
3557
|
+
* (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot
|
|
3558
|
+
* **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
|
|
2841
3559
|
*/
|
|
2842
|
-
install(key,
|
|
2843
|
-
|
|
3560
|
+
install(key, snap) {
|
|
3561
|
+
snap.installedAtMs = this.now();
|
|
2844
3562
|
this.snapshots.set(key, snap);
|
|
2845
3563
|
const forgetIfStale = () => {
|
|
2846
3564
|
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
2847
3565
|
};
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
3566
|
+
snap.onReader = (reader) => {
|
|
3567
|
+
reader.then((r) => {
|
|
3568
|
+
if (r === null) {
|
|
3569
|
+
forgetIfStale();
|
|
3570
|
+
return;
|
|
3571
|
+
}
|
|
3572
|
+
if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedBytes);
|
|
3573
|
+
}, forgetIfStale);
|
|
3574
|
+
};
|
|
3575
|
+
if (snap.target !== void 0) {
|
|
3576
|
+
snap.target.then(async (live) => {
|
|
3577
|
+
if (live === null) {
|
|
3578
|
+
forgetIfStale();
|
|
3579
|
+
return;
|
|
3580
|
+
}
|
|
3581
|
+
if (snap.openedReader === void 0 && this.snapshots.peek(key) === snap) {
|
|
3582
|
+
this.snapshots.setWeight(key, live.bytes);
|
|
3583
|
+
}
|
|
3584
|
+
await snap.adopt(live);
|
|
3585
|
+
}, forgetIfStale);
|
|
3586
|
+
}
|
|
3587
|
+
if (snap.openedReader !== void 0) snap.onReader(snap.openedReader);
|
|
2855
3588
|
return snap;
|
|
2856
3589
|
}
|
|
2857
3590
|
now() {
|
|
@@ -2883,12 +3616,12 @@ var CrbmStorageChunkSource = class {
|
|
|
2883
3616
|
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
2884
3617
|
*/
|
|
2885
3618
|
async currentGeneration(ref) {
|
|
2886
|
-
const
|
|
3619
|
+
const snap = this.liveSnapshot(ref);
|
|
2887
3620
|
try {
|
|
2888
|
-
return (await
|
|
3621
|
+
return (await snap.reader)?.generation ?? null;
|
|
2889
3622
|
} catch (err) {
|
|
2890
3623
|
if (!isNotFoundError(err)) throw err;
|
|
2891
|
-
this.dropStale(segmentKey(ref),
|
|
3624
|
+
this.dropStale(segmentKey(ref), snap);
|
|
2892
3625
|
return (await this.resolvedReader(ref))?.generation ?? null;
|
|
2893
3626
|
}
|
|
2894
3627
|
}
|
|
@@ -2908,6 +3641,7 @@ var CrbmStorageChunkSource = class {
|
|
|
2908
3641
|
const key = segmentKey(ref);
|
|
2909
3642
|
this.snapshots.delete(key);
|
|
2910
3643
|
const pinnedPrefix = `${key}@`;
|
|
3644
|
+
this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2911
3645
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2912
3646
|
this.replacedPins.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2913
3647
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
@@ -2928,13 +3662,13 @@ var CrbmStorageChunkSource = class {
|
|
|
2928
3662
|
* almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
|
|
2929
3663
|
*/
|
|
2930
3664
|
async currentVersion(ref) {
|
|
2931
|
-
const
|
|
3665
|
+
const snap = this.liveSnapshot(ref);
|
|
2932
3666
|
let reader;
|
|
2933
3667
|
try {
|
|
2934
|
-
reader = await
|
|
3668
|
+
reader = await snap.reader;
|
|
2935
3669
|
} catch (err) {
|
|
2936
3670
|
if (!isNotFoundError(err)) throw err;
|
|
2937
|
-
this.dropStale(segmentKey(ref),
|
|
3671
|
+
this.dropStale(segmentKey(ref), snap);
|
|
2938
3672
|
reader = await this.resolvedReader(ref);
|
|
2939
3673
|
}
|
|
2940
3674
|
return reader === null ? null : versionOf(reader.generation, reader.lineage);
|
|
@@ -2957,34 +3691,38 @@ var CrbmStorageChunkSource = class {
|
|
|
2957
3691
|
}
|
|
2958
3692
|
}
|
|
2959
3693
|
async pinOnce(ref) {
|
|
2960
|
-
const
|
|
2961
|
-
if (
|
|
3694
|
+
const live = await this.resolveLive(ref);
|
|
3695
|
+
if (live === null) return null;
|
|
3696
|
+
const target = live.target;
|
|
2962
3697
|
const version = versionOf(target.generation, target.lineage);
|
|
2963
3698
|
let reader;
|
|
2964
3699
|
let opened = true;
|
|
2965
3700
|
if (target.lineage === void 0) {
|
|
2966
|
-
reader = await this.
|
|
2967
|
-
this.install(
|
|
3701
|
+
reader = await this.openLive(ref, live);
|
|
3702
|
+
this.install(
|
|
3703
|
+
this.pinnedKey(ref, version, reader.fingerprint),
|
|
3704
|
+
Snapshot.eager(Promise.resolve(reader))
|
|
3705
|
+
);
|
|
2968
3706
|
} else {
|
|
2969
3707
|
const epoch = this.invalidations;
|
|
2970
3708
|
const key = this.pinnedKey(ref, version);
|
|
2971
3709
|
let entry = this.snapshots.get(key);
|
|
2972
3710
|
opened = entry === void 0;
|
|
2973
|
-
entry ??= this.install(key, this.
|
|
3711
|
+
entry ??= this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2974
3712
|
reader = await entry.reader;
|
|
2975
3713
|
if (reader === null) return null;
|
|
2976
3714
|
const replaced = (r) => this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0;
|
|
2977
3715
|
const known = !opened && replaced(reader);
|
|
2978
3716
|
if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
|
|
2979
|
-
this.install(key, Promise.resolve(reader));
|
|
3717
|
+
this.install(key, Snapshot.eager(Promise.resolve(reader)));
|
|
2980
3718
|
}
|
|
2981
3719
|
if (known) {
|
|
2982
3720
|
const current = this.snapshots.peek(key);
|
|
2983
|
-
let fresh = current !== void 0 && current !== entry ? current : this.install(key, this.
|
|
3721
|
+
let fresh = current !== void 0 && current !== entry ? current : this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2984
3722
|
reader = await fresh.reader;
|
|
2985
3723
|
if (reader === null) return null;
|
|
2986
3724
|
if (fresh === current && replaced(reader)) {
|
|
2987
|
-
fresh = this.install(key, this.
|
|
3725
|
+
fresh = this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2988
3726
|
reader = await fresh.reader;
|
|
2989
3727
|
if (reader === null) return null;
|
|
2990
3728
|
}
|
|
@@ -3050,7 +3788,7 @@ var CrbmStorageChunkSource = class {
|
|
|
3050
3788
|
if (entry === void 0) {
|
|
3051
3789
|
if (fingerprint !== void 0 && this.replacedPins.get(this.heldKey(ref, at, fingerprint)) !== void 0)
|
|
3052
3790
|
throw notThePinned(ref, generation);
|
|
3053
|
-
entry = this.install(key, this.openAt(ref, generation));
|
|
3791
|
+
entry = this.install(key, Snapshot.eager(this.openAt(ref, generation)));
|
|
3054
3792
|
}
|
|
3055
3793
|
let reader;
|
|
3056
3794
|
try {
|
|
@@ -3185,58 +3923,94 @@ var CrbmStorageChunkSource = class {
|
|
|
3185
3923
|
return false;
|
|
3186
3924
|
}
|
|
3187
3925
|
/**
|
|
3188
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
3926
|
+
* Evict a snapshot we just failed to read from — matched by the snapshot itself, so a concurrent call's
|
|
3189
3927
|
* fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
|
|
3190
3928
|
* wholesale, so comparing anything else would drop a good snapshot on the floor.
|
|
3191
3929
|
*/
|
|
3192
|
-
dropStale(key,
|
|
3193
|
-
|
|
3194
|
-
if (cur !== void 0 && cur.reader === pending) this.snapshots.delete(key);
|
|
3195
|
-
}
|
|
3196
|
-
async resolveLatest(ref) {
|
|
3197
|
-
const target = await this.resolveTarget(ref);
|
|
3198
|
-
if (target === null) return null;
|
|
3199
|
-
return this.openForTarget(ref, target);
|
|
3930
|
+
dropStale(key, snap) {
|
|
3931
|
+
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
3200
3932
|
}
|
|
3201
3933
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
3202
3934
|
async openForTarget(ref, target) {
|
|
3935
|
+
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
3936
|
+
return this.openGeneration(ref, target, crypto);
|
|
3937
|
+
}
|
|
3938
|
+
openGeneration(ref, target, crypto) {
|
|
3203
3939
|
const genKey = {
|
|
3204
3940
|
namespace: ref.namespace,
|
|
3205
3941
|
segment: ref.segment,
|
|
3206
3942
|
generation: target.generation
|
|
3207
3943
|
};
|
|
3208
|
-
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
3209
3944
|
return openChecked(this.driver, genKey, {
|
|
3210
3945
|
...this.readerOptions,
|
|
3211
3946
|
crypto,
|
|
3212
3947
|
lineage: target.lineage
|
|
3213
3948
|
});
|
|
3214
3949
|
}
|
|
3950
|
+
/**
|
|
3951
|
+
* Open the generation a resolution found, and hold the row's summary against the object while it is open anyway:
|
|
3952
|
+
* the same count and metadata, or the summary is not believed again in this process ({@link distrusted}). This
|
|
3953
|
+
* costs no request; a mismatch is not an error, since the object is the truth and this read has it.
|
|
3954
|
+
*/
|
|
3955
|
+
async openLive(ref, live) {
|
|
3956
|
+
const reader = await this.openGeneration(ref, live.target, await live.crypto());
|
|
3957
|
+
const summary = await live.summary();
|
|
3958
|
+
if (summary !== void 0 && !summaryAgrees(summary, describe(reader))) {
|
|
3959
|
+
this.distrusted.set(this.distrustKey(ref, live.target.generation, live.target.lineage), true);
|
|
3960
|
+
}
|
|
3961
|
+
return reader;
|
|
3962
|
+
}
|
|
3963
|
+
distrustKey(ref, generation, lineage) {
|
|
3964
|
+
return `${segmentKey(ref)}@${generation}:${String(lineage)}`;
|
|
3965
|
+
}
|
|
3966
|
+
/** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
|
|
3967
|
+
async trustedSummary(ref, live) {
|
|
3968
|
+
if (this.distrusted.peek(this.distrustKey(ref, live.target.generation, live.target.lineage)) !== void 0) {
|
|
3969
|
+
return void 0;
|
|
3970
|
+
}
|
|
3971
|
+
return live.summary();
|
|
3972
|
+
}
|
|
3215
3973
|
/**
|
|
3216
3974
|
* The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
|
|
3217
3975
|
* `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
|
|
3218
|
-
* — there's nowhere a wrapped DEK could live).
|
|
3976
|
+
* — there's nowhere a wrapped DEK could live). The row's summary rides along, to be used or not by whoever
|
|
3977
|
+
* asks.
|
|
3219
3978
|
*/
|
|
3220
|
-
async
|
|
3979
|
+
async resolveLive(ref, prior) {
|
|
3221
3980
|
if (this.registry !== void 0) {
|
|
3222
3981
|
const record = await this.registry.get(ref);
|
|
3223
3982
|
if (record === null) return null;
|
|
3224
3983
|
if (record.status === "destroyed") return null;
|
|
3225
3984
|
if (record.currentGen === null) return null;
|
|
3985
|
+
const generation = record.currentGen;
|
|
3986
|
+
const keys = record.wrappedDeks;
|
|
3987
|
+
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
3988
|
+
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
3226
3989
|
return {
|
|
3227
|
-
generation: record.
|
|
3228
|
-
|
|
3229
|
-
|
|
3990
|
+
target: { generation, lineage: record.token, wrappedDeks: record.wrappedDeks },
|
|
3991
|
+
crypto,
|
|
3992
|
+
unwrap,
|
|
3993
|
+
// The same checks as a read of the object makes (`requireEncryption`, a keystore for an encrypted row)
|
|
3994
|
+
// come first, then the row's summary, used only if it names this generation in the shape the keys call for.
|
|
3995
|
+
summary: once(async () => usableSummary(ref, record, (await crypto())?.aead)),
|
|
3996
|
+
bytes: SNAPSHOT_BASE_BYTES + (record.summary === void 0 ? 0 : JSON.stringify(record.summary).length)
|
|
3230
3997
|
};
|
|
3231
3998
|
}
|
|
3232
3999
|
let maxGen = -1;
|
|
3233
4000
|
for await (const key of this.driver.list(ref)) {
|
|
3234
4001
|
if (key.generation > maxGen) maxGen = key.generation;
|
|
3235
4002
|
}
|
|
3236
|
-
|
|
4003
|
+
if (maxGen < 0) return null;
|
|
4004
|
+
return {
|
|
4005
|
+
target: { generation: maxGen },
|
|
4006
|
+
crypto: () => Promise.resolve(void 0),
|
|
4007
|
+
unwrap: void 0,
|
|
4008
|
+
summary: () => Promise.resolve(void 0),
|
|
4009
|
+
bytes: SNAPSHOT_BASE_BYTES
|
|
4010
|
+
};
|
|
3237
4011
|
}
|
|
3238
4012
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
3239
|
-
async cryptoForRead(ref, generation, wrappedDeks) {
|
|
4013
|
+
async cryptoForRead(ref, generation, wrappedDeks, unwrap) {
|
|
3240
4014
|
if (wrappedDeks === void 0 || wrappedDeks.length === 0) {
|
|
3241
4015
|
if (this.requireEncryption) {
|
|
3242
4016
|
throw new KeyUnavailableError(
|
|
@@ -3245,12 +4019,13 @@ var CrbmStorageChunkSource = class {
|
|
|
3245
4019
|
}
|
|
3246
4020
|
return void 0;
|
|
3247
4021
|
}
|
|
3248
|
-
|
|
4022
|
+
const keystore = this.keystore;
|
|
4023
|
+
if (keystore === void 0) {
|
|
3249
4024
|
throw new KeyUnavailableError(
|
|
3250
4025
|
`segment "${ref.segment}" is encrypted but this CrbmStorageChunkSource has no keystore`
|
|
3251
4026
|
);
|
|
3252
4027
|
}
|
|
3253
|
-
const aead = await
|
|
4028
|
+
const aead = await (unwrap ?? (() => keystore.openDek(wrappedDeks)))();
|
|
3254
4029
|
return { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
3255
4030
|
}
|
|
3256
4031
|
async getChunk(ref) {
|
|
@@ -3277,6 +4052,46 @@ var CrbmStorageChunkSource = class {
|
|
|
3277
4052
|
null
|
|
3278
4053
|
);
|
|
3279
4054
|
}
|
|
4055
|
+
/**
|
|
4056
|
+
* The current generation's number, id count and metadata. The answer comes from the row the snapshot resolved when
|
|
4057
|
+
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
4058
|
+
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
4059
|
+
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
4060
|
+
* With no summary it can use (an older row, one for another generation, one that does not open), it opens the
|
|
4061
|
+
* generation and answers from the object. A row that names an object that is gone still answers, while a read
|
|
4062
|
+
* of the object throws: the number is true of the generation the row names.
|
|
4063
|
+
*
|
|
4064
|
+
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
4065
|
+
* believed until the next time the object is opened, which holds it against the summary.
|
|
4066
|
+
*/
|
|
4067
|
+
async summary(ref) {
|
|
4068
|
+
validateUserRef(ref);
|
|
4069
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
4070
|
+
const snap = this.liveSnapshot(ref);
|
|
4071
|
+
try {
|
|
4072
|
+
const live = await snap.target;
|
|
4073
|
+
if (live === null) return null;
|
|
4074
|
+
const row = await this.trustedSummary(ref, live);
|
|
4075
|
+
if (row !== void 0) return summaryOfGeneration(live.target.generation, row);
|
|
4076
|
+
const reader = await snap.reader;
|
|
4077
|
+
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
4078
|
+
} catch (err) {
|
|
4079
|
+
if (attempt === 1) throw err;
|
|
4080
|
+
const checks = isNotFoundError(err) || isIntegrityError(err) || isValidationError(err);
|
|
4081
|
+
if (!checks || !isNotFoundError(err) && !await this.replacedUnder(ref, snap.reader, err)) {
|
|
4082
|
+
throw err;
|
|
4083
|
+
}
|
|
4084
|
+
this.dropStale(segmentKey(ref), snap);
|
|
4085
|
+
}
|
|
4086
|
+
}
|
|
4087
|
+
return null;
|
|
4088
|
+
}
|
|
4089
|
+
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
4090
|
+
async summaryAt(ref, generation, held) {
|
|
4091
|
+
validateUserRef(ref);
|
|
4092
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
4093
|
+
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
4094
|
+
}
|
|
3280
4095
|
/**
|
|
3281
4096
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
3282
4097
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -3305,40 +4120,143 @@ var CrbmStorageChunkSource = class {
|
|
|
3305
4120
|
*/
|
|
3306
4121
|
async withFreshSnapshot(ref, read, ifGone) {
|
|
3307
4122
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
3308
|
-
const
|
|
4123
|
+
const snap = this.liveSnapshot(ref);
|
|
4124
|
+
const pending = snap.reader;
|
|
3309
4125
|
try {
|
|
3310
4126
|
const reader = await pending;
|
|
3311
4127
|
if (reader === null) return ifGone;
|
|
3312
4128
|
return await read(reader);
|
|
3313
4129
|
} catch (err) {
|
|
3314
|
-
if (
|
|
3315
|
-
this.
|
|
4130
|
+
if (attempt === 1) throw err;
|
|
4131
|
+
if (!isNotFoundError(err) && !await this.replacedUnder(ref, pending, err)) throw err;
|
|
4132
|
+
this.dropStale(segmentKey(ref), snap);
|
|
3316
4133
|
}
|
|
3317
4134
|
}
|
|
3318
4135
|
return ifGone;
|
|
3319
4136
|
}
|
|
4137
|
+
/**
|
|
4138
|
+
* Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
|
|
4139
|
+
* reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
|
|
4140
|
+
* generations above a rolled-back pointer that held the id, a collection pass can stop part-way, and a load then
|
|
4141
|
+
* numbers `currentGen + 1` again), so a reader that still holds the old object's index reads the new object's
|
|
4142
|
+
* bytes at the old offsets: a chunk checksum fails, or a range runs past the end. The object's footer says which it
|
|
4143
|
+
* was, as it does for a pin ({@link CrbmReader.sameObject}): replaced, or gone, re-resolves; the same object means
|
|
4144
|
+
* the error is real, and it stands. Only these two errors pay for the footer read, never a read that succeeded. A
|
|
4145
|
+
* transient fault in the check is rethrown; a footer that cannot be read says nothing, and the error stands.
|
|
4146
|
+
*/
|
|
4147
|
+
async replacedUnder(ref, pending, err) {
|
|
4148
|
+
if (!(isIntegrityError(err) || isValidationError(err))) return false;
|
|
4149
|
+
let reader;
|
|
4150
|
+
try {
|
|
4151
|
+
reader = await pending;
|
|
4152
|
+
} catch {
|
|
4153
|
+
return false;
|
|
4154
|
+
}
|
|
4155
|
+
if (reader === null) return false;
|
|
4156
|
+
const at = {
|
|
4157
|
+
namespace: ref.namespace,
|
|
4158
|
+
segment: ref.segment,
|
|
4159
|
+
generation: reader.generation
|
|
4160
|
+
};
|
|
4161
|
+
try {
|
|
4162
|
+
return !await CrbmReader.sameObject(storageBlobReader(this.driver, at), reader.fingerprint);
|
|
4163
|
+
} catch (checkErr) {
|
|
4164
|
+
if (isNotFoundError(checkErr)) return true;
|
|
4165
|
+
if (isTransientError(checkErr)) throw checkErr;
|
|
4166
|
+
return false;
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
3320
4169
|
};
|
|
3321
|
-
function
|
|
3322
|
-
const
|
|
4170
|
+
function* encodeEach(chunks) {
|
|
4171
|
+
for (const { chunkKey, bitmap } of chunks) {
|
|
4172
|
+
if (bitmap.isEmpty) continue;
|
|
4173
|
+
bitmap.optimize?.();
|
|
4174
|
+
yield { chunkKey, payload: bitmap.serialize(), cardinality: bitmap.size };
|
|
4175
|
+
}
|
|
4176
|
+
}
|
|
4177
|
+
async function writeEncodedChunks(driver, key, chunks, options) {
|
|
3323
4178
|
const tick = yieldEvery(options.clock);
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
4179
|
+
let chunkCount = 0;
|
|
4180
|
+
let cardinality = 0;
|
|
4181
|
+
let identity;
|
|
4182
|
+
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
4183
|
+
const writer = new CrbmWriter(sink, {
|
|
4184
|
+
generation: key.generation,
|
|
4185
|
+
crypto: options.crypto,
|
|
4186
|
+
metadata: options.metadata
|
|
4187
|
+
});
|
|
4188
|
+
for (const chunk of chunks) {
|
|
4189
|
+
await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
|
|
4190
|
+
chunkCount++;
|
|
4191
|
+
cardinality += chunk.cardinality;
|
|
3330
4192
|
const pause = tick();
|
|
3331
4193
|
if (pause !== null) await pause;
|
|
3332
4194
|
}
|
|
3333
4195
|
await writer.finish();
|
|
4196
|
+
identity = writer.identity;
|
|
3334
4197
|
});
|
|
4198
|
+
if (identity === void 0) {
|
|
4199
|
+
throw new IntegrityError(
|
|
4200
|
+
`the driver committed ${key.segment}.${key.generation} without writing it`
|
|
4201
|
+
);
|
|
4202
|
+
}
|
|
4203
|
+
return {
|
|
4204
|
+
size,
|
|
4205
|
+
sha256,
|
|
4206
|
+
chunkCount,
|
|
4207
|
+
cardinality,
|
|
4208
|
+
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
4209
|
+
};
|
|
4210
|
+
}
|
|
4211
|
+
async function holdsObject(storage, key, fingerprint) {
|
|
4212
|
+
try {
|
|
4213
|
+
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
4214
|
+
} catch {
|
|
4215
|
+
return false;
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
function objectIsEncrypted(storage, key) {
|
|
4219
|
+
return footerSaysEncrypted(storageBlobReader(storage, key));
|
|
4220
|
+
}
|
|
4221
|
+
function rememberingTail(blob) {
|
|
4222
|
+
let first;
|
|
4223
|
+
return {
|
|
4224
|
+
getRange: (offset, length) => blob.getRange(offset, length),
|
|
4225
|
+
async getTail(maxBytes) {
|
|
4226
|
+
if (first !== void 0 && (maxBytes <= first.bytes.length || first.bytes.length === first.size)) {
|
|
4227
|
+
const take = Math.min(maxBytes, first.bytes.length);
|
|
4228
|
+
return { bytes: first.bytes.subarray(first.bytes.length - take), size: first.size };
|
|
4229
|
+
}
|
|
4230
|
+
const read = await blob.getTail(maxBytes);
|
|
4231
|
+
first ??= read;
|
|
4232
|
+
return read;
|
|
4233
|
+
}
|
|
4234
|
+
};
|
|
4235
|
+
}
|
|
4236
|
+
async function openRollbackTarget(storage, key, crypto) {
|
|
4237
|
+
const blob = rememberingTail(storageBlobReader(storage, key));
|
|
4238
|
+
await blob.getTail(DEFAULT_TAIL_BYTES);
|
|
4239
|
+
const encrypted = await footerSaysEncrypted(blob);
|
|
4240
|
+
if (encrypted !== (crypto !== void 0)) return { encrypted, reader: void 0 };
|
|
4241
|
+
const reader = await CrbmReader.open(blob, { crypto });
|
|
4242
|
+
if (reader.generation !== key.generation) {
|
|
4243
|
+
throw new IntegrityError(
|
|
4244
|
+
`segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
|
|
4245
|
+
);
|
|
4246
|
+
}
|
|
4247
|
+
return { encrypted, reader };
|
|
3335
4248
|
}
|
|
3336
4249
|
async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
3337
4250
|
const chunkKeys = [];
|
|
3338
4251
|
let cardinality = 0;
|
|
3339
4252
|
const tick = yieldEvery(options.clock);
|
|
4253
|
+
let identity;
|
|
3340
4254
|
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
3341
|
-
const writer = new CrbmWriter(sink, {
|
|
4255
|
+
const writer = new CrbmWriter(sink, {
|
|
4256
|
+
generation: key.generation,
|
|
4257
|
+
crypto: options.crypto,
|
|
4258
|
+
metadata: options.metadata
|
|
4259
|
+
});
|
|
3342
4260
|
for await (const { chunkKey, bitmap } of chunks) {
|
|
3343
4261
|
if (bitmap.isEmpty) continue;
|
|
3344
4262
|
bitmap.optimize?.();
|
|
@@ -3349,12 +4267,57 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
|
3349
4267
|
if (pause !== null) await pause;
|
|
3350
4268
|
}
|
|
3351
4269
|
await writer.finish();
|
|
4270
|
+
identity = writer.identity;
|
|
3352
4271
|
});
|
|
3353
|
-
|
|
4272
|
+
if (identity === void 0) {
|
|
4273
|
+
throw new IntegrityError(
|
|
4274
|
+
`the driver committed ${key.segment}.${key.generation} without writing it`
|
|
4275
|
+
);
|
|
4276
|
+
}
|
|
4277
|
+
return {
|
|
4278
|
+
size,
|
|
4279
|
+
sha256,
|
|
4280
|
+
chunkKeys,
|
|
4281
|
+
cardinality,
|
|
4282
|
+
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
4283
|
+
};
|
|
4284
|
+
}
|
|
4285
|
+
function refuseCleartextOntoKey(key, record, cleartext) {
|
|
4286
|
+
if (cleartext !== true || record.wrappedDeks === void 0 || record.wrappedDeks.length === 0) {
|
|
4287
|
+
return;
|
|
4288
|
+
}
|
|
4289
|
+
throw new KeyUnavailableError(
|
|
4290
|
+
`segment "${key.segment}" is encrypted, and generation ${key.generation} was written in cleartext: another writer created the segment with a key while this one was writing. Nothing was published. Re-run the write with the keystore that holds the segment's key.`
|
|
4291
|
+
);
|
|
3354
4292
|
}
|
|
4293
|
+
var UNANSWERED_RESENDS = 3;
|
|
4294
|
+
var UNANSWERED_RESEND_BASE_MS = 500;
|
|
3355
4295
|
async function publishGeneration(registry, key, options = {}) {
|
|
4296
|
+
let fresh;
|
|
4297
|
+
let failedOn;
|
|
4298
|
+
let unanswered;
|
|
4299
|
+
let resends = 0;
|
|
4300
|
+
let sawUnanswered = false;
|
|
4301
|
+
let lastRecord;
|
|
3356
4302
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
3357
|
-
const
|
|
4303
|
+
const reused = attempt === 0 && options.row !== void 0;
|
|
4304
|
+
const record = fresh !== void 0 ? fresh : reused ? options.row ?? null : await registry.get(key);
|
|
4305
|
+
fresh = void 0;
|
|
4306
|
+
unanswered = void 0;
|
|
4307
|
+
lastRecord = record;
|
|
4308
|
+
if (failedOn !== void 0) {
|
|
4309
|
+
let landed;
|
|
4310
|
+
try {
|
|
4311
|
+
landed = await landedHere(record, failedOn, key, options.holdsOwnObject);
|
|
4312
|
+
} catch (proofErr) {
|
|
4313
|
+
throw outcomeUnknown(key, proofErr);
|
|
4314
|
+
}
|
|
4315
|
+
if (landed) return true;
|
|
4316
|
+
if (record !== null && record.status === "active" && record.currentGen === key.generation) {
|
|
4317
|
+
return false;
|
|
4318
|
+
}
|
|
4319
|
+
failedOn = void 0;
|
|
4320
|
+
}
|
|
3358
4321
|
try {
|
|
3359
4322
|
if (options.expectFrom !== void 0 && record?.currentGen !== options.expectFrom) {
|
|
3360
4323
|
return false;
|
|
@@ -3365,103 +4328,138 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
3365
4328
|
if (options.expectToken !== void 0 && record?.token !== options.expectToken) {
|
|
3366
4329
|
return false;
|
|
3367
4330
|
}
|
|
4331
|
+
if (record === null && options.summary !== void 0 && "sealed" in options.summary && options.wrappedDeks === void 0) {
|
|
4332
|
+
if (reused) continue;
|
|
4333
|
+
return false;
|
|
4334
|
+
}
|
|
3368
4335
|
if (record === null) {
|
|
3369
4336
|
await registry.create(key, {
|
|
3370
4337
|
currentGen: key.generation,
|
|
3371
|
-
wrappedDeks: options.wrappedDeks
|
|
4338
|
+
wrappedDeks: options.wrappedDeks,
|
|
4339
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
3372
4340
|
});
|
|
3373
4341
|
} else if (record.status === "destroyed") {
|
|
3374
4342
|
throw new ValidationError(
|
|
3375
4343
|
`segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
|
|
3376
4344
|
);
|
|
3377
4345
|
} else if (record.currentGen === null) {
|
|
4346
|
+
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
3378
4347
|
await registry.compareAndSwap(key, record.token, {
|
|
3379
4348
|
currentGen: key.generation,
|
|
3380
|
-
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks }
|
|
4349
|
+
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks },
|
|
4350
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
3381
4351
|
});
|
|
3382
4352
|
} else if (record.currentGen > key.generation) {
|
|
3383
4353
|
return false;
|
|
3384
4354
|
} else if (record.currentGen === key.generation) {
|
|
4355
|
+
if (reused) continue;
|
|
3385
4356
|
return true;
|
|
3386
4357
|
} else {
|
|
3387
4358
|
if (options.wrappedDeks !== void 0 && options.wrappedDeks.length > 0) {
|
|
3388
4359
|
throw new ValidationError(
|
|
3389
|
-
`publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen}
|
|
4360
|
+
`publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen}: its generations are not encrypted under this key, and nothing was published. If another writer published to the segment while this one was writing, re-run the write, which then uses the segment's own key; if the segment's generations are cleartext, encrypt a new segment and load into that instead.`
|
|
3390
4361
|
);
|
|
3391
4362
|
}
|
|
3392
|
-
|
|
4363
|
+
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
4364
|
+
await registry.compareAndSwap(key, record.token, {
|
|
4365
|
+
currentGen: key.generation,
|
|
4366
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
4367
|
+
});
|
|
3393
4368
|
}
|
|
3394
4369
|
return true;
|
|
3395
4370
|
} catch (err) {
|
|
3396
|
-
if (isWriteConflictError(err))
|
|
3397
|
-
|
|
4371
|
+
if (isWriteConflictError(err)) {
|
|
4372
|
+
failedOn = record;
|
|
4373
|
+
continue;
|
|
4374
|
+
}
|
|
4375
|
+
if (!isTransientError(err)) throw err;
|
|
4376
|
+
options.onUnanswered?.();
|
|
4377
|
+
sawUnanswered = true;
|
|
4378
|
+
let now;
|
|
4379
|
+
try {
|
|
4380
|
+
now = await registry.get(key);
|
|
4381
|
+
} catch (readErr) {
|
|
4382
|
+
if (isIntegrityError(readErr)) throw readErr;
|
|
4383
|
+
throw outcomeUnknown(key, err);
|
|
4384
|
+
}
|
|
4385
|
+
if (sameRow(now, record)) {
|
|
4386
|
+
const sleep = options.clock?.sleep;
|
|
4387
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS) throw outcomeUnknown(key, err);
|
|
4388
|
+
const bound = UNANSWERED_RESEND_BASE_MS * 2 ** resends;
|
|
4389
|
+
resends += 1;
|
|
4390
|
+
await sleep.call(options.clock, Math.floor((options.rng?.next() ?? 1) * bound));
|
|
4391
|
+
unanswered = outcomeUnknown(key, err);
|
|
4392
|
+
fresh = now;
|
|
4393
|
+
continue;
|
|
4394
|
+
}
|
|
4395
|
+
failedOn = record;
|
|
4396
|
+
fresh = now;
|
|
3398
4397
|
}
|
|
3399
4398
|
}
|
|
3400
4399
|
const final = await registry.get(key);
|
|
3401
|
-
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation)
|
|
3402
|
-
|
|
4400
|
+
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation) {
|
|
4401
|
+
if (final.currentGen !== key.generation) return false;
|
|
4402
|
+
if (!sawUnanswered) return true;
|
|
4403
|
+
try {
|
|
4404
|
+
return await landedHere(final, lastRecord ?? null, key, options.holdsOwnObject);
|
|
4405
|
+
} catch (proofErr) {
|
|
4406
|
+
throw outcomeUnknown(key, proofErr);
|
|
4407
|
+
}
|
|
4408
|
+
}
|
|
4409
|
+
if (unanswered !== void 0) throw unanswered;
|
|
3403
4410
|
throw new WriteConflictError(
|
|
3404
4411
|
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
3405
4412
|
);
|
|
3406
4413
|
}
|
|
4414
|
+
async function landedHere(now, before, key, holdsOwnObject) {
|
|
4415
|
+
if (now === null || now.status === "destroyed" || now.currentGen !== key.generation) return false;
|
|
4416
|
+
if (before !== null && !sameIncarnation(before, now)) return false;
|
|
4417
|
+
if (holdsOwnObject === void 0) {
|
|
4418
|
+
throw new TransientError(
|
|
4419
|
+
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled, since the caller gave no way to tell its own object from another's. Its object is kept; re-run the write, which numbers past it.`
|
|
4420
|
+
);
|
|
4421
|
+
}
|
|
4422
|
+
return holdsOwnObject();
|
|
4423
|
+
}
|
|
4424
|
+
function sameRow(a, b) {
|
|
4425
|
+
return a === null ? b === null : b !== null && a.token === b.token;
|
|
4426
|
+
}
|
|
4427
|
+
function outcomeUnknown(key, cause) {
|
|
4428
|
+
if (isTransientError(cause)) return cause;
|
|
4429
|
+
return new TransientError(
|
|
4430
|
+
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled (${cause?.name ?? "error"} while checking its object). Its object is kept, since the write may still land; re-run the write, which numbers past it.`,
|
|
4431
|
+
{ cause }
|
|
4432
|
+
);
|
|
4433
|
+
}
|
|
4434
|
+
async function provesOwnObject(storage, key, fingerprint) {
|
|
4435
|
+
try {
|
|
4436
|
+
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
4437
|
+
} catch (err) {
|
|
4438
|
+
if (isNotFoundError(err)) return false;
|
|
4439
|
+
throw err;
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
3407
4442
|
async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
3408
4443
|
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
3409
4444
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
3410
4445
|
}
|
|
3411
4446
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
for (const [chunkKey, rems] of pendingByChunk) {
|
|
3419
|
-
if (rems.length === 0) continue;
|
|
3420
|
-
const existingBitmap = byChunk.get(chunkKey);
|
|
3421
|
-
if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
|
|
3422
|
-
else existingBitmap.addMany(rems);
|
|
3423
|
-
rems.length = 0;
|
|
3424
|
-
const pause = tickChunk();
|
|
3425
|
-
if (pause !== null) await pause;
|
|
3426
|
-
}
|
|
3427
|
-
pending = 0;
|
|
3428
|
-
};
|
|
3429
|
-
const ingest = (id) => {
|
|
3430
|
-
const { chunkKey, remainder } = splitId(id);
|
|
3431
|
-
let bucket = pendingByChunk.get(chunkKey);
|
|
3432
|
-
if (bucket === void 0) {
|
|
3433
|
-
bucket = [];
|
|
3434
|
-
pendingByChunk.set(chunkKey, bucket);
|
|
3435
|
-
}
|
|
3436
|
-
bucket.push(remainder);
|
|
3437
|
-
return ++pending >= BULK_FLUSH_IDS;
|
|
3438
|
-
};
|
|
3439
|
-
if (Symbol.asyncIterator in ids) {
|
|
3440
|
-
for await (const id of ids) {
|
|
3441
|
-
if (ingest(id)) await flushPending();
|
|
3442
|
-
const pause = tickId();
|
|
3443
|
-
if (pause !== null) await pause;
|
|
3444
|
-
}
|
|
4447
|
+
let chunks;
|
|
4448
|
+
if (ids instanceof DecodedLoadInput && ids.bitmap.encodeChunks !== void 0) {
|
|
4449
|
+
const pause = yieldEvery(options.clock, 1);
|
|
4450
|
+
await pause();
|
|
4451
|
+
chunks = ids.bitmap.encodeChunks();
|
|
4452
|
+
await pause();
|
|
3445
4453
|
} else {
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
if (pause !== null) await pause;
|
|
3450
|
-
}
|
|
3451
|
-
}
|
|
3452
|
-
await flushPending();
|
|
3453
|
-
let cardinality = 0;
|
|
3454
|
-
const chunks = [];
|
|
3455
|
-
for (const [chunkKey, bitmap] of byChunk) {
|
|
3456
|
-
cardinality += bitmap.size;
|
|
3457
|
-
chunks.push({ chunkKey, bitmap });
|
|
3458
|
-
const pause = tickChunk();
|
|
3459
|
-
if (pause !== null) await pause;
|
|
4454
|
+
chunks = encodeEach(
|
|
4455
|
+
await bucketIds(ids instanceof DecodedLoadInput ? ids.bitmap : ids, codec, options.clock)
|
|
4456
|
+
);
|
|
3460
4457
|
}
|
|
3461
4458
|
if (options.keystore !== void 0 && options.registry === void 0) {
|
|
3462
4459
|
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
3463
4460
|
}
|
|
3464
|
-
const
|
|
4461
|
+
const passed = options.row;
|
|
4462
|
+
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(key) : null;
|
|
3465
4463
|
if (existing?.status === "destroyed") {
|
|
3466
4464
|
throw new ValidationError(
|
|
3467
4465
|
`segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
@@ -3490,13 +4488,25 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
3490
4488
|
crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
3491
4489
|
}
|
|
3492
4490
|
}
|
|
3493
|
-
const { size, sha256 } = await
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
4491
|
+
const { size, sha256, chunkCount, cardinality, fingerprint } = await writeEncodedChunks(
|
|
4492
|
+
driver,
|
|
4493
|
+
key,
|
|
4494
|
+
chunks,
|
|
4495
|
+
{ crypto, clock: options.clock, metadata: options.metadata }
|
|
4496
|
+
);
|
|
4497
|
+
const summary = summaryOf(
|
|
4498
|
+
key,
|
|
4499
|
+
key.generation,
|
|
4500
|
+
{ cardinality, metadata: options.metadata },
|
|
4501
|
+
crypto?.aead
|
|
4502
|
+
);
|
|
3497
4503
|
if (options.registry !== void 0 && options.publish !== false) {
|
|
3498
4504
|
const becameCurrent = await publishGeneration(options.registry, key, {
|
|
3499
|
-
wrappedDeks: newWrapped
|
|
4505
|
+
wrappedDeks: newWrapped,
|
|
4506
|
+
summary,
|
|
4507
|
+
cleartext: crypto === void 0,
|
|
4508
|
+
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
4509
|
+
clock: options.clock
|
|
3500
4510
|
});
|
|
3501
4511
|
if (becameCurrent) {
|
|
3502
4512
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -3506,16 +4516,72 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
3506
4516
|
generation: key.generation
|
|
3507
4517
|
});
|
|
3508
4518
|
}
|
|
3509
|
-
return {
|
|
3510
|
-
size,
|
|
3511
|
-
sha256,
|
|
3512
|
-
chunkCount
|
|
3513
|
-
cardinality,
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
4519
|
+
return {
|
|
4520
|
+
size,
|
|
4521
|
+
sha256,
|
|
4522
|
+
chunkCount,
|
|
4523
|
+
cardinality,
|
|
4524
|
+
summary,
|
|
4525
|
+
becameCurrent,
|
|
4526
|
+
wrappedDeks: newWrapped,
|
|
4527
|
+
encrypted: crypto !== void 0,
|
|
4528
|
+
fingerprint
|
|
4529
|
+
};
|
|
4530
|
+
}
|
|
4531
|
+
return {
|
|
4532
|
+
size,
|
|
4533
|
+
sha256,
|
|
4534
|
+
chunkCount,
|
|
4535
|
+
cardinality,
|
|
4536
|
+
summary,
|
|
4537
|
+
wrappedDeks: newWrapped,
|
|
4538
|
+
encrypted: crypto !== void 0,
|
|
4539
|
+
fingerprint
|
|
4540
|
+
};
|
|
4541
|
+
}
|
|
4542
|
+
async function bucketIds(ids, codec, clock) {
|
|
4543
|
+
const byChunk = /* @__PURE__ */ new Map();
|
|
4544
|
+
const pendingByChunk = /* @__PURE__ */ new Map();
|
|
4545
|
+
let pending = 0;
|
|
4546
|
+
const tickChunk = yieldEvery(clock);
|
|
4547
|
+
const tickId = yieldEvery(clock, YIELD_EVERY_IDS);
|
|
4548
|
+
const flushPending = async () => {
|
|
4549
|
+
for (const [chunkKey, rems] of pendingByChunk) {
|
|
4550
|
+
if (rems.length === 0) continue;
|
|
4551
|
+
const existingBitmap = byChunk.get(chunkKey);
|
|
4552
|
+
if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
|
|
4553
|
+
else existingBitmap.addMany(rems);
|
|
4554
|
+
rems.length = 0;
|
|
4555
|
+
const pause = tickChunk();
|
|
4556
|
+
if (pause !== null) await pause;
|
|
4557
|
+
}
|
|
4558
|
+
pending = 0;
|
|
4559
|
+
};
|
|
4560
|
+
const ingest = (id) => {
|
|
4561
|
+
const { chunkKey, remainder } = splitId(id);
|
|
4562
|
+
let bucket = pendingByChunk.get(chunkKey);
|
|
4563
|
+
if (bucket === void 0) {
|
|
4564
|
+
bucket = [];
|
|
4565
|
+
pendingByChunk.set(chunkKey, bucket);
|
|
4566
|
+
}
|
|
4567
|
+
bucket.push(remainder);
|
|
4568
|
+
return ++pending >= BULK_FLUSH_IDS;
|
|
4569
|
+
};
|
|
4570
|
+
if (Symbol.asyncIterator in ids) {
|
|
4571
|
+
for await (const id of ids) {
|
|
4572
|
+
if (ingest(id)) await flushPending();
|
|
4573
|
+
const pause = tickId();
|
|
4574
|
+
if (pause !== null) await pause;
|
|
4575
|
+
}
|
|
4576
|
+
} else {
|
|
4577
|
+
for (const id of ids) {
|
|
4578
|
+
if (ingest(id)) await flushPending();
|
|
4579
|
+
const pause = tickId();
|
|
4580
|
+
if (pause !== null) await pause;
|
|
4581
|
+
}
|
|
3517
4582
|
}
|
|
3518
|
-
|
|
4583
|
+
await flushPending();
|
|
4584
|
+
return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
|
|
3519
4585
|
}
|
|
3520
4586
|
function openGenerationReader(storage, key, crypto, options = {}) {
|
|
3521
4587
|
return openChecked(storage, key, { ...options, crypto });
|
|
@@ -3544,6 +4610,11 @@ async function verifyGeneration(storage, key, expected, crypto) {
|
|
|
3544
4610
|
`verify failed for ${key.segment}.${key.generation}: cardinality ${reader.count()} != ${expected.cardinality}`
|
|
3545
4611
|
);
|
|
3546
4612
|
}
|
|
4613
|
+
if (!summaryAgrees(expected, { cardinality: reader.count(), metadata: reader.metadata })) {
|
|
4614
|
+
throw new IntegrityError(
|
|
4615
|
+
`verify failed for ${key.segment}.${key.generation}: the object's metadata is not the metadata it was written with`
|
|
4616
|
+
);
|
|
4617
|
+
}
|
|
3547
4618
|
}
|
|
3548
4619
|
|
|
3549
4620
|
// src/core/pinned-storage-source.ts
|
|
@@ -3592,6 +4663,11 @@ var PinnedStorageChunkSource = class {
|
|
|
3592
4663
|
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
3593
4664
|
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
3594
4665
|
}
|
|
4666
|
+
summary(ref) {
|
|
4667
|
+
const pin = this.pinFor(ref);
|
|
4668
|
+
if (pin === void 0) return this.inner.summary(ref);
|
|
4669
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
4670
|
+
}
|
|
3595
4671
|
currentGeneration(ref) {
|
|
3596
4672
|
const pin = this.pinFor(ref);
|
|
3597
4673
|
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
@@ -3628,67 +4704,58 @@ var PinnedStorageChunkSource = class {
|
|
|
3628
4704
|
}
|
|
3629
4705
|
};
|
|
3630
4706
|
|
|
3631
|
-
// src/core/
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
4707
|
+
// src/core/retry.ts
|
|
4708
|
+
var DEFAULT_RETRY_POLICY = {
|
|
4709
|
+
maxAttempts: 4,
|
|
4710
|
+
baseDelayMs: 50,
|
|
4711
|
+
maxDelayMs: 2e3,
|
|
4712
|
+
backoffFactor: 2,
|
|
4713
|
+
jitter: "full"
|
|
4714
|
+
};
|
|
4715
|
+
function backoffDelayMs(policy, attempt) {
|
|
4716
|
+
const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
|
|
4717
|
+
return Math.min(policy.maxDelayMs, raw);
|
|
3639
4718
|
}
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
const
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
if (after === null)
|
|
3653
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
3654
|
-
if (record.status === "destroyed" && after.token !== record.token) {
|
|
3655
|
-
throw new WriteConflictError(
|
|
3656
|
-
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
4719
|
+
function applyJitter(policy, delayMs, rng) {
|
|
4720
|
+
if (policy.jitter === "none") return delayMs;
|
|
4721
|
+
return rng.next() * delayMs;
|
|
4722
|
+
}
|
|
4723
|
+
function retryRead(op, retry) {
|
|
4724
|
+
return retry === void 0 ? op() : withRetry(op, retry.policy ?? DEFAULT_RETRY_POLICY, retry);
|
|
4725
|
+
}
|
|
4726
|
+
async function withRetry(op, policy, deps) {
|
|
4727
|
+
const retryable = deps.isRetryable ?? isTransientError;
|
|
4728
|
+
if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
|
|
4729
|
+
throw new ValidationError(
|
|
4730
|
+
`retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
|
|
3657
4731
|
);
|
|
3658
4732
|
}
|
|
3659
|
-
const
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
}
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
4733
|
+
const attempts = Math.floor(policy.maxAttempts);
|
|
4734
|
+
let lastErr;
|
|
4735
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
4736
|
+
try {
|
|
4737
|
+
return await op();
|
|
4738
|
+
} catch (err) {
|
|
4739
|
+
lastErr = err;
|
|
4740
|
+
if (attempt >= attempts || !retryable(err)) throw err;
|
|
4741
|
+
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
4742
|
+
try {
|
|
4743
|
+
deps.onRetry?.({ attempt, delayMs, err });
|
|
4744
|
+
} catch {
|
|
4745
|
+
}
|
|
4746
|
+
await deps.clock.sleep(delayMs);
|
|
3670
4747
|
}
|
|
3671
|
-
};
|
|
3672
|
-
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
3673
|
-
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
3674
|
-
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
3675
|
-
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
3676
|
-
[]
|
|
3677
|
-
) : (
|
|
3678
|
-
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
3679
|
-
gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
|
|
3680
|
-
);
|
|
3681
|
-
for (const generation of toDelete) {
|
|
3682
|
-
await stillCollectable();
|
|
3683
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
3684
4748
|
}
|
|
3685
|
-
|
|
4749
|
+
throw lastErr;
|
|
3686
4750
|
}
|
|
3687
4751
|
|
|
3688
4752
|
// src/core/load.ts
|
|
3689
4753
|
async function currentCardinality(ref, deps, record) {
|
|
3690
|
-
if (record === null || record.currentGen === null || record.status === "destroyed")
|
|
4754
|
+
if (record === null || record.currentGen === null || record.status === "destroyed") {
|
|
4755
|
+
return { cardinality: null, fromSummary: false };
|
|
4756
|
+
}
|
|
3691
4757
|
const generation = record.currentGen;
|
|
4758
|
+
let aead;
|
|
3692
4759
|
let crypto;
|
|
3693
4760
|
const wrapped = record.wrappedDeks;
|
|
3694
4761
|
if (wrapped !== void 0 && wrapped.length > 0) {
|
|
@@ -3697,23 +4764,29 @@ async function currentCardinality(ref, deps, record) {
|
|
|
3697
4764
|
`segment "${ref.segment}" is encrypted but load was given no keystore`
|
|
3698
4765
|
);
|
|
3699
4766
|
}
|
|
3700
|
-
|
|
3701
|
-
|
|
4767
|
+
aead = await deps.keystore.openDek(wrapped);
|
|
4768
|
+
const opened = aead;
|
|
4769
|
+
crypto = { aead: opened, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
3702
4770
|
}
|
|
4771
|
+
const described = usableSummary(ref, record, aead);
|
|
4772
|
+
if (described !== void 0) return { cardinality: described.cardinality, fromSummary: true };
|
|
3703
4773
|
let reader;
|
|
3704
4774
|
try {
|
|
3705
|
-
reader = await
|
|
4775
|
+
reader = await retryRead(
|
|
4776
|
+
() => openGenerationReader(deps.storage, { ...ref, generation }, crypto),
|
|
4777
|
+
deps.readRetry
|
|
4778
|
+
);
|
|
3706
4779
|
} catch (err) {
|
|
3707
4780
|
if (!isNotFoundError(err)) throw err;
|
|
3708
|
-
return null;
|
|
4781
|
+
return { cardinality: null, fromSummary: false };
|
|
3709
4782
|
}
|
|
3710
4783
|
let total = 0;
|
|
3711
4784
|
for (const n of reader.cardinalities().values()) total += n;
|
|
3712
|
-
return total;
|
|
4785
|
+
return { cardinality: total, fromSummary: false };
|
|
3713
4786
|
}
|
|
3714
|
-
async function loadSegment(ref,
|
|
4787
|
+
async function loadSegment(ref, input, deps, options = {}) {
|
|
3715
4788
|
validateUserRef(ref);
|
|
3716
|
-
requireCodec(deps.codec, "loadSegment");
|
|
4789
|
+
const codec = requireCodec(deps.codec, "loadSegment");
|
|
3717
4790
|
const keep = options.keep ?? 1;
|
|
3718
4791
|
if (!Number.isInteger(keep) || keep < 0) {
|
|
3719
4792
|
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
@@ -3733,13 +4806,19 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3733
4806
|
throw new ValidationError(`guard.minRetained must be a fraction in 0..1; got ${String(s)}`);
|
|
3734
4807
|
}
|
|
3735
4808
|
}
|
|
4809
|
+
const metadata = copiedMetadata(options.metadata, (message) => {
|
|
4810
|
+
throw new ValidationError(message);
|
|
4811
|
+
});
|
|
3736
4812
|
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
4813
|
+
const ids = prepareLoadInput(input, codec);
|
|
4814
|
+
assertRegistryCanWrite(deps.registry, "load");
|
|
3737
4815
|
const row = await deps.registry.get(ref);
|
|
3738
4816
|
const fromToken = row?.token;
|
|
3739
4817
|
const fromGeneration = row?.currentGen ?? void 0;
|
|
3740
4818
|
const needsBefore = guard?.minRetained !== void 0 || options.allowEmpty !== true;
|
|
3741
|
-
const
|
|
3742
|
-
const
|
|
4819
|
+
const current = needsBefore ? await currentCardinality(ref, deps, row) : { cardinality: null, fromSummary: false };
|
|
4820
|
+
const before = current.cardinality;
|
|
4821
|
+
const { generation, checked } = await nextLoadGeneration(ref, deps, row);
|
|
3743
4822
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3744
4823
|
let written;
|
|
3745
4824
|
try {
|
|
@@ -3749,7 +4828,9 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3749
4828
|
keystore: deps.keystore,
|
|
3750
4829
|
requireEncryption: deps.requireEncryption,
|
|
3751
4830
|
codec: deps.codec,
|
|
3752
|
-
clock: deps.clock
|
|
4831
|
+
clock: deps.clock,
|
|
4832
|
+
metadata,
|
|
4833
|
+
row
|
|
3753
4834
|
});
|
|
3754
4835
|
} catch (err) {
|
|
3755
4836
|
if (!isWriteConflictError(err)) throw err;
|
|
@@ -3773,16 +4854,28 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3773
4854
|
collected: []
|
|
3774
4855
|
};
|
|
3775
4856
|
}
|
|
3776
|
-
const
|
|
4857
|
+
const reclaim = async () => {
|
|
3777
4858
|
const now = await deps.registry.get(ref);
|
|
3778
|
-
if (now
|
|
4859
|
+
if (now !== null && (now.token === fromToken || now.status === "destroyed")) {
|
|
4860
|
+
await deps.storage.delete(key);
|
|
4861
|
+
return;
|
|
4862
|
+
}
|
|
4863
|
+
const keyed = now !== null && now.wrappedDeks !== void 0 && now.wrappedDeks.length > 0;
|
|
4864
|
+
if ((now === null || keyed && !written.encrypted) && await holdsObject(deps.storage, key, written.fingerprint)) {
|
|
4865
|
+
await deps.storage.delete(key);
|
|
4866
|
+
}
|
|
4867
|
+
};
|
|
4868
|
+
let unanswered = false;
|
|
4869
|
+
const refuse = async (reason) => {
|
|
4870
|
+
await reclaim();
|
|
3779
4871
|
audit.onEvent({
|
|
3780
4872
|
kind: "segment.load-refused",
|
|
3781
4873
|
segment: ref.segment,
|
|
3782
4874
|
namespace: ref.namespace,
|
|
3783
4875
|
generation,
|
|
3784
4876
|
reason,
|
|
3785
|
-
cardinality: written.cardinality
|
|
4877
|
+
cardinality: written.cardinality,
|
|
4878
|
+
...unanswered ? { unanswered: true } : {}
|
|
3786
4879
|
});
|
|
3787
4880
|
return {
|
|
3788
4881
|
generation,
|
|
@@ -3805,19 +4898,37 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3805
4898
|
if (guard?.minRetained !== void 0 && before !== null) {
|
|
3806
4899
|
if (written.cardinality < before * guard.minRetained) return refuse("min-retained");
|
|
3807
4900
|
}
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
4901
|
+
let published;
|
|
4902
|
+
try {
|
|
4903
|
+
published = await publishGeneration(deps.registry, key, {
|
|
4904
|
+
row,
|
|
4905
|
+
wrappedDeks: written.wrappedDeks,
|
|
4906
|
+
summary: written.summary,
|
|
4907
|
+
cleartext: !written.encrypted,
|
|
4908
|
+
...fromToken === void 0 ? {} : { expectToken: fromToken },
|
|
4909
|
+
...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
|
|
4910
|
+
// The third case, and the one the two fences above structurally cannot cover: the guard judged a segment
|
|
4911
|
+
// that had NO ROW. Both `expectFrom` and `expectToken` compare against a value read from a row, so with no
|
|
4912
|
+
// row both are omitted and the publish becomes a bare forward-only advance — which lands over anything
|
|
4913
|
+
// that appeared in between. `before` was `null`, so the empty and `minRetained` bounds had nothing to
|
|
4914
|
+
// judge and passed vacuously. Verified: an empty generation published over a thousand ids that a
|
|
4915
|
+
// concurrent writer had created meanwhile, reporting success. A guarded write therefore has to fence on
|
|
4916
|
+
// the ABSENCE it relied on, exactly as it fences on the pointer it relied on.
|
|
4917
|
+
...needsBefore && row === null ? { expectAbsent: true } : {},
|
|
4918
|
+
// A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
|
|
4919
|
+
// is this load's publish only over the object this load wrote, which one footer read proves.
|
|
4920
|
+
holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
|
|
4921
|
+
// And the wait before a fresh write, when the first left the row as it was.
|
|
4922
|
+
clock: deps.clock,
|
|
4923
|
+
rng: deps.rng ?? deps.readRetry?.rng,
|
|
4924
|
+
onUnanswered: () => {
|
|
4925
|
+
unanswered = true;
|
|
4926
|
+
}
|
|
4927
|
+
});
|
|
4928
|
+
} catch (err) {
|
|
4929
|
+
if (isValidationError(err) || err instanceof KeyUnavailableError) await reclaim();
|
|
4930
|
+
throw err;
|
|
4931
|
+
}
|
|
3821
4932
|
if (!published) return refuse("superseded");
|
|
3822
4933
|
audit.onEvent({
|
|
3823
4934
|
kind: "segment.publish",
|
|
@@ -3825,7 +4936,13 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3825
4936
|
namespace: ref.namespace,
|
|
3826
4937
|
generation
|
|
3827
4938
|
});
|
|
3828
|
-
const
|
|
4939
|
+
const currentObjectGone = needsBefore && before === null;
|
|
4940
|
+
const collected = await collectAfterLoad(ref, deps, {
|
|
4941
|
+
generation,
|
|
4942
|
+
keep,
|
|
4943
|
+
byName: checked && !currentObjectGone && deps.collectByListing !== true,
|
|
4944
|
+
...current.fromSummary && fromGeneration !== void 0 ? { proveCurrent: { ...ref, generation: fromGeneration } } : {}
|
|
4945
|
+
});
|
|
3829
4946
|
return {
|
|
3830
4947
|
generation,
|
|
3831
4948
|
published: true,
|
|
@@ -3845,7 +4962,25 @@ async function listGenerations(ref, deps) {
|
|
|
3845
4962
|
const current = record?.currentGen ?? null;
|
|
3846
4963
|
const seen = /* @__PURE__ */ new Set();
|
|
3847
4964
|
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
3848
|
-
|
|
4965
|
+
const described = record === null ? void 0 : await currentDescription(ref, record, deps);
|
|
4966
|
+
return [...seen].sort((a, b) => a - b).map(
|
|
4967
|
+
(generation) => generation === current ? { generation, current: true, ...described } : { generation, current: false }
|
|
4968
|
+
);
|
|
4969
|
+
}
|
|
4970
|
+
async function currentDescription(ref, record, deps) {
|
|
4971
|
+
if (record.summary === void 0) return void 0;
|
|
4972
|
+
let aead;
|
|
4973
|
+
if ("sealed" in record.summary && record.wrappedDeks !== void 0 && deps.keystore !== void 0) {
|
|
4974
|
+
try {
|
|
4975
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
4976
|
+
} catch {
|
|
4977
|
+
return void 0;
|
|
4978
|
+
}
|
|
4979
|
+
}
|
|
4980
|
+
const usable = usableSummary(ref, record, aead);
|
|
4981
|
+
if (usable === void 0) return void 0;
|
|
4982
|
+
const metadata = usable.metadata;
|
|
4983
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { cardinality: usable.cardinality } : { cardinality: usable.cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
3849
4984
|
}
|
|
3850
4985
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
3851
4986
|
validateUserRef(ref);
|
|
@@ -3879,8 +5014,34 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3879
5014
|
`rollback: generation ${toGeneration} of "${ref.segment}" is above the current pointer (${record.currentGen}) \u2014 it may never have been published. Pass { allowForward: true } if you are undoing an earlier rollback.`
|
|
3880
5015
|
);
|
|
3881
5016
|
}
|
|
5017
|
+
const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
5018
|
+
let aead;
|
|
5019
|
+
if (keyed && deps.keystore !== void 0) {
|
|
5020
|
+
try {
|
|
5021
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
5022
|
+
} catch {
|
|
5023
|
+
}
|
|
5024
|
+
}
|
|
5025
|
+
const crypto = aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, toGeneration, scope) };
|
|
5026
|
+
const { encrypted, reader } = await openRollbackTarget(
|
|
5027
|
+
deps.storage,
|
|
5028
|
+
{ ...ref, generation: toGeneration },
|
|
5029
|
+
crypto
|
|
5030
|
+
);
|
|
5031
|
+
if (encrypted !== keyed) {
|
|
5032
|
+
throw new IntegrityError(
|
|
5033
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" is ${encrypted ? "encrypted" : "cleartext"}, but the segment is ${keyed ? "encrypted" : "cleartext"}, so no read of the segment could use it; it was never one of its generations. Roll back to another generation.`
|
|
5034
|
+
);
|
|
5035
|
+
}
|
|
5036
|
+
const summary = reader === void 0 ? void 0 : summaryOf(
|
|
5037
|
+
ref,
|
|
5038
|
+
toGeneration,
|
|
5039
|
+
{ cardinality: reader.count(), metadata: reader.metadata },
|
|
5040
|
+
aead
|
|
5041
|
+
);
|
|
3882
5042
|
const { token } = await deps.registry.compareAndSwap(ref, record.token, {
|
|
3883
|
-
currentGen: toGeneration
|
|
5043
|
+
currentGen: toGeneration,
|
|
5044
|
+
summary
|
|
3884
5045
|
});
|
|
3885
5046
|
let stillThere = false;
|
|
3886
5047
|
for await (const key of deps.storage.list(ref)) {
|
|
@@ -3889,15 +5050,31 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3889
5050
|
break;
|
|
3890
5051
|
}
|
|
3891
5052
|
}
|
|
3892
|
-
|
|
5053
|
+
let replaced = false;
|
|
5054
|
+
if (stillThere && reader !== void 0 && toGeneration > (record.currentGen ?? -1)) {
|
|
5055
|
+
try {
|
|
5056
|
+
replaced = !await provesOwnObject(
|
|
5057
|
+
deps.storage,
|
|
5058
|
+
{ ...ref, generation: toGeneration },
|
|
5059
|
+
reader.fingerprint
|
|
5060
|
+
);
|
|
5061
|
+
} catch (err) {
|
|
5062
|
+
if (!isIntegrityError(err)) throw err;
|
|
5063
|
+
replaced = true;
|
|
5064
|
+
}
|
|
5065
|
+
}
|
|
5066
|
+
if (!stillThere || replaced) {
|
|
3893
5067
|
let undone = false;
|
|
3894
5068
|
try {
|
|
3895
|
-
await deps.registry.compareAndSwap(ref, token, {
|
|
5069
|
+
await deps.registry.compareAndSwap(ref, token, {
|
|
5070
|
+
currentGen: record.currentGen,
|
|
5071
|
+
summary: describingSummary(record)
|
|
5072
|
+
});
|
|
3896
5073
|
undone = true;
|
|
3897
5074
|
} catch {
|
|
3898
5075
|
}
|
|
3899
5076
|
throw new NotFoundError(
|
|
3900
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" was collected while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
5077
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
3901
5078
|
);
|
|
3902
5079
|
}
|
|
3903
5080
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -3909,6 +5086,12 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3909
5086
|
});
|
|
3910
5087
|
return { fromGeneration: record.currentGen, generation: toGeneration };
|
|
3911
5088
|
}
|
|
5089
|
+
function describingSummary(record) {
|
|
5090
|
+
const { summary } = record;
|
|
5091
|
+
if (summary === void 0 || summary.generation !== record.currentGen) return void 0;
|
|
5092
|
+
const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
5093
|
+
return "sealed" in summary === keyed ? summary : void 0;
|
|
5094
|
+
}
|
|
3912
5095
|
|
|
3913
5096
|
// src/core/erase-id.ts
|
|
3914
5097
|
var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
|
|
@@ -3918,6 +5101,7 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3918
5101
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
3919
5102
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
|
|
3920
5103
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
5104
|
+
const read = (op) => retryRead(op, deps.readRetry);
|
|
3921
5105
|
const record = await deps.registry.get(ref);
|
|
3922
5106
|
if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
|
|
3923
5107
|
if (record.status === "destroyed")
|
|
@@ -3954,13 +5138,18 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3954
5138
|
return row.currentGen === from && row.token === fromToken ? null : "superseded";
|
|
3955
5139
|
};
|
|
3956
5140
|
const holds = async (generation2) => {
|
|
5141
|
+
const key2 = { ...base, generation: generation2 };
|
|
5142
|
+
const chunkIn = (crypto) => read(async () => (await openGenerationReader(deps.storage, key2, crypto)).getChunk(chunkKey));
|
|
3957
5143
|
try {
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
5144
|
+
let bytes;
|
|
5145
|
+
try {
|
|
5146
|
+
bytes = await chunkIn(cryptoAt(generation2));
|
|
5147
|
+
} catch (err) {
|
|
5148
|
+
if (!isIntegrityError(err) || cryptoAt(generation2) === void 0 || await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
5149
|
+
throw err;
|
|
5150
|
+
}
|
|
5151
|
+
bytes = await chunkIn(void 0);
|
|
5152
|
+
}
|
|
3964
5153
|
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
3965
5154
|
} catch (err) {
|
|
3966
5155
|
if (isNotFoundError(err)) return null;
|
|
@@ -4030,27 +5219,40 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
4030
5219
|
const stage = async () => {
|
|
4031
5220
|
let written;
|
|
4032
5221
|
try {
|
|
4033
|
-
const reader = await openGenerationReader(deps.storage, fromKey, cryptoAt(from));
|
|
4034
|
-
const bytes = await reader.getChunk(chunkKey);
|
|
5222
|
+
const reader = await read(() => openGenerationReader(deps.storage, fromKey, cryptoAt(from)));
|
|
5223
|
+
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
4035
5224
|
if (bytes === null) return await notInCurrent();
|
|
4036
5225
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
4037
5226
|
assertRemaindersInRange(target, chunkKey);
|
|
4038
5227
|
if (!target.has(remainder)) return await notInCurrent();
|
|
4039
5228
|
target.remove(remainder);
|
|
5229
|
+
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
4040
5230
|
const generation2 = await nextGeneration(ref, deps);
|
|
4041
5231
|
const key2 = { ...base, generation: generation2 };
|
|
5232
|
+
const metadata = metadataToCarry(
|
|
5233
|
+
reader.metadata,
|
|
5234
|
+
aead === void 0 ? void 0 : usableSummary(ref, record, aead)
|
|
5235
|
+
);
|
|
4042
5236
|
const tally = await writeCrbmGenerationStream(
|
|
4043
5237
|
deps.storage,
|
|
4044
5238
|
key2,
|
|
4045
|
-
rewrite(reader, chunkKey, target, codec, maxBytes),
|
|
4046
|
-
{ crypto: cryptoAt(generation2), clock: deps.clock }
|
|
5239
|
+
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
5240
|
+
{ crypto: cryptoAt(generation2), clock: deps.clock, metadata }
|
|
4047
5241
|
);
|
|
4048
5242
|
written = generation2;
|
|
4049
5243
|
const beforeVerify = await deps.registry.get(ref);
|
|
4050
5244
|
const early = rowVerdict(beforeVerify);
|
|
4051
5245
|
if (early !== null) return refused(early, written);
|
|
4052
|
-
await
|
|
4053
|
-
|
|
5246
|
+
await read(
|
|
5247
|
+
() => verifyGeneration(deps.storage, key2, { ...tally, metadata }, cryptoAt(generation2))
|
|
5248
|
+
);
|
|
5249
|
+
const summary2 = summaryOf(
|
|
5250
|
+
ref,
|
|
5251
|
+
generation2,
|
|
5252
|
+
{ cardinality: tally.cardinality, metadata },
|
|
5253
|
+
aead
|
|
5254
|
+
);
|
|
5255
|
+
return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
|
|
4054
5256
|
} catch (err) {
|
|
4055
5257
|
if (!isNotFoundError(err)) throw err;
|
|
4056
5258
|
let now;
|
|
@@ -4075,10 +5277,16 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
4075
5277
|
if (!staged.erased && staged.generation !== void 0) await discardRefused(staged.generation);
|
|
4076
5278
|
return staged;
|
|
4077
5279
|
}
|
|
4078
|
-
const { generation, key } = staged;
|
|
5280
|
+
const { generation, key, fingerprint, summary } = staged;
|
|
4079
5281
|
const published = await publishGeneration(deps.registry, key, {
|
|
4080
5282
|
expectFrom: from,
|
|
4081
|
-
expectToken: fromToken
|
|
5283
|
+
expectToken: fromToken,
|
|
5284
|
+
summary,
|
|
5285
|
+
// A write that ends without an answer is settled by reading the row, and a pointer at this number is this
|
|
5286
|
+
// rewrite's only over the object it wrote: the footer proves it, so another incarnation's cannot pass for it.
|
|
5287
|
+
holdsOwnObject: () => provesOwnObject(deps.storage, key, fingerprint),
|
|
5288
|
+
clock: deps.clock,
|
|
5289
|
+
rng: deps.rng ?? deps.readRetry?.rng
|
|
4082
5290
|
});
|
|
4083
5291
|
if (!published) {
|
|
4084
5292
|
await discardRefused(generation);
|
|
@@ -4103,14 +5311,14 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
4103
5311
|
if (left !== void 0) throw cannotRemove(left);
|
|
4104
5312
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
4105
5313
|
}
|
|
4106
|
-
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes) {
|
|
5314
|
+
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
4107
5315
|
const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
4108
5316
|
for (const k of keys) {
|
|
4109
5317
|
if (k === chunkKey) {
|
|
4110
5318
|
yield { chunkKey: k, bitmap: replacement };
|
|
4111
5319
|
continue;
|
|
4112
5320
|
}
|
|
4113
|
-
const bytes = await reader.getChunk(k);
|
|
5321
|
+
const bytes = await read(() => reader.getChunk(k));
|
|
4114
5322
|
if (bytes === null) continue;
|
|
4115
5323
|
const bitmap = codec.safeDeserialize(bytes, maxBytes);
|
|
4116
5324
|
assertRemaindersInRange(bitmap, k);
|
|
@@ -4129,8 +5337,8 @@ function assertRemaindersInRange(bitmap, chunkKey) {
|
|
|
4129
5337
|
// src/drivers/crypto.ts
|
|
4130
5338
|
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
4131
5339
|
var KEY_BYTES = 32;
|
|
4132
|
-
var
|
|
4133
|
-
var
|
|
5340
|
+
var NONCE_BYTES2 = 12;
|
|
5341
|
+
var TAG_BYTES2 = 16;
|
|
4134
5342
|
var NodeAead = class {
|
|
4135
5343
|
key;
|
|
4136
5344
|
constructor(key) {
|
|
@@ -4139,14 +5347,14 @@ var NodeAead = class {
|
|
|
4139
5347
|
this.key = Buffer.from(key);
|
|
4140
5348
|
}
|
|
4141
5349
|
seal(plaintext, aad) {
|
|
4142
|
-
const nonce = randomBytes(
|
|
5350
|
+
const nonce = randomBytes(NONCE_BYTES2);
|
|
4143
5351
|
const cipher = createCipheriv("aes-256-gcm", this.key, nonce);
|
|
4144
5352
|
cipher.setAAD(aad);
|
|
4145
5353
|
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
4146
5354
|
return { nonce, ciphertext, tag: cipher.getAuthTag() };
|
|
4147
5355
|
}
|
|
4148
5356
|
open(sealed, aad) {
|
|
4149
|
-
if (sealed.nonce.length !==
|
|
5357
|
+
if (sealed.nonce.length !== NONCE_BYTES2 || sealed.tag.length !== TAG_BYTES2)
|
|
4150
5358
|
throw new IntegrityError("AEAD nonce/tag have the wrong length");
|
|
4151
5359
|
const decipher = createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
|
|
4152
5360
|
decipher.setAAD(aad);
|
|
@@ -4166,11 +5374,11 @@ function wrapKey(dek, kek, keyId) {
|
|
|
4166
5374
|
}
|
|
4167
5375
|
function unwrapKey(wrapped, kek, keyId) {
|
|
4168
5376
|
const blob = Buffer.from(wrapped, "base64");
|
|
4169
|
-
if (blob.length <
|
|
5377
|
+
if (blob.length < NONCE_BYTES2 + TAG_BYTES2)
|
|
4170
5378
|
throw new IntegrityError("wrapped DEK is too short to be valid");
|
|
4171
|
-
const nonce = blob.subarray(0,
|
|
4172
|
-
const tag = blob.subarray(blob.length -
|
|
4173
|
-
const ciphertext = blob.subarray(
|
|
5379
|
+
const nonce = blob.subarray(0, NONCE_BYTES2);
|
|
5380
|
+
const tag = blob.subarray(blob.length - TAG_BYTES2);
|
|
5381
|
+
const ciphertext = blob.subarray(NONCE_BYTES2, blob.length - TAG_BYTES2);
|
|
4174
5382
|
return new NodeAead(kek).open({ nonce, ciphertext, tag }, new TextEncoder().encode(keyId));
|
|
4175
5383
|
}
|
|
4176
5384
|
var InProcessKeystore = class {
|
|
@@ -4411,7 +5619,10 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
4411
5619
|
status: "destroyed",
|
|
4412
5620
|
wrappedDeks: void 0,
|
|
4413
5621
|
// ← the crypto-shred: the only copy of the DEK wrappings is gone
|
|
4414
|
-
keyId: void 0
|
|
5622
|
+
keyId: void 0,
|
|
5623
|
+
// The current generation's cached count and metadata go with it: sealed, they cannot be opened without
|
|
5624
|
+
// the wrappings; clear, they would outlive the segment on its tombstone.
|
|
5625
|
+
summary: void 0
|
|
4415
5626
|
});
|
|
4416
5627
|
return { ...base, destroyed: true, cryptoShredded: encrypted };
|
|
4417
5628
|
} catch (err) {
|
|
@@ -4550,39 +5761,108 @@ function assertNotDestroyed(record, ref) {
|
|
|
4550
5761
|
|
|
4551
5762
|
// src/core/retention-sweep.ts
|
|
4552
5763
|
var DEFAULT_RETIRE_LIMIT = 100;
|
|
5764
|
+
var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
4553
5765
|
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
4554
5766
|
var DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
4555
5767
|
async function rowsFromDueIndex(registry, options) {
|
|
4556
5768
|
const rows = [];
|
|
4557
|
-
const
|
|
5769
|
+
const pointers = /* @__PURE__ */ new Map();
|
|
5770
|
+
const litter = [];
|
|
5771
|
+
const gone = /* @__PURE__ */ new Set();
|
|
4558
5772
|
for (const bucket of dueBucketsAt(options.now, options.lookbackBuckets)) {
|
|
4559
5773
|
for await (const pointer of registry.list(dueNamespace(bucket))) {
|
|
4560
5774
|
const ref = decodeDueName(pointer.segment);
|
|
4561
5775
|
if (ref === null) continue;
|
|
4562
5776
|
if (isReservedNamespace(ref.namespace)) continue;
|
|
4563
5777
|
if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
|
|
4564
|
-
const
|
|
4565
|
-
|
|
4566
|
-
|
|
5778
|
+
const found = {
|
|
5779
|
+
ref: { namespace: pointer.namespace, segment: pointer.segment },
|
|
5780
|
+
target: ref,
|
|
5781
|
+
token: pointer.token
|
|
5782
|
+
};
|
|
5783
|
+
const key = segmentKey(ref);
|
|
5784
|
+
const known = pointers.get(key);
|
|
5785
|
+
if (known !== void 0) {
|
|
5786
|
+
known.push(found);
|
|
5787
|
+
continue;
|
|
5788
|
+
}
|
|
5789
|
+
if (gone.has(key)) {
|
|
5790
|
+
litter.push(found);
|
|
5791
|
+
continue;
|
|
5792
|
+
}
|
|
4567
5793
|
if (rows.length >= options.maxScanSegments) {
|
|
4568
5794
|
throw new BudgetExceededError(
|
|
4569
5795
|
`retireExpired: the due index yielded more than ${options.maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Raise \`maxScanSegments\`, narrow with \`namespace\`, or reduce \`lookbackBuckets\`.`
|
|
4570
5796
|
);
|
|
4571
5797
|
}
|
|
4572
5798
|
const live = await registry.get(ref);
|
|
4573
|
-
if (live === null)
|
|
5799
|
+
if (live === null) {
|
|
5800
|
+
gone.add(key);
|
|
5801
|
+
litter.push(found);
|
|
5802
|
+
continue;
|
|
5803
|
+
}
|
|
4574
5804
|
rows.push(live);
|
|
5805
|
+
pointers.set(key, [found]);
|
|
4575
5806
|
}
|
|
4576
5807
|
}
|
|
4577
|
-
return rows;
|
|
5808
|
+
return { rows, pointers, litter };
|
|
5809
|
+
}
|
|
5810
|
+
async function rowsFromFleet(registry, options) {
|
|
5811
|
+
const held = [];
|
|
5812
|
+
const rows = await drainRegistry(registry, {
|
|
5813
|
+
namespace: options.namespace,
|
|
5814
|
+
maxScanSegments: options.maxScanSegments,
|
|
5815
|
+
op: "retireExpired",
|
|
5816
|
+
onReserved: (pointer) => {
|
|
5817
|
+
if (held.length >= options.maxScanSegments) return;
|
|
5818
|
+
const target = decodeDueName(pointer.segment);
|
|
5819
|
+
if (target === null || isReservedNamespace(target.namespace)) return;
|
|
5820
|
+
held.push({
|
|
5821
|
+
ref: { namespace: pointer.namespace, segment: pointer.segment },
|
|
5822
|
+
target,
|
|
5823
|
+
token: pointer.token
|
|
5824
|
+
});
|
|
5825
|
+
}
|
|
5826
|
+
});
|
|
5827
|
+
const pointers = /* @__PURE__ */ new Map();
|
|
5828
|
+
const litter = [];
|
|
5829
|
+
if (held.length === 0) return { rows, pointers, litter };
|
|
5830
|
+
const present = new Set(rows.map((rec) => segmentKey(rec)));
|
|
5831
|
+
for (const found of held) {
|
|
5832
|
+
const key = segmentKey(found.target);
|
|
5833
|
+
if (!present.has(key)) litter.push(found);
|
|
5834
|
+
else if (pointers.has(key)) pointers.get(key).push(found);
|
|
5835
|
+
else pointers.set(key, [found]);
|
|
5836
|
+
}
|
|
5837
|
+
return { rows, pointers, litter };
|
|
4578
5838
|
}
|
|
4579
|
-
async function forgetDuePointer(registry, ref, expiresAt) {
|
|
5839
|
+
async function forgetDuePointer(registry, ref, expiresAt, onFault) {
|
|
4580
5840
|
if (!canIndex(ref)) return;
|
|
4581
5841
|
try {
|
|
4582
5842
|
await registry.delete(dueIndexRef(dueBucket(expiresAt), ref));
|
|
5843
|
+
} catch (err) {
|
|
5844
|
+
onFault(err);
|
|
5845
|
+
}
|
|
5846
|
+
}
|
|
5847
|
+
async function filePurgePointer(registry, ref, purgeAt) {
|
|
5848
|
+
if (!canIndex(ref)) return;
|
|
5849
|
+
try {
|
|
5850
|
+
await registry.create(dueIndexRef(dueBucket(purgeAt), ref), { currentGen: null });
|
|
4583
5851
|
} catch {
|
|
4584
5852
|
}
|
|
4585
5853
|
}
|
|
5854
|
+
async function forgetPurgePointers(registry, ref, purgeAt, found, onFault) {
|
|
5855
|
+
if (!canIndex(ref)) return;
|
|
5856
|
+
const filed = dueIndexRef(dueBucket(purgeAt), ref);
|
|
5857
|
+
const all = found.some((p) => p.ref.namespace === filed.namespace) ? found : [...found, { ref: filed, target: ref }];
|
|
5858
|
+
for (const pointer of all) {
|
|
5859
|
+
try {
|
|
5860
|
+
await registry.delete(pointer.ref, pointer.token);
|
|
5861
|
+
} catch (err) {
|
|
5862
|
+
onFault(err);
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
}
|
|
4586
5866
|
async function retireExpired(deps, options) {
|
|
4587
5867
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
4588
5868
|
const now = options.now;
|
|
@@ -4615,32 +5895,40 @@ async function retireExpired(deps, options) {
|
|
|
4615
5895
|
}
|
|
4616
5896
|
const dryRun = options.dryRun === true;
|
|
4617
5897
|
const scan = options.scan ?? "fleet";
|
|
4618
|
-
const
|
|
5898
|
+
const scanned = scan === "index" ? await rowsFromDueIndex(deps.registry, {
|
|
4619
5899
|
now,
|
|
4620
5900
|
lookbackBuckets: options.lookbackBuckets ?? DEFAULT_LOOKBACK_BUCKETS,
|
|
4621
5901
|
namespace: options.namespace,
|
|
4622
5902
|
maxScanSegments
|
|
4623
|
-
}) : await
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
op: "retireExpired"
|
|
4627
|
-
});
|
|
5903
|
+
}) : await rowsFromFleet(deps.registry, { namespace: options.namespace, maxScanSegments });
|
|
5904
|
+
const rows = scanned.rows;
|
|
5905
|
+
const removesRows = deps.registry.capabilities().conditionalDelete === true;
|
|
4628
5906
|
const shards = options.shards;
|
|
4629
5907
|
const totalShards = options.totalShards ?? 0;
|
|
4630
|
-
const
|
|
5908
|
+
const owned = (ref) => shards === void 0 || totalShards <= 1 || shards.includes(shardOf(segmentKey(ref), totalShards));
|
|
5909
|
+
const mine = rows.filter(owned);
|
|
4631
5910
|
const entries = [];
|
|
4632
5911
|
let eligible = 0;
|
|
4633
5912
|
let retired = 0;
|
|
4634
5913
|
let wouldRetire = 0;
|
|
4635
5914
|
let tombstonesPurged = 0;
|
|
4636
5915
|
let limited = false;
|
|
5916
|
+
let purgeFaults = 0;
|
|
5917
|
+
let firstPurgeFault;
|
|
5918
|
+
const noteFault = (err) => {
|
|
5919
|
+
if (isWriteConflictError(err)) return;
|
|
5920
|
+
purgeFaults += 1;
|
|
5921
|
+
firstPurgeFault ??= failureReason(err);
|
|
5922
|
+
};
|
|
5923
|
+
let purgeRun = 0;
|
|
5924
|
+
let purging = true;
|
|
4637
5925
|
let attempted = 0;
|
|
4638
5926
|
for (const rec of mine) {
|
|
4639
5927
|
const ref = { namespace: rec.namespace, segment: rec.segment };
|
|
4640
5928
|
const base = { segment: rec.segment, namespace: rec.namespace };
|
|
4641
5929
|
const policy = readRetentionPolicy(rec.retention);
|
|
4642
5930
|
if (rec.status === "destroyed") {
|
|
4643
|
-
if (!purgeTombstones) continue;
|
|
5931
|
+
if (!purgeTombstones || !purging) continue;
|
|
4644
5932
|
const retiredAt = retirementStamp(rec.retention);
|
|
4645
5933
|
if (retiredAt === null) continue;
|
|
4646
5934
|
if (now - retiredAt < grace) continue;
|
|
@@ -4657,13 +5945,31 @@ async function retireExpired(deps, options) {
|
|
|
4657
5945
|
continue;
|
|
4658
5946
|
}
|
|
4659
5947
|
}
|
|
4660
|
-
if (!dryRun)
|
|
4661
|
-
|
|
5948
|
+
if (!dryRun) {
|
|
5949
|
+
await deps.registry.delete(ref, rec.token);
|
|
5950
|
+
tombstonesPurged += 1;
|
|
5951
|
+
purgeRun = 0;
|
|
5952
|
+
if (removesRows) {
|
|
5953
|
+
await forgetPurgePointers(
|
|
5954
|
+
deps.registry,
|
|
5955
|
+
ref,
|
|
5956
|
+
retiredAt + grace,
|
|
5957
|
+
scanned.pointers.get(segmentKey(ref)) ?? [],
|
|
5958
|
+
noteFault
|
|
5959
|
+
);
|
|
5960
|
+
}
|
|
5961
|
+
}
|
|
4662
5962
|
entries.push({
|
|
4663
5963
|
...base,
|
|
4664
5964
|
action: dryRun ? "would-purge-tombstone" : "purged-tombstone"
|
|
4665
5965
|
});
|
|
4666
5966
|
} catch (err) {
|
|
5967
|
+
if (!isWriteConflictError(err)) {
|
|
5968
|
+
attempted -= 1;
|
|
5969
|
+
purgeRun += 1;
|
|
5970
|
+
if (purgeRun >= MAX_CONSECUTIVE_PURGE_FAULTS) purging = false;
|
|
5971
|
+
noteFault(err);
|
|
5972
|
+
}
|
|
4667
5973
|
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4668
5974
|
}
|
|
4669
5975
|
continue;
|
|
@@ -4707,22 +6013,27 @@ async function retireExpired(deps, options) {
|
|
|
4707
6013
|
}
|
|
4708
6014
|
retired += 1;
|
|
4709
6015
|
entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
4710
|
-
await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt);
|
|
6016
|
+
await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt, noteFault);
|
|
4711
6017
|
if (purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
4712
6018
|
const tombstone = await deps.registry.get(ref).catch(() => null);
|
|
4713
6019
|
if (tombstone?.status === "destroyed") {
|
|
4714
6020
|
const deleted = await deps.registry.delete(ref, tombstone.token).then(
|
|
4715
6021
|
() => true,
|
|
4716
|
-
() =>
|
|
6022
|
+
(err) => {
|
|
6023
|
+
noteFault(err);
|
|
6024
|
+
return false;
|
|
6025
|
+
}
|
|
4717
6026
|
);
|
|
4718
6027
|
if (deleted) continue;
|
|
4719
6028
|
}
|
|
4720
6029
|
}
|
|
4721
|
-
await stampRetirement(deps.registry, ref, now).catch(() =>
|
|
6030
|
+
const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
|
|
6031
|
+
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
4722
6032
|
} catch (err) {
|
|
4723
6033
|
const after = await deps.registry.get(ref).catch(() => null);
|
|
4724
6034
|
if (after?.status === "destroyed") {
|
|
4725
|
-
await stampRetirement(deps.registry, ref, now).catch(() =>
|
|
6035
|
+
const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
|
|
6036
|
+
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
4726
6037
|
retired += 1;
|
|
4727
6038
|
entries.push({
|
|
4728
6039
|
...base,
|
|
@@ -4743,6 +6054,20 @@ async function retireExpired(deps, options) {
|
|
|
4743
6054
|
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4744
6055
|
}
|
|
4745
6056
|
}
|
|
6057
|
+
if (!dryRun && removesRows) {
|
|
6058
|
+
let attemptedLitter = 0;
|
|
6059
|
+
for (const pointer of scanned.litter) {
|
|
6060
|
+
if (!owned(pointer.target)) continue;
|
|
6061
|
+
if (attemptedLitter >= limit) break;
|
|
6062
|
+
attemptedLitter += 1;
|
|
6063
|
+
try {
|
|
6064
|
+
if (await deps.registry.get(pointer.target) !== null) continue;
|
|
6065
|
+
await deps.registry.delete(pointer.ref, pointer.token);
|
|
6066
|
+
} catch (err) {
|
|
6067
|
+
noteFault(err);
|
|
6068
|
+
}
|
|
6069
|
+
}
|
|
6070
|
+
}
|
|
4746
6071
|
return {
|
|
4747
6072
|
scanned: mine.length,
|
|
4748
6073
|
eligible,
|
|
@@ -4750,6 +6075,8 @@ async function retireExpired(deps, options) {
|
|
|
4750
6075
|
wouldRetire,
|
|
4751
6076
|
tombstonesPurged,
|
|
4752
6077
|
limited,
|
|
6078
|
+
purgeFaults,
|
|
6079
|
+
...firstPurgeFault === void 0 ? {} : { firstPurgeFault },
|
|
4753
6080
|
dryRun,
|
|
4754
6081
|
entries
|
|
4755
6082
|
};
|
|
@@ -4765,16 +6092,17 @@ function retirementStamp(meta) {
|
|
|
4765
6092
|
async function stampRetirement(registry, ref, now) {
|
|
4766
6093
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4767
6094
|
const rec = await registry.get(ref);
|
|
4768
|
-
if (rec === null || rec.status !== "destroyed") return;
|
|
6095
|
+
if (rec === null || rec.status !== "destroyed") return false;
|
|
4769
6096
|
try {
|
|
4770
6097
|
await registry.compareAndSwap(ref, rec.token, {
|
|
4771
6098
|
retention: { ...rec.retention, [RETIRED_AT]: now }
|
|
4772
6099
|
});
|
|
4773
|
-
return;
|
|
6100
|
+
return true;
|
|
4774
6101
|
} catch (err) {
|
|
4775
6102
|
if (!isWriteConflictError(err)) throw err;
|
|
4776
6103
|
}
|
|
4777
6104
|
}
|
|
6105
|
+
return false;
|
|
4778
6106
|
}
|
|
4779
6107
|
async function isFullyReclaimed(deps, ref) {
|
|
4780
6108
|
for await (const key of deps.storage.list(ref)) {
|
|
@@ -4795,8 +6123,45 @@ async function generationsPresent(storage, ref) {
|
|
|
4795
6123
|
for await (const key of storage.list(ref)) present.add(key.generation);
|
|
4796
6124
|
return present;
|
|
4797
6125
|
}
|
|
6126
|
+
async function checkSummary(ref, live, deps) {
|
|
6127
|
+
const generation = live.currentGen;
|
|
6128
|
+
const keyed = live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0;
|
|
6129
|
+
let aead;
|
|
6130
|
+
if (keyed) {
|
|
6131
|
+
if (deps.keystore === void 0) return { kind: "ok", unchecked: true };
|
|
6132
|
+
try {
|
|
6133
|
+
aead = await deps.keystore.openDek(live.wrappedDeks);
|
|
6134
|
+
} catch (err) {
|
|
6135
|
+
if (isIntegrityError(err) || isCloudRoaringError(err) && err.name === "KeyUnavailableError")
|
|
6136
|
+
return { kind: "ok", unchecked: true };
|
|
6137
|
+
throw err;
|
|
6138
|
+
}
|
|
6139
|
+
}
|
|
6140
|
+
const described = usableSummary(ref, live, aead);
|
|
6141
|
+
if (described === void 0) return { kind: "ok", unchecked: keyed };
|
|
6142
|
+
const reader = await openGenerationReader(
|
|
6143
|
+
deps.storage,
|
|
6144
|
+
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
6145
|
+
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) }
|
|
6146
|
+
);
|
|
6147
|
+
let cardinality = 0;
|
|
6148
|
+
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
6149
|
+
if (summaryAgrees(described, { cardinality, metadata: reader.metadata })) return { kind: "ok" };
|
|
6150
|
+
return {
|
|
6151
|
+
kind: "issue",
|
|
6152
|
+
issue: {
|
|
6153
|
+
segment: ref.segment,
|
|
6154
|
+
namespace: ref.namespace,
|
|
6155
|
+
currentGen: generation,
|
|
6156
|
+
issue: "summary-mismatch"
|
|
6157
|
+
}
|
|
6158
|
+
};
|
|
6159
|
+
}
|
|
4798
6160
|
async function runConsistencyCheck(deps, options = {}) {
|
|
4799
6161
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
6162
|
+
if (options.summaries !== void 0 && typeof options.summaries !== "boolean") {
|
|
6163
|
+
throw new ValidationError(`summaries must be a boolean; got ${String(options.summaries)}`);
|
|
6164
|
+
}
|
|
4800
6165
|
const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
|
|
4801
6166
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
4802
6167
|
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
@@ -4818,7 +6183,10 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4818
6183
|
if (!live || live.status === "destroyed") return { kind: "ok" };
|
|
4819
6184
|
if (live.currentGen === null) return { kind: "ok" };
|
|
4820
6185
|
const present = await generationsPresent(deps.storage, ref);
|
|
4821
|
-
if (present.has(live.currentGen))
|
|
6186
|
+
if (present.has(live.currentGen)) {
|
|
6187
|
+
if (options.summaries !== true || live.summary === void 0) return { kind: "ok" };
|
|
6188
|
+
return await checkSummary(ref, live, deps);
|
|
6189
|
+
}
|
|
4822
6190
|
return {
|
|
4823
6191
|
kind: "issue",
|
|
4824
6192
|
issue: {
|
|
@@ -4841,53 +6209,13 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4841
6209
|
});
|
|
4842
6210
|
const inconsistent = [];
|
|
4843
6211
|
const errored = [];
|
|
6212
|
+
let summariesUnchecked = 0;
|
|
4844
6213
|
for (const r of results) {
|
|
4845
6214
|
if (r.kind === "issue") inconsistent.push(r.issue);
|
|
4846
6215
|
else if (r.kind === "error") errored.push(r.error);
|
|
6216
|
+
else if (r.unchecked === true) summariesUnchecked += 1;
|
|
4847
6217
|
}
|
|
4848
|
-
return { checked: recs.length, inconsistent, errored };
|
|
4849
|
-
}
|
|
4850
|
-
|
|
4851
|
-
// src/core/retry.ts
|
|
4852
|
-
var DEFAULT_RETRY_POLICY = {
|
|
4853
|
-
maxAttempts: 4,
|
|
4854
|
-
baseDelayMs: 50,
|
|
4855
|
-
maxDelayMs: 2e3,
|
|
4856
|
-
backoffFactor: 2,
|
|
4857
|
-
jitter: "full"
|
|
4858
|
-
};
|
|
4859
|
-
function backoffDelayMs(policy, attempt) {
|
|
4860
|
-
const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
|
|
4861
|
-
return Math.min(policy.maxDelayMs, raw);
|
|
4862
|
-
}
|
|
4863
|
-
function applyJitter(policy, delayMs, rng) {
|
|
4864
|
-
if (policy.jitter === "none") return delayMs;
|
|
4865
|
-
return rng.next() * delayMs;
|
|
4866
|
-
}
|
|
4867
|
-
async function withRetry(op, policy, deps) {
|
|
4868
|
-
const retryable = deps.isRetryable ?? isTransientError;
|
|
4869
|
-
if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
|
|
4870
|
-
throw new ValidationError(
|
|
4871
|
-
`retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
|
|
4872
|
-
);
|
|
4873
|
-
}
|
|
4874
|
-
const attempts = Math.floor(policy.maxAttempts);
|
|
4875
|
-
let lastErr;
|
|
4876
|
-
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
4877
|
-
try {
|
|
4878
|
-
return await op();
|
|
4879
|
-
} catch (err) {
|
|
4880
|
-
lastErr = err;
|
|
4881
|
-
if (attempt >= attempts || !retryable(err)) throw err;
|
|
4882
|
-
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
4883
|
-
try {
|
|
4884
|
-
deps.onRetry?.({ attempt, delayMs, err });
|
|
4885
|
-
} catch {
|
|
4886
|
-
}
|
|
4887
|
-
await deps.clock.sleep(delayMs);
|
|
4888
|
-
}
|
|
4889
|
-
}
|
|
4890
|
-
throw lastErr;
|
|
6218
|
+
return options.summaries === true ? { checked: recs.length, inconsistent, errored, summariesUnchecked } : { checked: recs.length, inconsistent, errored };
|
|
4891
6219
|
}
|
|
4892
6220
|
|
|
4893
6221
|
// src/drivers/retry/retrying-chunk-source.ts
|
|
@@ -4909,6 +6237,7 @@ var RetryingStorageChunkSource = class {
|
|
|
4909
6237
|
/** Present only when the inner source supports it — so capability detection stays honest. */
|
|
4910
6238
|
sizeOf;
|
|
4911
6239
|
cardinalities;
|
|
6240
|
+
summary;
|
|
4912
6241
|
currentGeneration;
|
|
4913
6242
|
invalidate;
|
|
4914
6243
|
exists;
|
|
@@ -4933,6 +6262,10 @@ var RetryingStorageChunkSource = class {
|
|
|
4933
6262
|
if (innerCardinalities) {
|
|
4934
6263
|
this.cardinalities = (ref) => withRetry(() => innerCardinalities.call(inner, ref), this.policy, this.deps);
|
|
4935
6264
|
}
|
|
6265
|
+
const innerSummary = inner.summary;
|
|
6266
|
+
if (innerSummary) {
|
|
6267
|
+
this.summary = (ref) => withRetry(() => innerSummary.call(inner, ref), this.policy, this.deps);
|
|
6268
|
+
}
|
|
4936
6269
|
const innerCurrentGeneration = inner.currentGeneration;
|
|
4937
6270
|
if (innerCurrentGeneration) {
|
|
4938
6271
|
this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);
|