@cloudbitmaps/core 0.11.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/{chunk-5OMC5U7U.js → chunk-5W3XS2IU.js} +292 -22
  2. package/dist/chunk-5W3XS2IU.js.map +7 -0
  3. package/dist/core/audit.d.ts +6 -0
  4. package/dist/core/codec.d.ts +33 -1
  5. package/dist/core/consistency.d.ts +21 -2
  6. package/dist/core/cost.d.ts +91 -10
  7. package/dist/core/crbm/format.d.ts +31 -1
  8. package/dist/core/crbm/reader.d.ts +44 -2
  9. package/dist/core/crbm/writer.d.ts +32 -3
  10. package/dist/core/crbm-storage-source.d.ts +203 -22
  11. package/dist/core/crypto.d.ts +11 -7
  12. package/dist/core/determinism.d.ts +7 -0
  13. package/dist/core/engine.d.ts +17 -5
  14. package/dist/core/erase-id.d.ts +20 -0
  15. package/dist/core/erasure.d.ts +8 -4
  16. package/dist/core/generation-gc.d.ts +102 -4
  17. package/dist/core/load-input.d.ts +55 -0
  18. package/dist/core/load.d.ts +65 -10
  19. package/dist/core/metadata.d.ts +42 -0
  20. package/dist/core/pinned-storage-source.d.ts +2 -1
  21. package/dist/core/ports.d.ts +119 -13
  22. package/dist/core/registry-scan.d.ts +6 -0
  23. package/dist/core/retention-sweep.d.ts +47 -2
  24. package/dist/core/retry.d.ts +14 -0
  25. package/dist/core/rollback.d.ts +34 -4
  26. package/dist/core/summary.d.ts +61 -0
  27. package/dist/core/token.d.ts +36 -0
  28. package/dist/core/validate.d.ts +2 -0
  29. package/dist/driver-kit.js +39 -20
  30. package/dist/driver-kit.js.map +2 -2
  31. package/dist/drivers/_shared/entropy.d.ts +5 -0
  32. package/dist/drivers/_shared/object-registry.d.ts +36 -6
  33. package/dist/drivers/_shared/registry.d.ts +47 -19
  34. package/dist/drivers/localfs/registry.d.ts +10 -0
  35. package/dist/drivers/memory.d.ts +9 -3
  36. package/dist/drivers/retry/retrying-chunk-source.d.ts +2 -1
  37. package/dist/index.d.ts +4 -3
  38. package/dist/index.js +1794 -461
  39. package/dist/index.js.map +4 -4
  40. package/dist/testing/fuzz-core.d.ts +6 -5
  41. package/package.json +1 -1
  42. package/dist/chunk-5OMC5U7U.js.map +0 -7
package/dist/index.js CHANGED
@@ -5,6 +5,7 @@ import {
5
5
  DEFAULT_NAMESPACE,
6
6
  IntegrityError,
7
7
  KeyUnavailableError,
8
+ MAX_METADATA_BYTES,
8
9
  MAX_NAME_LENGTH,
9
10
  NotFoundError,
10
11
  RESERVED_NAMESPACE_PREFIX,
@@ -13,11 +14,18 @@ import {
13
14
  ValidationError,
14
15
  WriteConflictError,
15
16
  applyRegistryPatch,
17
+ assertRegistryCanWrite,
16
18
  brandAsBackend,
19
+ canonicalMetadataJson,
20
+ copiedMetadata,
17
21
  decodeNameFromPath,
22
+ drawIncarnation,
23
+ drawWrite,
18
24
  encodeNameForKey,
19
25
  encodeNameForPath,
20
26
  encodedNameLength,
27
+ entropyIsAvailable,
28
+ incarnationOf,
21
29
  isCloudRoaringError,
22
30
  isIntegrityError,
23
31
  isNotFoundError,
@@ -27,18 +35,23 @@ import {
27
35
  isValidationError,
28
36
  isWriteConflictError,
29
37
  mapWithConcurrency,
38
+ metadataBytes,
39
+ metadataFromBytes,
30
40
  namespacePathPart,
41
+ newIncarnationToken,
42
+ nextRegistryToken,
31
43
  parseRegistryEnvelope,
32
44
  recordFromNew,
33
- registryCounterOf,
45
+ sameIncarnation,
34
46
  serializeRegistryEnvelope,
35
47
  validateChunkRef,
36
48
  validateNewRegistryRecord,
37
49
  validateRegistryPatch,
38
50
  validateSegmentRef,
39
51
  validateUserNamespace,
40
- validateUserRef
41
- } from "./chunk-5OMC5U7U.js";
52
+ validateUserRef,
53
+ webCryptoEntropy
54
+ } from "./chunk-5W3XS2IU.js";
42
55
 
43
56
  // src/core/bit-route.ts
44
57
  var U32_MAX = 4294967295;
@@ -313,15 +326,19 @@ var SegmentEngine = class {
313
326
  return chunk ? chunk.has(remainder) : false;
314
327
  }
315
328
  /**
316
- * Cardinality. When the Storage source can serve per-chunk cardinality from its `.crbm` index (the `.crbm`
317
- * source can), the count is summed from the index with **zero payload reads**. A source without that
318
- * capability (the in-memory source) falls back to fetching every chunk.
329
+ * Cardinality. When the Storage source can describe the current generation (the `.crbm` source can), the count is
330
+ * that description's: the registry row's summary when the row has one it can use, which is one request when cold
331
+ * and no read of the object, and otherwise the sum of the object's index. Either way, **zero payload reads**. A
332
+ * source that can only serve per-chunk cardinality sums those, and one with neither (the in-memory source) falls back
333
+ * to fetching every chunk.
319
334
  *
320
- * The index sum is trusted, not confirmed: the reader checked the index for internal consistency when it opened
335
+ * The row's summary and the index sum are trusted, not confirmed: the source holds the summary against the object
336
+ * whenever it opens it, and the reader checked the index for internal consistency when it opened
321
337
  * the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
322
338
  * combines decode the payloads.
323
339
  */
324
340
  async count(seg) {
341
+ if (this.storage.summary) return (await this.storage.summary(seg))?.cardinality ?? 0;
325
342
  const cardinalities = this.storage.cardinalities ? await this.storage.cardinalities(seg) : null;
326
343
  if (cardinalities) {
327
344
  let total2 = 0;
@@ -339,6 +356,17 @@ var SegmentEngine = class {
339
356
  for (let i = 0; i < chunkKeys.length; i++) total += (await window.take())?.size ?? 0;
340
357
  return total;
341
358
  }
359
+ /**
360
+ * What the segment's current generation is: its number, its id count and its metadata, from one resolution, so the
361
+ * three cannot straddle a publish. A source with no summary answers from `count()` and the generation it resolves.
362
+ */
363
+ async stat(seg) {
364
+ if (this.storage.summary) {
365
+ return await this.storage.summary(seg) ?? { generation: null, cardinality: 0 };
366
+ }
367
+ const generation = this.storage.currentGeneration ? await this.storage.currentGeneration(seg) : null;
368
+ return { generation, cardinality: await this.count(seg) };
369
+ }
342
370
  /** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
343
371
  get supportsStorageSize() {
344
372
  return typeof this.storage.sizeOf === "function";
@@ -871,6 +899,111 @@ var BoundedLru = class {
871
899
  }
872
900
  };
873
901
 
902
+ // src/core/generation-gc.ts
903
+ async function nextGeneration(ref, deps) {
904
+ return aboveEverything(ref, deps, await deps.registry.get(ref));
905
+ }
906
+ async function aboveEverything(ref, deps, record) {
907
+ let highest = record?.currentGen ?? -1;
908
+ for await (const key of deps.storage.list(ref)) {
909
+ if (key.generation > highest) highest = key.generation;
910
+ }
911
+ return highest + 1;
912
+ }
913
+ async function nextLoadGeneration(ref, deps, row) {
914
+ const generation = (row?.currentGen ?? -1) + 1;
915
+ try {
916
+ await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
917
+ } catch (err) {
918
+ if (isNotFoundError(err)) return { generation, checked: true };
919
+ }
920
+ return { generation: await aboveEverything(ref, deps, row), checked: false };
921
+ }
922
+ var LIST_COLLECTION_CADENCE = 16;
923
+ async function gcOrphanGenerations(ref, deps, options = {}) {
924
+ const keep = options.keep ?? 1;
925
+ if (!Number.isInteger(keep) || keep < 0) {
926
+ throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
927
+ }
928
+ const record = await deps.registry.get(ref);
929
+ if (record === null) return [];
930
+ const current = record.currentGen;
931
+ const seen = /* @__PURE__ */ new Set();
932
+ for await (const key of deps.storage.list(ref)) seen.add(key.generation);
933
+ const gens = [...seen];
934
+ const after = await deps.registry.get(ref);
935
+ if (after === null)
936
+ throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
937
+ if (record.status === "destroyed" && after.token !== record.token) {
938
+ throw new WriteConflictError(
939
+ `segment ${ref.segment} changed incarnation while its generations were being listed`
940
+ );
941
+ }
942
+ const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
943
+ const stillCollectable = async () => {
944
+ const still = await deps.registry.get(ref);
945
+ if (still === null) {
946
+ throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
947
+ }
948
+ const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
949
+ if (!ok) {
950
+ throw new WriteConflictError(
951
+ `segment ${ref.segment} changed incarnation while its generations were being collected`
952
+ );
953
+ }
954
+ };
955
+ const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
956
+ // No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
957
+ // here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
958
+ // that publish into a dangling pointer. It is collected once a pointer exists.
959
+ []
960
+ ) : (
961
+ // Delete generations below the cutoff, except the newest `keep` of them (the grace window).
962
+ gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
963
+ );
964
+ for (const generation of toDelete) {
965
+ await stillCollectable();
966
+ await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
967
+ }
968
+ return toDelete;
969
+ }
970
+ async function collectAfterLoad(ref, deps, options) {
971
+ const { generation, keep, byName, proveCurrent } = options;
972
+ if (keep >= generation) return [];
973
+ const periodic = generation % LIST_COLLECTION_CADENCE === 0;
974
+ if (byName && keep <= 1 && !periodic) {
975
+ if (keep === 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
976
+ return gcOrphanGenerations(ref, deps, { keep });
977
+ }
978
+ return collectByName(ref, deps, { generation, keep });
979
+ }
980
+ return gcOrphanGenerations(ref, deps, { keep });
981
+ }
982
+ async function objectIsThere(storage, key) {
983
+ try {
984
+ await storage.getTail(key, 0);
985
+ return true;
986
+ } catch {
987
+ return false;
988
+ }
989
+ }
990
+ async function collectByName(ref, deps, options) {
991
+ const { generation, keep } = options;
992
+ if (!Number.isInteger(keep) || keep < 0 || keep > 1) {
993
+ throw new ValidationError(`collecting by name needs a keep of 0 or 1; got ${String(keep)}`);
994
+ }
995
+ const evicted = generation - keep - 1;
996
+ if (evicted < 0) return [];
997
+ const still = await deps.registry.get(ref);
998
+ if (still === null || still.currentGen === null || still.currentGen < generation) return [];
999
+ await deps.storage.delete({
1000
+ namespace: ref.namespace,
1001
+ segment: ref.segment,
1002
+ generation: evicted
1003
+ });
1004
+ return [evicted];
1005
+ }
1006
+
874
1007
  // src/core/reader-defaults.ts
875
1008
  var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
876
1009
  var REFRESH_RETRY_MS = 500;
@@ -918,9 +1051,20 @@ var AWS_US_EAST_1_ONDEMAND = deepFreeze({
918
1051
  var HOURS_PER_MONTH = 730;
919
1052
  var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
920
1053
  var GIB = 1024 ** 3;
921
- var SIZED_READS_PER_COLD_OPERAND = 2;
922
- var STORE_LOAD_PUT_CLASS = 3;
923
- var STORE_LOAD_SIZED_READS = 9;
1054
+ var STORE_LOAD_PUT_CLASS = 1 + 1 / LIST_COLLECTION_CADENCE;
1055
+ var STORE_LOAD_POINTER_READS = 3 + 2 / LIST_COLLECTION_CADENCE;
1056
+ var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
1057
+ var RETENTION_SWEEP_REQUESTS = deepFreeze({
1058
+ conditionalDelete: {
1059
+ retirement: { reads: 9, writes: 3, deletes: 1 },
1060
+ purge: { reads: 4, writes: 0, deletes: 2 }
1061
+ },
1062
+ tombstoning: {
1063
+ retirement: { reads: 8, writes: 3, deletes: 0 },
1064
+ purge: { reads: 3, writes: 1, deletes: 0 }
1065
+ }
1066
+ });
1067
+ var shown = (n) => String(Number(n.toFixed(2)));
924
1068
  function requireFiniteNonNeg(n, field) {
925
1069
  if (n === void 0 || !Number.isFinite(n) || n < 0) {
926
1070
  throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
@@ -1057,8 +1201,8 @@ function bytesWords(bytes) {
1057
1201
  const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
1058
1202
  let value = bytes;
1059
1203
  let unit = 0;
1060
- const shown = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
1061
- while (unit < units.length - 1 && shown(value, unit) >= 1024) {
1204
+ const shown2 = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
1205
+ while (unit < units.length - 1 && shown2(value, unit) >= 1024) {
1062
1206
  value /= 1024;
1063
1207
  unit += 1;
1064
1208
  }
@@ -1085,6 +1229,10 @@ function buildReport(input) {
1085
1229
  requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
1086
1230
  requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
1087
1231
  requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
1232
+ const pointerRead = requireFiniteNonNeg(
1233
+ storage.requestsPerPointerRead ?? 1,
1234
+ "pricing.storage.requestsPerPointerRead"
1235
+ );
1088
1236
  const sizedRead = requireFiniteNonNeg(
1089
1237
  storage.requestsPerSizedRead ?? 1,
1090
1238
  "pricing.storage.requestsPerSizedRead"
@@ -1131,6 +1279,15 @@ function buildReport(input) {
1131
1279
  input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
1132
1280
  "genTtlMs"
1133
1281
  );
1282
+ const retirementsPerMonth = requireFiniteNonNeg(
1283
+ input.workload.retirementsPerMonth ?? 0,
1284
+ "retirementsPerMonth"
1285
+ );
1286
+ const purgesPerMonth = requireFiniteNonNeg(input.workload.purgesPerMonth ?? 0, "purgesPerMonth");
1287
+ const conditionalDelete = input.workload.conditionalDelete ?? true;
1288
+ if (typeof conditionalDelete !== "boolean") {
1289
+ throw new ValidationError(`conditionalDelete must be a boolean; got ${conditionalDelete}`);
1290
+ }
1134
1291
  const storageGetUSD = storage.getPerMillion / 1e6;
1135
1292
  const putUSD = storage.putPerMillion / 1e6;
1136
1293
  const missFraction = 1 - cacheHitRate;
@@ -1139,16 +1296,26 @@ function buildReport(input) {
1139
1296
  const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
1140
1297
  const storageUSD = storedGiB * storage.storagePerGiBMonth;
1141
1298
  const readsUSD = readMisses * storageGetUSD;
1142
- const intersectGets = chunksPerIntersect + SIZED_READS_PER_COLD_OPERAND * operandsPerIntersect * sizedRead;
1299
+ const getsPerColdOperand = pointerRead + sizedRead;
1300
+ const intersectGets = chunksPerIntersect + getsPerColdOperand * operandsPerIntersect;
1143
1301
  const intersectsUSD = intersects * intersectGets * storageGetUSD;
1144
- const loadGets = STORE_LOAD_SIZED_READS * sizedRead;
1302
+ const loadGets = STORE_LOAD_POINTER_READS * pointerRead + STORE_LOAD_EXISTENCE_CHECKS;
1145
1303
  const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
1146
- const refreshUSD = refreshes * sizedRead * storageGetUSD;
1147
- const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD;
1304
+ const refreshUSD = refreshes * pointerRead * storageGetUSD;
1305
+ const sweep = RETENTION_SWEEP_REQUESTS[conditionalDelete ? "conditionalDelete" : "tombstoning"];
1306
+ const retentionUSD = retirementsPerMonth * (sweep.retirement.reads * storageGetUSD + sweep.retirement.writes * putUSD) + purgesPerMonth * (sweep.purge.reads * storageGetUSD + sweep.purge.writes * putUSD);
1307
+ const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD + retentionUSD;
1148
1308
  const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
1149
1309
  const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
1150
1310
  const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
1151
- const dominant = Math.max(readsUSD, intersectsUSD, storageUSD, loadsUSD, refreshUSD);
1311
+ const dominant = Math.max(
1312
+ readsUSD,
1313
+ intersectsUSD,
1314
+ storageUSD,
1315
+ loadsUSD,
1316
+ refreshUSD,
1317
+ retentionUSD
1318
+ );
1152
1319
  let driver = "storage";
1153
1320
  if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
1154
1321
  else if (dominant === intersectsUSD && intersectsUSD > 0) {
@@ -1156,6 +1323,8 @@ function buildReport(input) {
1156
1323
  } else if (dominant === loadsUSD && loadsUSD > 0)
1157
1324
  driver = "loads (object, listing and pointer requests)";
1158
1325
  else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
1326
+ else if (dominant === retentionUSD && retentionUSD > 0)
1327
+ driver = "the retention sweep (registry reads and writes)";
1159
1328
  const sized = baseline.sized;
1160
1329
  const noBytes = Math.round(storageBytes) === 0;
1161
1330
  const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
@@ -1165,12 +1334,15 @@ function buildReport(input) {
1165
1334
  const notes = [
1166
1335
  "Same-region egress treated as free; internet egress is not modeled.",
1167
1336
  "Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
1168
- loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${STORE_LOAD_PUT_CLASS} PUT-class and ${loadGets} GETs that store.load() adds (listings, pointer, index).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
1337
+ loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${shown(STORE_LOAD_PUT_CLASS)} PUT-class and ${shown(loadGets)} GETs that store.load() adds (the pointer, a check that the next generation number is free, a check that the current generation is there, and a listing every ${LIST_COLLECTION_CADENCE}th load).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
1169
1338
  ...intersects > 0 ? [
1170
- `Intersections priced cold: ${intersectGets} GETs each, ${SIZED_READS_PER_COLD_OPERAND * sizedRead} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk read(s); cacheHitRate does not apply.`
1339
+ `Intersections priced cold: ${intersectGets} GETs each, ${getsPerColdOperand} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk read(s); cacheHitRate does not apply.`
1171
1340
  ] : [],
1172
1341
  hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
1173
1342
  hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
1343
+ ...retirementsPerMonth > 0 || purgesPerMonth > 0 ? [
1344
+ `Retention sweep modeled: ${retirementsPerMonth}/mo retirements at ${sweep.retirement.reads} reads, ${sweep.retirement.writes} writes and ${sweep.retirement.deletes} delete(s) each, ${purgesPerMonth}/mo purges at ${sweep.purge.reads} reads, ${sweep.purge.writes} writes and ${sweep.purge.deletes} delete(s) each; reads are priced at the GET rate and writes at the PUT rate, deletes at nothing, as S3 bills none` + (conditionalDelete ? "." : "; a registry that only tombstones is read again, two reads per purged segment, by every later full sweep, which is not priced.")
1345
+ ] : [],
1174
1346
  ...input.extraNotes ?? [],
1175
1347
  // Last, so the notes a report already carried keep their places.
1176
1348
  redisNote
@@ -1182,7 +1354,8 @@ function buildReport(input) {
1182
1354
  intersects: intersectsUSD,
1183
1355
  storage: storageUSD,
1184
1356
  loads: loadsUSD,
1185
- pointerRefresh: refreshUSD
1357
+ pointerRefresh: refreshUSD,
1358
+ retention: retentionUSD
1186
1359
  },
1187
1360
  total
1188
1361
  },
@@ -1291,7 +1464,10 @@ async function drainRegistry(registry, options) {
1291
1464
  const advice = options.narrowable === false ? `${raise[0].toUpperCase()}${raise.slice(1)} if the namespace really is that large and the memory is ` : `Narrow it with \`namespace\`, or ${raise} if the fleet really is that large and the memory is `;
1292
1465
  const rows = [];
1293
1466
  for await (const rec of registry.list(options.namespace)) {
1294
- if (options.namespace === void 0 && isReservedRow(rec)) continue;
1467
+ if (options.namespace === void 0 && isReservedRow(rec)) {
1468
+ options.onReserved?.(rec);
1469
+ continue;
1470
+ }
1295
1471
  if (rows.length >= maxScanSegments) {
1296
1472
  throw new BudgetExceededError(
1297
1473
  `${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident).`
@@ -1463,16 +1639,19 @@ var BufferReader = class {
1463
1639
  var MemoryRegistryDriver = class {
1464
1640
  rows = /* @__PURE__ */ new Map();
1465
1641
  now;
1642
+ entropy;
1466
1643
  seq = 0;
1467
1644
  constructor(options = {}) {
1468
1645
  this.now = options.now ?? (() => Date.now());
1646
+ this.entropy = options.entropy ?? webCryptoEntropy;
1469
1647
  }
1470
- nextToken() {
1648
+ nextToken(incarnation) {
1471
1649
  this.seq += 1;
1472
- return String(this.seq);
1650
+ return `${incarnation}.${this.seq}.${drawWrite(this.entropy)}`;
1473
1651
  }
1652
+ /** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
1474
1653
  capabilities() {
1475
- return { strongRead: true };
1654
+ return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
1476
1655
  }
1477
1656
  async get(ref) {
1478
1657
  validateSegmentRef(ref);
@@ -1481,25 +1660,25 @@ var MemoryRegistryDriver = class {
1481
1660
  }
1482
1661
  async create(ref, record) {
1483
1662
  validateSegmentRef(ref);
1484
- validateNewRegistryRecord(record);
1663
+ const checked = validateNewRegistryRecord(record);
1485
1664
  const key = segmentKey(ref);
1486
1665
  if (this.rows.has(key)) {
1487
1666
  throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
1488
1667
  }
1489
- const token = this.nextToken();
1490
- this.rows.set(key, structuredClone(recordFromNew(ref, record, this.now(), token)));
1668
+ const token = this.nextToken(drawIncarnation(this.entropy));
1669
+ this.rows.set(key, structuredClone(recordFromNew(ref, checked, this.now(), token)));
1491
1670
  return { token };
1492
1671
  }
1493
1672
  async compareAndSwap(ref, expected, patch) {
1494
1673
  validateSegmentRef(ref);
1495
- validateRegistryPatch(patch);
1674
+ const checked = validateRegistryPatch(patch);
1496
1675
  const key = segmentKey(ref);
1497
1676
  const existing = this.rows.get(key);
1498
1677
  if (!existing || existing.token !== expected) {
1499
1678
  throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
1500
1679
  }
1501
- const token = this.nextToken();
1502
- this.rows.set(key, structuredClone(applyRegistryPatch(existing, patch, this.now(), token)));
1680
+ const token = this.nextToken(incarnationOf(existing.token) ?? drawIncarnation(this.entropy));
1681
+ this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
1503
1682
  return { token };
1504
1683
  }
1505
1684
  async *list(namespace) {
@@ -1816,43 +1995,48 @@ var LocalFsRegistryDriver = class {
1816
1995
  constructor(root, options = {}) {
1817
1996
  this.root = root;
1818
1997
  this.now = options.now ?? (() => Date.now());
1998
+ this.entropy = options.entropy ?? webCryptoEntropy;
1819
1999
  }
1820
2000
  root;
1821
2001
  now;
2002
+ entropy;
2003
+ /**
2004
+ * `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
2005
+ * the root in this process takes, so no write can land between the check and the unlink.
2006
+ */
1822
2007
  capabilities() {
1823
- return { strongRead: true };
2008
+ return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
1824
2009
  }
1825
2010
  async get(ref) {
1826
2011
  const env = await this.readRow(registryRowPath(this.root, ref));
1827
2012
  return env && !env.deleted ? env.record : null;
1828
2013
  }
1829
2014
  async create(ref, record) {
1830
- validateNewRegistryRecord(record);
2015
+ const checked = validateNewRegistryRecord(record);
1831
2016
  const path = registryRowPath(this.root, ref);
1832
2017
  return this.withRowLock(path, async () => {
1833
2018
  const current = await this.readRow(path);
1834
2019
  if (current !== null && !current.deleted) {
1835
2020
  throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
1836
2021
  }
1837
- const counter = current ? registryCounterOf(current.record) + 1 : 0;
1838
- const token = String(counter);
1839
- await this.writeRow(path, false, recordFromNew(ref, record, this.now(), token));
2022
+ const token = newIncarnationToken(this.entropy, current?.record);
2023
+ await this.writeRow(path, false, recordFromNew(ref, checked, this.now(), token));
1840
2024
  return { token };
1841
2025
  });
1842
2026
  }
1843
2027
  async compareAndSwap(ref, expected, patch) {
1844
- validateRegistryPatch(patch);
2028
+ const checked = validateRegistryPatch(patch);
1845
2029
  const path = registryRowPath(this.root, ref);
1846
2030
  return this.withRowLock(path, async () => {
1847
2031
  const current = await this.readRow(path);
1848
2032
  if (current === null || current.deleted || current.record.token !== expected) {
1849
2033
  throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
1850
2034
  }
1851
- const token = String(registryCounterOf(current.record) + 1);
2035
+ const token = nextRegistryToken(current.record, this.entropy);
1852
2036
  await this.writeRow(
1853
2037
  path,
1854
2038
  false,
1855
- applyRegistryPatch(current.record, patch, this.now(), token)
2039
+ applyRegistryPatch(current.record, checked, this.now(), token)
1856
2040
  );
1857
2041
  return { token };
1858
2042
  });
@@ -1886,10 +2070,23 @@ var LocalFsRegistryDriver = class {
1886
2070
  } else if (current === null || current.deleted) {
1887
2071
  return;
1888
2072
  }
1889
- const token = String(registryCounterOf(current.record) + 1);
2073
+ if (incarnationOf(current.record.token) !== void 0) {
2074
+ await this.unlinkRow(path);
2075
+ return;
2076
+ }
2077
+ const token = nextRegistryToken(current.record, this.entropy);
1890
2078
  await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
1891
2079
  });
1892
2080
  }
2081
+ /** Remove a row's file and make the removal durable (fsync the directory). */
2082
+ async unlinkRow(path) {
2083
+ try {
2084
+ await unlink2(path);
2085
+ } catch (err) {
2086
+ if (!isCode(err, "ENOENT")) throw mapFsError(err);
2087
+ }
2088
+ await fsyncDir(dirname2(path));
2089
+ }
1893
2090
  /** Namespaces to scan: just the one requested, or every namespace dir under the root. */
1894
2091
  async namespaceDirs(namespace) {
1895
2092
  if (namespace !== void 0) return [namespace];
@@ -2045,6 +2242,8 @@ function yieldEvery(clock, everyN = YIELD_EVERY) {
2045
2242
  var AAD_VERSION = 1;
2046
2243
  var SCOPE_CHUNK = 0;
2047
2244
  var SCOPE_INDEX = 1;
2245
+ var SCOPE_METADATA = 2;
2246
+ var SCOPE_SUMMARY = 3;
2048
2247
  function aadFor(ref, generation, scope) {
2049
2248
  const enc = new TextEncoder();
2050
2249
  const ns = enc.encode(ref.namespace ?? "");
@@ -2064,12 +2263,82 @@ function aadFor(ref, generation, scope) {
2064
2263
  o += seg.length;
2065
2264
  view.setBigUint64(o, BigInt(generation), true);
2066
2265
  o += 8;
2067
- out[o] = scope === "index" ? SCOPE_INDEX : SCOPE_CHUNK;
2266
+ out[o] = scope === "index" ? SCOPE_INDEX : scope === "metadata" ? SCOPE_METADATA : scope === "summary" ? SCOPE_SUMMARY : SCOPE_CHUNK;
2068
2267
  o += 1;
2069
- view.setUint32(o, scope === "index" ? 0 : scope, true);
2268
+ view.setUint32(o, typeof scope === "number" ? scope : 0, true);
2070
2269
  return out;
2071
2270
  }
2072
2271
 
2272
+ // src/core/crbm/format.ts
2273
+ var MAGIC = Uint8Array.of(67, 82, 66, 77);
2274
+ var VERSION_MAJOR = 1;
2275
+ var VERSION_MINOR = 0;
2276
+ var PREAMBLE_BYTES = 8;
2277
+ var PAYLOAD_START = PREAMBLE_BYTES;
2278
+ var FOOTER_BYTES = 104;
2279
+ var FOOTER = {
2280
+ indexOffset: 0,
2281
+ // u64
2282
+ indexLength: 8,
2283
+ // u64
2284
+ indexCrc32c: 16,
2285
+ // u32
2286
+ flags: 20,
2287
+ // u32
2288
+ payloadCodecId: 24,
2289
+ // u16 — which codec wrote the chunk payloads
2290
+ elementWidth: 26,
2291
+ // u8
2292
+ containerCodec: 27,
2293
+ // u8
2294
+ versionMajor: 28,
2295
+ // u8
2296
+ versionMinor: 29,
2297
+ // u8
2298
+ reserved2: 30,
2299
+ // u16
2300
+ generation: 32,
2301
+ // u64
2302
+ indexNonce: 40,
2303
+ // 12 B
2304
+ indexTag: 52,
2305
+ // 16 B
2306
+ keyId: 68,
2307
+ // 16 B
2308
+ chunkCount: 84,
2309
+ // u32
2310
+ totalCardinality: 88,
2311
+ // u64
2312
+ footerCrc32c: 96,
2313
+ // u32 — covers footer bytes [0, 96)
2314
+ endMagic: 100
2315
+ // char[4]
2316
+ };
2317
+ var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
2318
+ var FLAG_ENCRYPTED = 1 << 0;
2319
+ var FLAG_INDEX_COMPRESSED = 1 << 1;
2320
+ var FLAG_LITTLE_ENDIAN = 1 << 2;
2321
+ var FLAG_EXTENSION = 1 << 3;
2322
+ var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
2323
+ var ELEMENT_WIDTH_32 = 32;
2324
+ var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
2325
+ var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
2326
+ PAYLOAD_CODEC_ROARING_PORTABLE
2327
+ ]);
2328
+ var CONTAINER_CODEC_NONE = 0;
2329
+ var DEFAULT_TAIL_BYTES = 256 * 1024;
2330
+ var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
2331
+ var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
2332
+ var MAX_CHUNK_CARDINALITY = 65536;
2333
+ var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
2334
+ var EXT_TRAILER_BYTES = 12;
2335
+ var EXT_SECTION_HEADER_BYTES = 5;
2336
+ var MAX_EXT_BYTES = 4 * 1024;
2337
+ var EXT_SECTION_METADATA = 1;
2338
+ var CRC32C_BYTES = 4;
2339
+ var AEAD_NONCE_BYTES = 12;
2340
+ var AEAD_TAG_BYTES = 16;
2341
+
2073
2342
  // src/core/crbm/crc32c.ts
2074
2343
  var TABLES = (() => {
2075
2344
  const POLY = 2197175160;
@@ -2148,71 +2417,9 @@ function readVarint(bytes, offset) {
2148
2417
  throw new IntegrityError("varint exceeds u32 range");
2149
2418
  }
2150
2419
 
2151
- // src/core/crbm/format.ts
2152
- var MAGIC = Uint8Array.of(67, 82, 66, 77);
2153
- var VERSION_MAJOR = 1;
2154
- var VERSION_MINOR = 0;
2155
- var PREAMBLE_BYTES = 8;
2156
- var PAYLOAD_START = PREAMBLE_BYTES;
2157
- var FOOTER_BYTES = 104;
2158
- var FOOTER = {
2159
- indexOffset: 0,
2160
- // u64
2161
- indexLength: 8,
2162
- // u64
2163
- indexCrc32c: 16,
2164
- // u32
2165
- flags: 20,
2166
- // u32
2167
- payloadCodecId: 24,
2168
- // u16 — which codec wrote the chunk payloads
2169
- elementWidth: 26,
2170
- // u8
2171
- containerCodec: 27,
2172
- // u8
2173
- versionMajor: 28,
2174
- // u8
2175
- versionMinor: 29,
2176
- // u8
2177
- reserved2: 30,
2178
- // u16
2179
- generation: 32,
2180
- // u64
2181
- indexNonce: 40,
2182
- // 12 B
2183
- indexTag: 52,
2184
- // 16 B
2185
- keyId: 68,
2186
- // 16 B
2187
- chunkCount: 84,
2188
- // u32
2189
- totalCardinality: 88,
2190
- // u64
2191
- footerCrc32c: 96,
2192
- // u32 — covers footer bytes [0, 96)
2193
- endMagic: 100
2194
- // char[4]
2195
- };
2196
- var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
2197
- var FLAG_ENCRYPTED = 1 << 0;
2198
- var FLAG_INDEX_COMPRESSED = 1 << 1;
2199
- var FLAG_LITTLE_ENDIAN = 1 << 2;
2200
- var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN;
2201
- var ELEMENT_WIDTH_32 = 32;
2202
- var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
2203
- var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
2204
- PAYLOAD_CODEC_ROARING_PORTABLE
2205
- ]);
2206
- var CONTAINER_CODEC_NONE = 0;
2207
- var DEFAULT_TAIL_BYTES = 256 * 1024;
2208
- var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
2209
- var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
2210
- var MAX_CHUNK_CARDINALITY = 65536;
2211
- var CRC32C_BYTES = 4;
2212
- var AEAD_NONCE_BYTES = 12;
2213
- var AEAD_TAG_BYTES = 16;
2214
-
2215
2420
  // src/core/crbm/reader.ts
2421
+ var RETAINED_BYTES_PER_METADATA_BYTE = 2;
2422
+ var RETAINED_BYTES_PER_METADATA_KEY = 160;
2216
2423
  var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
2217
2424
  function indexCapacity(indexLength) {
2218
2425
  return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
@@ -2243,6 +2450,11 @@ function checkedFooter(tail, size) {
2243
2450
  }
2244
2451
  return { footer, fview, storedFooterCrc };
2245
2452
  }
2453
+ async function footerSaysEncrypted(blob) {
2454
+ const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
2455
+ const { fview } = checkedFooter(tail, size);
2456
+ return (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0;
2457
+ }
2246
2458
  function readU64(view, offset, field) {
2247
2459
  const big = view.getBigUint64(offset, true);
2248
2460
  if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
@@ -2251,7 +2463,7 @@ function readU64(view, offset, field) {
2251
2463
  return Number(big);
2252
2464
  }
2253
2465
  var CrbmReader = class _CrbmReader {
2254
- constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc) {
2466
+ constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc, metadata, metadataWeight) {
2255
2467
  this.blob = blob;
2256
2468
  this.objectSize = objectSize;
2257
2469
  this.generation = generation;
@@ -2261,6 +2473,8 @@ var CrbmReader = class _CrbmReader {
2261
2473
  this.servedFromTail = servedFromTail;
2262
2474
  this.crypto = crypto;
2263
2475
  this.footerCrc = footerCrc;
2476
+ this.metadata = metadata;
2477
+ this.metadataWeight = metadataWeight;
2264
2478
  }
2265
2479
  blob;
2266
2480
  objectSize;
@@ -2271,6 +2485,8 @@ var CrbmReader = class _CrbmReader {
2271
2485
  servedFromTail;
2272
2486
  crypto;
2273
2487
  footerCrc;
2488
+ metadata;
2489
+ metadataWeight;
2274
2490
  /**
2275
2491
  * Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
2276
2492
  * objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
@@ -2292,6 +2508,15 @@ var CrbmReader = class _CrbmReader {
2292
2508
  const { keys, cardinalityMinusOne, lengths, crcs, offsets } = this.index;
2293
2509
  return keys.byteLength + cardinalityMinusOne.byteLength + lengths.byteLength + crcs.byteLength + offsets.byteLength;
2294
2510
  }
2511
+ /**
2512
+ * What this reader holds, the weight the storage reader cache bounds on: its parsed index
2513
+ * ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
2514
+ * of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
2515
+ * how the metadata is weighed may change in any release.
2516
+ */
2517
+ get retainedBytes() {
2518
+ return this.retainedIndexBytes + this.metadataWeight;
2519
+ }
2295
2520
  /** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
2296
2521
  cardinalities() {
2297
2522
  const { keys, cardinalityMinusOne } = this.index;
@@ -2358,6 +2583,13 @@ var CrbmReader = class _CrbmReader {
2358
2583
  const chunkCount = fview.getUint32(FOOTER.chunkCount, true);
2359
2584
  const totalCardinality = readU64(fview, FOOTER.totalCardinality, "total_cardinality");
2360
2585
  const generation = readU64(fview, FOOTER.generation, "generation");
2586
+ if (!encrypted && options.crypto !== void 0) {
2587
+ throw new IntegrityError(
2588
+ `.crbm generation ${generation} is not encrypted, but it was opened with a key: a cleartext object where an encrypted one belongs is forged, corrupt, or a write that never published (a crash, before the segment's key was made); a load's collection, dropSegment or deleting the object removes it`
2589
+ );
2590
+ }
2591
+ const versionMinor = footer[FOOTER.versionMinor];
2592
+ const hasExtension = (flags & FLAG_EXTENSION) !== 0;
2361
2593
  if (indexOffset < PAYLOAD_START || indexOffset + indexLength > size - FOOTER_BYTES) {
2362
2594
  throw new IntegrityError(
2363
2595
  `.crbm index region [${indexOffset}, +${indexLength}) out of bounds`
@@ -2368,20 +2600,27 @@ var CrbmReader = class _CrbmReader {
2368
2600
  }
2369
2601
  const tailCoversFrom = size - tail.length;
2370
2602
  if (tailCoversFrom === 0) {
2371
- if (!magicMatches(tail, 0) || tail[4] !== versionMajor) {
2603
+ if (!magicMatches(tail, 0) || tail[4] !== versionMajor || tail[5] !== versionMinor) {
2372
2604
  throw new IntegrityError(".crbm preamble magic/version mismatch with footer");
2373
2605
  }
2374
2606
  }
2375
- let indexBytes;
2607
+ const lowest = hasExtension ? Math.max(PAYLOAD_START, indexOffset - EXT_TRAILER_BYTES - MAX_EXT_BYTES) : indexOffset;
2608
+ let region;
2609
+ let regionStart;
2376
2610
  let servedFromTail;
2377
2611
  if (indexOffset >= tailCoversFrom) {
2378
- const start = indexOffset - tailCoversFrom;
2379
- indexBytes = tail.subarray(start, start + indexLength);
2612
+ region = tail;
2613
+ regionStart = tailCoversFrom;
2380
2614
  servedFromTail = true;
2381
2615
  } else {
2382
- indexBytes = await blob.getRange(indexOffset, indexLength);
2616
+ region = await blob.getRange(lowest, indexOffset + indexLength - lowest);
2617
+ regionStart = lowest;
2383
2618
  servedFromTail = false;
2384
2619
  }
2620
+ const indexBytes = region.subarray(
2621
+ indexOffset - regionStart,
2622
+ indexOffset - regionStart + indexLength
2623
+ );
2385
2624
  if (crc32c(indexBytes) !== indexCrc) {
2386
2625
  throw new IntegrityError(".crbm index CRC mismatch");
2387
2626
  }
@@ -2393,9 +2632,56 @@ var CrbmReader = class _CrbmReader {
2393
2632
  },
2394
2633
  options.crypto.aadFor("index")
2395
2634
  ) : indexBytes;
2635
+ let payloadEnd = indexOffset;
2636
+ let metadata;
2637
+ let metadataLength = 0;
2638
+ if (hasExtension) {
2639
+ const trailerStart = indexOffset - EXT_TRAILER_BYTES;
2640
+ if (trailerStart < PAYLOAD_START) {
2641
+ throw new IntegrityError(".crbm flags an extension block, but has no room for one");
2642
+ }
2643
+ let window = { bytes: region, start: regionStart };
2644
+ if (trailerStart < regionStart) {
2645
+ window = { bytes: await blob.getRange(lowest, indexOffset - lowest), start: lowest };
2646
+ servedFromTail = false;
2647
+ }
2648
+ const trailer = within(window.bytes, window.start, trailerStart, indexOffset);
2649
+ const tview = new DataView(trailer.buffer, trailer.byteOffset, trailer.byteLength);
2650
+ if (!EXT_MAGIC.every((b, i) => trailer[8 + i] === b)) {
2651
+ throw new IntegrityError(".crbm extension block trailer magic mismatch");
2652
+ }
2653
+ const sectionsLength = tview.getUint32(0, true);
2654
+ if (sectionsLength > MAX_EXT_BYTES) {
2655
+ throw new IntegrityError(
2656
+ `.crbm extension block ${sectionsLength}B exceeds cap ${MAX_EXT_BYTES}B`
2657
+ );
2658
+ }
2659
+ const extStart = trailerStart - sectionsLength;
2660
+ if (extStart < PAYLOAD_START) {
2661
+ throw new IntegrityError(`.crbm extension block of ${sectionsLength}B out of bounds`);
2662
+ }
2663
+ let covered;
2664
+ if (extStart >= window.start) {
2665
+ covered = within(window.bytes, window.start, extStart, trailerStart + 4);
2666
+ } else {
2667
+ covered = await blob.getRange(extStart, sectionsLength + 4);
2668
+ servedFromTail = false;
2669
+ if (covered.length !== sectionsLength + 4) {
2670
+ throw new IntegrityError(".crbm extension block read short");
2671
+ }
2672
+ }
2673
+ if (crc32c(covered) !== tview.getUint32(4, true)) {
2674
+ throw new IntegrityError(".crbm extension block CRC mismatch");
2675
+ }
2676
+ ({ metadata, metadataLength } = extensionContents(
2677
+ covered.subarray(0, sectionsLength),
2678
+ options.crypto
2679
+ ));
2680
+ payloadEnd = extStart;
2681
+ }
2396
2682
  const index = parseIndex(
2397
2683
  indexForParse,
2398
- indexOffset,
2684
+ payloadEnd,
2399
2685
  maxPayloadBytes,
2400
2686
  encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
2401
2687
  );
@@ -2426,7 +2712,9 @@ var CrbmReader = class _CrbmReader {
2426
2712
  index,
2427
2713
  servedFromTail,
2428
2714
  options.crypto,
2429
- storedFooterCrc
2715
+ storedFooterCrc,
2716
+ metadata,
2717
+ metadata === void 0 ? 0 : RETAINED_BYTES_PER_METADATA_BYTE * metadataLength + RETAINED_BYTES_PER_METADATA_KEY * Object.keys(metadata).length
2430
2718
  );
2431
2719
  }
2432
2720
  /** Chunk keys present in this generation, ascending. */
@@ -2486,16 +2774,77 @@ var CrbmReader = class _CrbmReader {
2486
2774
  );
2487
2775
  }
2488
2776
  };
2489
- function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1) {
2490
- const capacity = indexCapacity(indexBytes.length);
2491
- const keys = new Uint16Array(capacity);
2492
- const cardinalityMinusOne = new Uint16Array(capacity);
2493
- const lengths = new Uint32Array(capacity);
2494
- const crcs = new Uint32Array(capacity);
2495
- const offsets = new Float64Array(capacity);
2496
- let count = 0;
2777
+ function within(bytes, start, from, to) {
2778
+ if (from < start || to > start + bytes.length) {
2779
+ throw new IntegrityError(".crbm extension block read short");
2780
+ }
2781
+ return bytes.subarray(from - start, to - start);
2782
+ }
2783
+ function extensionCorrupt(message) {
2784
+ throw new IntegrityError(`.crbm extension block: ${message}`);
2785
+ }
2786
+ function extensionContents(sections, crypto) {
2787
+ const view = new DataView(sections.buffer, sections.byteOffset, sections.byteLength);
2788
+ let metadata;
2789
+ let metadataLength = 0;
2790
+ let lastType = 0;
2497
2791
  let pos = 0;
2498
- let prevKey = 0;
2792
+ while (pos < sections.length) {
2793
+ if (sections.length - pos < EXT_SECTION_HEADER_BYTES)
2794
+ extensionCorrupt("a section header is cut off");
2795
+ const type = sections[pos];
2796
+ const length = view.getUint32(pos + 1, true);
2797
+ pos += EXT_SECTION_HEADER_BYTES;
2798
+ if (type <= lastType) {
2799
+ extensionCorrupt(
2800
+ type === 0 ? "section type 0 is not a section type" : `section type ${type} after ${lastType}`
2801
+ );
2802
+ }
2803
+ if (length > sections.length - pos) extensionCorrupt(`section ${type} runs past the block`);
2804
+ const body = sections.subarray(pos, pos + length);
2805
+ pos += length;
2806
+ lastType = type;
2807
+ if (type === EXT_SECTION_METADATA)
2808
+ ({ metadata, metadataLength } = metadataSection(body, crypto));
2809
+ }
2810
+ return { metadata, metadataLength };
2811
+ }
2812
+ function metadataSection(body, crypto) {
2813
+ if (crypto === void 0) {
2814
+ return { metadata: metadataFromBytes(body, extensionCorrupt), metadataLength: body.length };
2815
+ }
2816
+ const framing = AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
2817
+ if (body.length <= framing || body.length > framing + MAX_METADATA_BYTES) {
2818
+ extensionCorrupt(`sealed metadata of ${body.length}B is not a nonce, up to 1 KiB and a tag`);
2819
+ }
2820
+ let plain;
2821
+ try {
2822
+ plain = crypto.aead.open(
2823
+ {
2824
+ nonce: body.subarray(0, AEAD_NONCE_BYTES),
2825
+ ciphertext: body.subarray(AEAD_NONCE_BYTES, body.length - AEAD_TAG_BYTES),
2826
+ tag: body.subarray(body.length - AEAD_TAG_BYTES)
2827
+ },
2828
+ crypto.aadFor("metadata")
2829
+ );
2830
+ } catch (err) {
2831
+ if (!isIntegrityError(err)) throw err;
2832
+ extensionCorrupt(
2833
+ "the sealed metadata does not open: tampered bytes, a block moved from another object, or a CrbmCrypto whose aadFor does not map the 'metadata' scope"
2834
+ );
2835
+ }
2836
+ return { metadata: metadataFromBytes(plain, extensionCorrupt), metadataLength: plain.length };
2837
+ }
2838
+ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1) {
2839
+ const capacity = indexCapacity(indexBytes.length);
2840
+ const keys = new Uint16Array(capacity);
2841
+ const cardinalityMinusOne = new Uint16Array(capacity);
2842
+ const lengths = new Uint32Array(capacity);
2843
+ const crcs = new Uint32Array(capacity);
2844
+ const offsets = new Float64Array(capacity);
2845
+ let count = 0;
2846
+ let pos = 0;
2847
+ let prevKey = 0;
2499
2848
  let prevEnd = PAYLOAD_START;
2500
2849
  let cardinalitySum = 0;
2501
2850
  for (let i = 0; pos < indexBytes.length; i++) {
@@ -2561,6 +2910,9 @@ var CrbmWriter = class {
2561
2910
  `generation must be a non-negative integer; got ${options.generation}`
2562
2911
  );
2563
2912
  }
2913
+ this.metadata = metadataBytes(options.metadata, (message) => {
2914
+ throw new ValidationError(message);
2915
+ });
2564
2916
  }
2565
2917
  sink;
2566
2918
  options;
@@ -2570,6 +2922,11 @@ var CrbmWriter = class {
2570
2922
  totalCardinality = 0;
2571
2923
  preambleWritten = false;
2572
2924
  finished = false;
2925
+ /** The metadata's canonical JSON, copied at construction; `undefined` leaves the footer's extension flag clear. */
2926
+ metadata;
2927
+ /** Bytes written so far, the footer included once {@link finish} has run. */
2928
+ written = 0;
2929
+ footerCrc32c;
2573
2930
  /**
2574
2931
  * Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
2575
2932
  * strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
@@ -2591,7 +2948,8 @@ var CrbmWriter = class {
2591
2948
  }
2592
2949
  if (payload.length === 0) throw new ValidationError("chunk payload must be non-empty");
2593
2950
  await this.ensurePreamble();
2594
- const stored = this.options.crypto ? this.sealChunk(chunkKey, payload) : payload;
2951
+ const crypto = this.options.crypto;
2952
+ const stored = crypto ? framed(crypto.aead.seal(payload, crypto.aadFor(chunkKey))) : payload;
2595
2953
  await this.sink.write(stored);
2596
2954
  this.entries.push({
2597
2955
  chunkKey,
@@ -2604,20 +2962,18 @@ var CrbmWriter = class {
2604
2962
  this.totalCardinality += cardinality;
2605
2963
  this.lastKey = chunkKey;
2606
2964
  }
2607
- /** Seal one chunk payload into the on-disk frame `nonce ‖ ciphertext ‖ tag`, bound to its chunkKey. */
2608
- sealChunk(chunkKey, payload) {
2609
- const crypto = this.options.crypto;
2610
- const sealed = crypto.aead.seal(payload, crypto.aadFor(chunkKey));
2611
- const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
2612
- out.set(sealed.nonce, 0);
2613
- out.set(sealed.ciphertext, sealed.nonce.length);
2614
- out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
2615
- return out;
2616
- }
2617
- /** Write the index region + the fixed footer. After this the object is complete and immutable. */
2965
+ /**
2966
+ * Write the extension block (when there is metadata), the index region and the fixed footer. After this the object
2967
+ * is complete and immutable.
2968
+ */
2618
2969
  async finish() {
2619
2970
  if (this.finished) throw new ValidationError("CrbmWriter already finished");
2620
2971
  await this.ensurePreamble();
2972
+ if (this.metadata !== void 0) {
2973
+ const block = this.extensionBlock(this.metadata);
2974
+ await this.sink.write(block);
2975
+ this.offset += block.length;
2976
+ }
2621
2977
  const indexPlain = this.encodeIndex();
2622
2978
  let indexRegion = indexPlain;
2623
2979
  let indexNonce;
@@ -2633,8 +2989,22 @@ var CrbmWriter = class {
2633
2989
  this.offset += indexRegion.length;
2634
2990
  const footer = this.buildFooter(indexOffset, indexRegion, indexNonce, indexTag);
2635
2991
  await this.sink.write(footer);
2992
+ this.written = this.offset + footer.length;
2993
+ this.footerCrc32c = new DataView(footer.buffer, footer.byteOffset).getUint32(
2994
+ FOOTER.footerCrc32c,
2995
+ true
2996
+ );
2636
2997
  this.finished = true;
2637
2998
  }
2999
+ /**
3000
+ * The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
3001
+ * has to tell its own object from another stored later under the same key compares these with the object's
3002
+ * footer. Throws before {@link finish}.
3003
+ */
3004
+ get identity() {
3005
+ if (this.footerCrc32c === void 0) throw new ValidationError("CrbmWriter not finished");
3006
+ return { size: this.written, footerCrc: this.footerCrc32c };
3007
+ }
2638
3008
  async ensurePreamble() {
2639
3009
  if (this.preambleWritten) return;
2640
3010
  const preamble = new Uint8Array(PREAMBLE_BYTES);
@@ -2644,6 +3014,25 @@ var CrbmWriter = class {
2644
3014
  await this.sink.write(preamble);
2645
3015
  this.preambleWritten = true;
2646
3016
  }
3017
+ /**
3018
+ * The extension block: one metadata section (`u8 type ‖ u32 length ‖ bytes`), then the trailer
3019
+ * (`u32 sectionsLength ‖ u32 crc32c(sections ‖ sectionsLength) ‖ "CRBX"`). Encrypted, the section holds
3020
+ * `nonce ‖ ciphertext ‖ tag`, sealed with the metadata scope of the associated data.
3021
+ */
3022
+ extensionBlock(metadata) {
3023
+ const crypto = this.options.crypto;
3024
+ const body = crypto ? framed(crypto.aead.seal(metadata, crypto.aadFor("metadata"))) : metadata;
3025
+ const sectionsLength = EXT_SECTION_HEADER_BYTES + body.length;
3026
+ const block = new Uint8Array(sectionsLength + EXT_TRAILER_BYTES);
3027
+ const view = new DataView(block.buffer);
3028
+ block[0] = EXT_SECTION_METADATA;
3029
+ view.setUint32(1, body.length, true);
3030
+ block.set(body, EXT_SECTION_HEADER_BYTES);
3031
+ view.setUint32(sectionsLength, sectionsLength, true);
3032
+ view.setUint32(sectionsLength + 4, crc32c(block.subarray(0, sectionsLength + 4)), true);
3033
+ block.set(EXT_MAGIC, sectionsLength + 8);
3034
+ return block;
3035
+ }
2647
3036
  encodeIndex() {
2648
3037
  const out = [];
2649
3038
  let prevKey = 0;
@@ -2671,7 +3060,11 @@ var CrbmWriter = class {
2671
3060
  view.setBigUint64(FOOTER.indexOffset, BigInt(indexOffset), true);
2672
3061
  view.setBigUint64(FOOTER.indexLength, BigInt(indexRegion.length), true);
2673
3062
  view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
2674
- view.setUint32(FOOTER.flags, FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0), true);
3063
+ view.setUint32(
3064
+ FOOTER.flags,
3065
+ FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0) | (this.metadata === void 0 ? 0 : FLAG_EXTENSION),
3066
+ true
3067
+ );
2675
3068
  view.setUint16(
2676
3069
  FOOTER.payloadCodecId,
2677
3070
  this.options.payloadCodecId ?? PAYLOAD_CODEC_ROARING_PORTABLE,
@@ -2691,6 +3084,226 @@ var CrbmWriter = class {
2691
3084
  return footer;
2692
3085
  }
2693
3086
  };
3087
+ function framed(sealed) {
3088
+ const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
3089
+ out.set(sealed.nonce, 0);
3090
+ out.set(sealed.ciphertext, sealed.nonce.length);
3091
+ out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
3092
+ return out;
3093
+ }
3094
+
3095
+ // src/core/load-input.ts
3096
+ var MAX_SERIALIZED_LOAD_BYTES = 4 + 8192 + 8 * 65536 + 65536 * 8192;
3097
+ var DecodedLoadInput = class {
3098
+ constructor(bitmap) {
3099
+ this.bitmap = bitmap;
3100
+ }
3101
+ bitmap;
3102
+ };
3103
+ function tagOf(value) {
3104
+ return Object.prototype.toString.call(value).slice(8, -1);
3105
+ }
3106
+ var VIEW = Object.getPrototypeOf(Uint8Array.prototype);
3107
+ var viewGetter = (name) => Object.getOwnPropertyDescriptor(VIEW, name)?.get;
3108
+ var bufferOf = viewGetter("buffer");
3109
+ var byteOffsetOf = viewGetter("byteOffset");
3110
+ var byteLengthOf = viewGetter("byteLength");
3111
+ function plainView(bytes, what) {
3112
+ let buffer;
3113
+ let offset;
3114
+ let length;
3115
+ try {
3116
+ buffer = bufferOf.call(bytes);
3117
+ offset = byteOffsetOf.call(bytes);
3118
+ length = byteLengthOf.call(bytes);
3119
+ } catch {
3120
+ throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
3121
+ }
3122
+ return length === 0 ? new Uint8Array(0) : new Uint8Array(buffer, offset, length);
3123
+ }
3124
+ var sharedByteLength = typeof SharedArrayBuffer === "function" ? Object.getOwnPropertyDescriptor(SharedArrayBuffer.prototype, "byteLength")?.get : void 0;
3125
+ function isShared(buffer) {
3126
+ if (sharedByteLength === void 0) return false;
3127
+ try {
3128
+ sharedByteLength.call(buffer);
3129
+ return true;
3130
+ } catch {
3131
+ return false;
3132
+ }
3133
+ }
3134
+ function overCap(what, length) {
3135
+ return new ValidationError(
3136
+ `${what} is ${length} bytes, more than any canonical 32-bit bitmap serializes to (${MAX_SERIALIZED_LOAD_BYTES}). Call runOptimize() on the bitmap before serializing it.`
3137
+ );
3138
+ }
3139
+ function prepareLoadInput(input, codec) {
3140
+ if (typeof input !== "object" || input === null) {
3141
+ throw new ValidationError(
3142
+ `a load takes ids, { serialized } or { bitmap }; got ${String(input)}`
3143
+ );
3144
+ }
3145
+ const tag = tagOf(input);
3146
+ if (tag === "Uint8Array" || tag === "Uint8ClampedArray") {
3147
+ throw new ValidationError(
3148
+ `a load's ids cannot be a ${tag} or Buffer: each byte would be loaded as an id. Pass portable Roaring bytes as { serialized }, and ids as a Uint32Array or an array of numbers.`
3149
+ );
3150
+ }
3151
+ if (Symbol.iterator in input || Symbol.asyncIterator in input) return input;
3152
+ const keys = Object.keys(input);
3153
+ if (keys.length === 1 && keys[0] === "serialized") {
3154
+ return decode(input.serialized, "{ serialized }", codec);
3155
+ }
3156
+ if (keys.length === 1 && keys[0] === "bitmap") {
3157
+ const bitmap = input.bitmap;
3158
+ if (typeof bitmap !== "object" || bitmap === null || typeof bitmap.serialize !== "function") {
3159
+ throw new ValidationError(`{ bitmap } needs an object with serialize('portable')`);
3160
+ }
3161
+ const what = "{ bitmap }'s serialize('portable')";
3162
+ if (typeof bitmap.getSerializationSizeInBytes === "function") {
3163
+ const size = bitmap.getSerializationSizeInBytes("portable");
3164
+ if (typeof size === "number" && size > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, size);
3165
+ }
3166
+ return decode(bitmap.serialize("portable"), what, codec);
3167
+ }
3168
+ throw new ValidationError(
3169
+ `a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
3170
+ );
3171
+ }
3172
+ function decode(bytes, what, codec) {
3173
+ if (typeof bytes !== "object" || bytes === null || tagOf(bytes) !== "Uint8Array") {
3174
+ throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
3175
+ }
3176
+ const view = plainView(bytes, what);
3177
+ if (view.byteLength > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, view.byteLength);
3178
+ const own = isShared(view.buffer) ? new Uint8Array(view) : view;
3179
+ try {
3180
+ return new DecodedLoadInput(
3181
+ codec.safeDeserialize(own, MAX_SERIALIZED_LOAD_BYTES, { whole: true })
3182
+ );
3183
+ } catch (err) {
3184
+ if (!isIntegrityError(err)) throw err;
3185
+ throw new ValidationError(`${what} is not a valid bitmap: ${err.message}`);
3186
+ }
3187
+ }
3188
+
3189
+ // src/core/summary.ts
3190
+ var NONCE_BYTES = 12;
3191
+ var TAG_BYTES = 16;
3192
+ var COUNT_BYTES = 8;
3193
+ var MAX_CARDINALITY = 2 ** 32;
3194
+ var BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
3195
+ function toBase64(bytes) {
3196
+ let out = "";
3197
+ for (let i = 0; i < bytes.length; i += 3) {
3198
+ const a = bytes[i];
3199
+ const b = bytes[i + 1];
3200
+ const c = bytes[i + 2];
3201
+ out += BASE64[a >> 2] + BASE64[(a & 3) << 4 | (b ?? 0) >> 4];
3202
+ out += b === void 0 ? "=" : BASE64[(b & 15) << 2 | (c ?? 0) >> 6];
3203
+ out += c === void 0 ? "=" : BASE64[c & 63];
3204
+ }
3205
+ return out;
3206
+ }
3207
+ function fromBase64(text) {
3208
+ if (text.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(text)) return void 0;
3209
+ const pad = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
3210
+ const out = new Uint8Array(text.length / 4 * 3 - pad);
3211
+ let o = 0;
3212
+ for (let i = 0; i < text.length; i += 4) {
3213
+ const n = BASE64.indexOf(text[i]) << 18 | BASE64.indexOf(text[i + 1]) << 12 | Math.max(BASE64.indexOf(text[i + 2]), 0) << 6 | Math.max(BASE64.indexOf(text[i + 3]), 0);
3214
+ if (o < out.length) out[o++] = n >> 16 & 255;
3215
+ if (o < out.length) out[o++] = n >> 8 & 255;
3216
+ if (o < out.length) out[o++] = n & 255;
3217
+ }
3218
+ return out;
3219
+ }
3220
+ function clearSummary(generation, cardinality, metadata) {
3221
+ return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata };
3222
+ }
3223
+ function sealSummary(aead, ref, generation, cardinality, metadata) {
3224
+ const meta = metadataBytes(metadata, (message) => {
3225
+ throw new IntegrityError(`summary metadata: ${message}`);
3226
+ });
3227
+ const plain = new Uint8Array(COUNT_BYTES + (meta?.length ?? 0));
3228
+ new DataView(plain.buffer).setBigUint64(0, BigInt(cardinality), true);
3229
+ if (meta !== void 0) plain.set(meta, COUNT_BYTES);
3230
+ const { nonce, ciphertext, tag } = aead.seal(plain, aadFor(ref, generation, "summary"));
3231
+ const framed2 = new Uint8Array(nonce.length + ciphertext.length + tag.length);
3232
+ framed2.set(nonce, 0);
3233
+ framed2.set(ciphertext, nonce.length);
3234
+ framed2.set(tag, nonce.length + ciphertext.length);
3235
+ return { generation, sealed: toBase64(framed2) };
3236
+ }
3237
+ function openSummary(aead, ref, summary) {
3238
+ const fail = (message) => {
3239
+ throw new IntegrityError(`registry record summary: ${message}`);
3240
+ };
3241
+ const framed2 = fromBase64(summary.sealed);
3242
+ const framing = NONCE_BYTES + TAG_BYTES;
3243
+ if (framed2 === void 0 || framed2.length < framing + COUNT_BYTES || framed2.length > framing + COUNT_BYTES + MAX_METADATA_BYTES) {
3244
+ fail("the sealed summary is not a nonce, a count, up to 1 KiB of metadata and a tag");
3245
+ }
3246
+ const bytes = framed2;
3247
+ let plain;
3248
+ try {
3249
+ plain = aead.open(
3250
+ {
3251
+ nonce: bytes.subarray(0, NONCE_BYTES),
3252
+ ciphertext: bytes.subarray(NONCE_BYTES, bytes.length - TAG_BYTES),
3253
+ tag: bytes.subarray(bytes.length - TAG_BYTES)
3254
+ },
3255
+ aadFor(ref, summary.generation, "summary")
3256
+ );
3257
+ } catch (err) {
3258
+ if (!isIntegrityError(err)) throw err;
3259
+ return fail(
3260
+ "the sealed summary does not open: tampered bytes, or a summary moved from another generation"
3261
+ );
3262
+ }
3263
+ if (plain.length < COUNT_BYTES) fail("the sealed summary is shorter than its count");
3264
+ const count = new DataView(plain.buffer, plain.byteOffset, plain.byteLength).getBigUint64(
3265
+ 0,
3266
+ true
3267
+ );
3268
+ if (count > BigInt(MAX_CARDINALITY)) fail(`cardinality ${count} is outside 0 to 2^32`);
3269
+ const metadata = plain.length === COUNT_BYTES ? void 0 : metadataFromBytes(plain.subarray(COUNT_BYTES), fail);
3270
+ return { cardinality: Number(count), metadata };
3271
+ }
3272
+ function usableSummary(ref, row, aead) {
3273
+ const summary = row.summary;
3274
+ if (summary === void 0 || row.status !== "active" || summary.generation !== row.currentGen) {
3275
+ return void 0;
3276
+ }
3277
+ const keyed = row.wrappedDeks !== void 0 && row.wrappedDeks.length > 0;
3278
+ if ("sealed" in summary) {
3279
+ if (!keyed || aead === void 0) return void 0;
3280
+ try {
3281
+ return openSummary(aead, ref, summary);
3282
+ } catch (err) {
3283
+ if (isIntegrityError(err)) return void 0;
3284
+ throw err;
3285
+ }
3286
+ }
3287
+ if (keyed) return void 0;
3288
+ return { cardinality: summary.cardinality, metadata: summary.metadata };
3289
+ }
3290
+ function canonicalOrNone(metadata) {
3291
+ if (metadata === void 0) return void 0;
3292
+ const json = canonicalMetadataJson(metadata, (message) => {
3293
+ throw new IntegrityError(`metadata: ${message}`);
3294
+ });
3295
+ return json === "{}" ? void 0 : json;
3296
+ }
3297
+ function summaryAgrees(summary, object) {
3298
+ return summary.cardinality === object.cardinality && canonicalOrNone(summary.metadata) === canonicalOrNone(object.metadata);
3299
+ }
3300
+ function metadataToCarry(object, authenticated) {
3301
+ const empty = (metadata) => metadata === void 0 || Object.keys(metadata).length === 0;
3302
+ return empty(object) && authenticated !== void 0 && !empty(authenticated.metadata) ? authenticated.metadata : object;
3303
+ }
3304
+ function summaryOf(ref, generation, description, aead) {
3305
+ return aead === void 0 ? clearSummary(generation, description.cardinality, description.metadata) : sealSummary(aead, ref, generation, description.cardinality, description.metadata);
3306
+ }
2694
3307
 
2695
3308
  // src/core/crbm-storage-source.ts
2696
3309
  function storageBlobReader(driver, key) {
@@ -2709,6 +3322,88 @@ var notThePinned = (ref, generation) => new NotFoundError(
2709
3322
  function versionOf(generation, lineage) {
2710
3323
  return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
2711
3324
  }
3325
+ function summaryOfGeneration(generation, description) {
3326
+ const { cardinality, metadata } = description;
3327
+ return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata: Object.freeze({ ...metadata }) };
3328
+ }
3329
+ var sameKeys = (a, b) => a !== void 0 && JSON.stringify(a) === JSON.stringify(b);
3330
+ function once(fn) {
3331
+ let pending;
3332
+ return () => {
3333
+ if (pending === void 0) {
3334
+ const mine = fn();
3335
+ pending = mine;
3336
+ mine.catch(() => {
3337
+ if (pending === mine) pending = void 0;
3338
+ });
3339
+ }
3340
+ return pending;
3341
+ };
3342
+ }
3343
+ var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
3344
+ function describe(reader) {
3345
+ let cardinality = 0;
3346
+ for (const n of reader.cardinalities().values()) cardinality += n;
3347
+ return { cardinality, metadata: reader.metadata };
3348
+ }
3349
+ var Snapshot = class _Snapshot {
3350
+ constructor(target, open4, prior) {
3351
+ this.target = target;
3352
+ this.open = open4;
3353
+ this.prior = prior;
3354
+ }
3355
+ target;
3356
+ open;
3357
+ prior;
3358
+ /** When the snapshot's refresh clock started. A refresh that failed transiently moves it back, so it lapses sooner. */
3359
+ installedAtMs = 0;
3360
+ /** Set by the cache, to hear of a reader as it is opened. */
3361
+ onReader;
3362
+ opened;
3363
+ static live(target, open4, prior) {
3364
+ return new _Snapshot(target, open4, prior);
3365
+ }
3366
+ static eager(reader) {
3367
+ const snap = new _Snapshot(void 0, void 0, void 0);
3368
+ snap.opened = reader;
3369
+ return snap;
3370
+ }
3371
+ /** The reader, opened by the first call. */
3372
+ get reader() {
3373
+ if (this.opened === void 0) {
3374
+ const target = this.target;
3375
+ this.opened = target.then(async (live) => {
3376
+ if (live === null) return null;
3377
+ const kept = await this.reuse(live);
3378
+ this.prior = void 0;
3379
+ return kept ?? this.open(live);
3380
+ });
3381
+ this.onReader?.(this.opened);
3382
+ }
3383
+ return this.opened;
3384
+ }
3385
+ /** The reader if one has been asked for, without asking for one. */
3386
+ get openedReader() {
3387
+ return this.opened;
3388
+ }
3389
+ /** The prior snapshot's reader, if it is of what this snapshot resolved. */
3390
+ async reuse(live) {
3391
+ const prior = this.prior;
3392
+ const reader = prior === void 0 ? null : await prior.catch(() => null);
3393
+ return reader !== null && readerIsOf(reader, live) ? reader : null;
3394
+ }
3395
+ /** Keep the prior snapshot's reader when the refresh found its generation, so a count does not drop it. */
3396
+ async adopt(live) {
3397
+ if (this.opened !== void 0 || this.prior === void 0) return;
3398
+ const reader = await this.reuse(live);
3399
+ this.prior = void 0;
3400
+ if (reader !== null && this.opened === void 0) {
3401
+ this.opened = Promise.resolve(reader);
3402
+ this.onReader?.(this.opened);
3403
+ }
3404
+ }
3405
+ };
3406
+ var SNAPSHOT_BASE_BYTES = 256;
2712
3407
  var BULK_FLUSH_IDS = 1 << 20;
2713
3408
  var YIELD_EVERY_IDS = 1 << 14;
2714
3409
  var CrbmStorageChunkSource = class {
@@ -2746,6 +3441,10 @@ var CrbmStorageChunkSource = class {
2746
3441
  maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
2747
3442
  clock: clock ?? { now: () => 0 }
2748
3443
  });
3444
+ this.distrusted = new BoundedLru({
3445
+ maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
3446
+ clock: { now: () => 0 }
3447
+ });
2749
3448
  this.replacedPins = new BoundedLru({
2750
3449
  maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
2751
3450
  clock: { now: () => 0 }
@@ -2787,23 +3486,46 @@ var CrbmStorageChunkSource = class {
2787
3486
  * check again.
2788
3487
  */
2789
3488
  invalidations = 0;
3489
+ /**
3490
+ * Generations whose row summary this process found to disagree with their object, by segment and generation. A
3491
+ * summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
3492
+ * long-lived process cannot grow it without limit; one that falls out is trusted again until its next open finds
3493
+ * the disagreement again.
3494
+ */
3495
+ distrusted;
2790
3496
  registry;
2791
3497
  keystore;
2792
3498
  requireEncryption;
2793
3499
  readerOptions;
2794
3500
  clock;
2795
3501
  currentGenTtlMs;
2796
- /** The current resolved reader for a segment, refreshing on the TTL. Cheap within the TTL window. */
2797
- resolvedReader(ref) {
3502
+ /**
3503
+ * The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
3504
+ * target, and its reader once a read has asked for one.
3505
+ */
3506
+ liveSnapshot(ref) {
2798
3507
  const key = segmentKey(ref);
2799
3508
  const existing = this.snapshots.get(key);
2800
- if (existing === void 0) return this.install(key, this.resolveLatest(ref)).reader;
2801
- if (!this.expired(existing.installedAtMs)) return existing.reader;
3509
+ if (existing === void 0) {
3510
+ return this.install(
3511
+ key,
3512
+ Snapshot.live(this.resolveLive(ref), (live) => this.openLive(ref, live))
3513
+ );
3514
+ }
3515
+ if (!this.expired(existing.installedAtMs)) return existing;
2802
3516
  const snap = this.install(
2803
3517
  key,
2804
- this.computeRefreshedReader(ref, existing, () => this.retrySoon(snap))
3518
+ Snapshot.live(
3519
+ this.refreshedTarget(ref, existing, () => this.retrySoon(snap)),
3520
+ (live) => this.openLive(ref, live),
3521
+ existing.openedReader
3522
+ )
2805
3523
  );
2806
- return snap.reader;
3524
+ return snap;
3525
+ }
3526
+ /** The current resolved reader for a segment, refreshing on the TTL. */
3527
+ resolvedReader(ref) {
3528
+ return this.liveSnapshot(ref).reader;
2807
3529
  }
2808
3530
  /**
2809
3531
  * A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
@@ -2813,45 +3535,56 @@ var CrbmStorageChunkSource = class {
2813
3535
  retrySoon(snap) {
2814
3536
  snap.installedAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
2815
3537
  }
2816
- /** TTL elapsed: cheaply re-resolve `currentGen`; reopen only if it actually changed, else reuse the prior reader. */
2817
- async computeRefreshedReader(ref, existing, onTransientFault) {
2818
- const current = await existing.reader.catch(() => null);
2819
- let target;
3538
+ /**
3539
+ * TTL elapsed: cheaply re-resolve the pointer. The row read is always fresh, so what the new snapshot says of the
3540
+ * generation (its summary included) is what the row says now, never the prior snapshot's. The reader is reopened
3541
+ * only if the generation or the row changed, and only when a read asks for it.
3542
+ */
3543
+ async refreshedTarget(ref, existing, onTransientFault) {
3544
+ const prior = await existing.target.catch(() => null);
2820
3545
  try {
2821
- target = await this.resolveTarget(ref);
3546
+ return await this.resolveLive(ref, prior);
2822
3547
  } catch (err) {
2823
3548
  if (!isTransientError(err)) throw err;
2824
- target = void 0;
2825
- }
2826
- if (target === void 0) {
2827
- if (current === null) return this.resolveLatest(ref);
3549
+ const alive = prior !== null;
3550
+ if (!alive) return this.resolveLive(ref);
2828
3551
  onTransientFault();
2829
- return current;
3552
+ return prior;
2830
3553
  }
2831
- if (target === null) return null;
2832
- if (current !== null && target.generation === current.generation && target.lineage === current.lineage) {
2833
- return current;
2834
- }
2835
- return this.openForTarget(ref, target);
2836
3554
  }
2837
3555
  /**
2838
- * Memoize a reader-promise as the segment's snapshot at `now()`, forgetting it later if it resolves to
2839
- * null / throws (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the
2840
- * (pending) promise **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
3556
+ * Memoize a snapshot as the segment's at `now()`, forgetting it later if it resolves to null / throws
3557
+ * (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot
3558
+ * **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
2841
3559
  */
2842
- install(key, reader) {
2843
- const snap = { reader, installedAtMs: this.now() };
3560
+ install(key, snap) {
3561
+ snap.installedAtMs = this.now();
2844
3562
  this.snapshots.set(key, snap);
2845
3563
  const forgetIfStale = () => {
2846
3564
  if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
2847
3565
  };
2848
- reader.then((r) => {
2849
- if (r === null) {
2850
- forgetIfStale();
2851
- return;
2852
- }
2853
- if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedIndexBytes);
2854
- }, forgetIfStale);
3566
+ snap.onReader = (reader) => {
3567
+ reader.then((r) => {
3568
+ if (r === null) {
3569
+ forgetIfStale();
3570
+ return;
3571
+ }
3572
+ if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedBytes);
3573
+ }, forgetIfStale);
3574
+ };
3575
+ if (snap.target !== void 0) {
3576
+ snap.target.then(async (live) => {
3577
+ if (live === null) {
3578
+ forgetIfStale();
3579
+ return;
3580
+ }
3581
+ if (snap.openedReader === void 0 && this.snapshots.peek(key) === snap) {
3582
+ this.snapshots.setWeight(key, live.bytes);
3583
+ }
3584
+ await snap.adopt(live);
3585
+ }, forgetIfStale);
3586
+ }
3587
+ if (snap.openedReader !== void 0) snap.onReader(snap.openedReader);
2855
3588
  return snap;
2856
3589
  }
2857
3590
  now() {
@@ -2883,12 +3616,12 @@ var CrbmStorageChunkSource = class {
2883
3616
  * concurrent sweep. One retry, then propagate — same contract, same eviction rule.
2884
3617
  */
2885
3618
  async currentGeneration(ref) {
2886
- const pending = this.resolvedReader(ref);
3619
+ const snap = this.liveSnapshot(ref);
2887
3620
  try {
2888
- return (await pending)?.generation ?? null;
3621
+ return (await snap.reader)?.generation ?? null;
2889
3622
  } catch (err) {
2890
3623
  if (!isNotFoundError(err)) throw err;
2891
- this.dropStale(segmentKey(ref), pending);
3624
+ this.dropStale(segmentKey(ref), snap);
2892
3625
  return (await this.resolvedReader(ref))?.generation ?? null;
2893
3626
  }
2894
3627
  }
@@ -2908,6 +3641,7 @@ var CrbmStorageChunkSource = class {
2908
3641
  const key = segmentKey(ref);
2909
3642
  this.snapshots.delete(key);
2910
3643
  const pinnedPrefix = `${key}@`;
3644
+ this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
2911
3645
  this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
2912
3646
  this.replacedPins.deleteWhere((k) => k.startsWith(pinnedPrefix));
2913
3647
  for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
@@ -2928,13 +3662,13 @@ var CrbmStorageChunkSource = class {
2928
3662
  * almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
2929
3663
  */
2930
3664
  async currentVersion(ref) {
2931
- const pending = this.resolvedReader(ref);
3665
+ const snap = this.liveSnapshot(ref);
2932
3666
  let reader;
2933
3667
  try {
2934
- reader = await pending;
3668
+ reader = await snap.reader;
2935
3669
  } catch (err) {
2936
3670
  if (!isNotFoundError(err)) throw err;
2937
- this.dropStale(segmentKey(ref), pending);
3671
+ this.dropStale(segmentKey(ref), snap);
2938
3672
  reader = await this.resolvedReader(ref);
2939
3673
  }
2940
3674
  return reader === null ? null : versionOf(reader.generation, reader.lineage);
@@ -2957,34 +3691,38 @@ var CrbmStorageChunkSource = class {
2957
3691
  }
2958
3692
  }
2959
3693
  async pinOnce(ref) {
2960
- const target = await this.resolveTarget(ref);
2961
- if (target === null) return null;
3694
+ const live = await this.resolveLive(ref);
3695
+ if (live === null) return null;
3696
+ const target = live.target;
2962
3697
  const version = versionOf(target.generation, target.lineage);
2963
3698
  let reader;
2964
3699
  let opened = true;
2965
3700
  if (target.lineage === void 0) {
2966
- reader = await this.openForTarget(ref, target);
2967
- this.install(this.pinnedKey(ref, version, reader.fingerprint), Promise.resolve(reader));
3701
+ reader = await this.openLive(ref, live);
3702
+ this.install(
3703
+ this.pinnedKey(ref, version, reader.fingerprint),
3704
+ Snapshot.eager(Promise.resolve(reader))
3705
+ );
2968
3706
  } else {
2969
3707
  const epoch = this.invalidations;
2970
3708
  const key = this.pinnedKey(ref, version);
2971
3709
  let entry = this.snapshots.get(key);
2972
3710
  opened = entry === void 0;
2973
- entry ??= this.install(key, this.openForTarget(ref, target));
3711
+ entry ??= this.install(key, Snapshot.eager(this.openLive(ref, live)));
2974
3712
  reader = await entry.reader;
2975
3713
  if (reader === null) return null;
2976
3714
  const replaced = (r) => this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0;
2977
3715
  const known = !opened && replaced(reader);
2978
3716
  if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
2979
- this.install(key, Promise.resolve(reader));
3717
+ this.install(key, Snapshot.eager(Promise.resolve(reader)));
2980
3718
  }
2981
3719
  if (known) {
2982
3720
  const current = this.snapshots.peek(key);
2983
- let fresh = current !== void 0 && current !== entry ? current : this.install(key, this.openForTarget(ref, target));
3721
+ let fresh = current !== void 0 && current !== entry ? current : this.install(key, Snapshot.eager(this.openLive(ref, live)));
2984
3722
  reader = await fresh.reader;
2985
3723
  if (reader === null) return null;
2986
3724
  if (fresh === current && replaced(reader)) {
2987
- fresh = this.install(key, this.openForTarget(ref, target));
3725
+ fresh = this.install(key, Snapshot.eager(this.openLive(ref, live)));
2988
3726
  reader = await fresh.reader;
2989
3727
  if (reader === null) return null;
2990
3728
  }
@@ -3050,7 +3788,7 @@ var CrbmStorageChunkSource = class {
3050
3788
  if (entry === void 0) {
3051
3789
  if (fingerprint !== void 0 && this.replacedPins.get(this.heldKey(ref, at, fingerprint)) !== void 0)
3052
3790
  throw notThePinned(ref, generation);
3053
- entry = this.install(key, this.openAt(ref, generation));
3791
+ entry = this.install(key, Snapshot.eager(this.openAt(ref, generation)));
3054
3792
  }
3055
3793
  let reader;
3056
3794
  try {
@@ -3185,58 +3923,94 @@ var CrbmStorageChunkSource = class {
3185
3923
  return false;
3186
3924
  }
3187
3925
  /**
3188
- * Evict a snapshot we just failed to read from — matched by its reader *promise*, so a concurrent call's
3926
+ * Evict a snapshot we just failed to read from — matched by the snapshot itself, so a concurrent call's
3189
3927
  * fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
3190
3928
  * wholesale, so comparing anything else would drop a good snapshot on the floor.
3191
3929
  */
3192
- dropStale(key, pending) {
3193
- const cur = this.snapshots.get(key);
3194
- if (cur !== void 0 && cur.reader === pending) this.snapshots.delete(key);
3195
- }
3196
- async resolveLatest(ref) {
3197
- const target = await this.resolveTarget(ref);
3198
- if (target === null) return null;
3199
- return this.openForTarget(ref, target);
3930
+ dropStale(key, snap) {
3931
+ if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
3200
3932
  }
3201
3933
  /** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
3202
3934
  async openForTarget(ref, target) {
3935
+ const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
3936
+ return this.openGeneration(ref, target, crypto);
3937
+ }
3938
+ openGeneration(ref, target, crypto) {
3203
3939
  const genKey = {
3204
3940
  namespace: ref.namespace,
3205
3941
  segment: ref.segment,
3206
3942
  generation: target.generation
3207
3943
  };
3208
- const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
3209
3944
  return openChecked(this.driver, genKey, {
3210
3945
  ...this.readerOptions,
3211
3946
  crypto,
3212
3947
  lineage: target.lineage
3213
3948
  });
3214
3949
  }
3950
+ /**
3951
+ * Open the generation a resolution found, and hold the row's summary against the object while it is open anyway:
3952
+ * the same count and metadata, or the summary is not believed again in this process ({@link distrusted}). This
3953
+ * costs no request; a mismatch is not an error, since the object is the truth and this read has it.
3954
+ */
3955
+ async openLive(ref, live) {
3956
+ const reader = await this.openGeneration(ref, live.target, await live.crypto());
3957
+ const summary = await live.summary();
3958
+ if (summary !== void 0 && !summaryAgrees(summary, describe(reader))) {
3959
+ this.distrusted.set(this.distrustKey(ref, live.target.generation, live.target.lineage), true);
3960
+ }
3961
+ return reader;
3962
+ }
3963
+ distrustKey(ref, generation, lineage) {
3964
+ return `${segmentKey(ref)}@${generation}:${String(lineage)}`;
3965
+ }
3966
+ /** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
3967
+ async trustedSummary(ref, live) {
3968
+ if (this.distrusted.peek(this.distrustKey(ref, live.target.generation, live.target.lineage)) !== void 0) {
3969
+ return void 0;
3970
+ }
3971
+ return live.summary();
3972
+ }
3215
3973
  /**
3216
3974
  * The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
3217
3975
  * `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
3218
- * — there's nowhere a wrapped DEK could live).
3976
+ * — there's nowhere a wrapped DEK could live). The row's summary rides along, to be used or not by whoever
3977
+ * asks.
3219
3978
  */
3220
- async resolveTarget(ref) {
3979
+ async resolveLive(ref, prior) {
3221
3980
  if (this.registry !== void 0) {
3222
3981
  const record = await this.registry.get(ref);
3223
3982
  if (record === null) return null;
3224
3983
  if (record.status === "destroyed") return null;
3225
3984
  if (record.currentGen === null) return null;
3985
+ const generation = record.currentGen;
3986
+ const keys = record.wrappedDeks;
3987
+ const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
3988
+ const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
3226
3989
  return {
3227
- generation: record.currentGen,
3228
- lineage: record.token,
3229
- wrappedDeks: record.wrappedDeks
3990
+ target: { generation, lineage: record.token, wrappedDeks: record.wrappedDeks },
3991
+ crypto,
3992
+ unwrap,
3993
+ // The same checks as a read of the object makes (`requireEncryption`, a keystore for an encrypted row)
3994
+ // come first, then the row's summary, used only if it names this generation in the shape the keys call for.
3995
+ summary: once(async () => usableSummary(ref, record, (await crypto())?.aead)),
3996
+ bytes: SNAPSHOT_BASE_BYTES + (record.summary === void 0 ? 0 : JSON.stringify(record.summary).length)
3230
3997
  };
3231
3998
  }
3232
3999
  let maxGen = -1;
3233
4000
  for await (const key of this.driver.list(ref)) {
3234
4001
  if (key.generation > maxGen) maxGen = key.generation;
3235
4002
  }
3236
- return maxGen < 0 ? null : { generation: maxGen };
4003
+ if (maxGen < 0) return null;
4004
+ return {
4005
+ target: { generation: maxGen },
4006
+ crypto: () => Promise.resolve(void 0),
4007
+ unwrap: void 0,
4008
+ summary: () => Promise.resolve(void 0),
4009
+ bytes: SNAPSHOT_BASE_BYTES
4010
+ };
3237
4011
  }
3238
4012
  /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
3239
- async cryptoForRead(ref, generation, wrappedDeks) {
4013
+ async cryptoForRead(ref, generation, wrappedDeks, unwrap) {
3240
4014
  if (wrappedDeks === void 0 || wrappedDeks.length === 0) {
3241
4015
  if (this.requireEncryption) {
3242
4016
  throw new KeyUnavailableError(
@@ -3245,12 +4019,13 @@ var CrbmStorageChunkSource = class {
3245
4019
  }
3246
4020
  return void 0;
3247
4021
  }
3248
- if (this.keystore === void 0) {
4022
+ const keystore = this.keystore;
4023
+ if (keystore === void 0) {
3249
4024
  throw new KeyUnavailableError(
3250
4025
  `segment "${ref.segment}" is encrypted but this CrbmStorageChunkSource has no keystore`
3251
4026
  );
3252
4027
  }
3253
- const aead = await this.keystore.openDek(wrappedDeks);
4028
+ const aead = await (unwrap ?? (() => keystore.openDek(wrappedDeks)))();
3254
4029
  return { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
3255
4030
  }
3256
4031
  async getChunk(ref) {
@@ -3277,6 +4052,46 @@ var CrbmStorageChunkSource = class {
3277
4052
  null
3278
4053
  );
3279
4054
  }
4055
+ /**
4056
+ * The current generation's number, id count and metadata. The answer comes from the row the snapshot resolved when
4057
+ * it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
4058
+ * (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
4059
+ * object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
4060
+ * With no summary it can use (an older row, one for another generation, one that does not open), it opens the
4061
+ * generation and answers from the object. A row that names an object that is gone still answers, while a read
4062
+ * of the object throws: the number is true of the generation the row names.
4063
+ *
4064
+ * What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
4065
+ * believed until the next time the object is opened, which holds it against the summary.
4066
+ */
4067
+ async summary(ref) {
4068
+ validateUserRef(ref);
4069
+ for (let attempt = 0; attempt < 2; attempt++) {
4070
+ const snap = this.liveSnapshot(ref);
4071
+ try {
4072
+ const live = await snap.target;
4073
+ if (live === null) return null;
4074
+ const row = await this.trustedSummary(ref, live);
4075
+ if (row !== void 0) return summaryOfGeneration(live.target.generation, row);
4076
+ const reader = await snap.reader;
4077
+ return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
4078
+ } catch (err) {
4079
+ if (attempt === 1) throw err;
4080
+ const checks = isNotFoundError(err) || isIntegrityError(err) || isValidationError(err);
4081
+ if (!checks || !isNotFoundError(err) && !await this.replacedUnder(ref, snap.reader, err)) {
4082
+ throw err;
4083
+ }
4084
+ this.dropStale(segmentKey(ref), snap);
4085
+ }
4086
+ }
4087
+ return null;
4088
+ }
4089
+ /** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
4090
+ async summaryAt(ref, generation, held) {
4091
+ validateUserRef(ref);
4092
+ const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
4093
+ return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
4094
+ }
3280
4095
  /**
3281
4096
  * Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
3282
4097
  * generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
@@ -3305,40 +4120,143 @@ var CrbmStorageChunkSource = class {
3305
4120
  */
3306
4121
  async withFreshSnapshot(ref, read, ifGone) {
3307
4122
  for (let attempt = 0; attempt < 2; attempt++) {
3308
- const pending = this.resolvedReader(ref);
4123
+ const snap = this.liveSnapshot(ref);
4124
+ const pending = snap.reader;
3309
4125
  try {
3310
4126
  const reader = await pending;
3311
4127
  if (reader === null) return ifGone;
3312
4128
  return await read(reader);
3313
4129
  } catch (err) {
3314
- if (!isNotFoundError(err) || attempt === 1) throw err;
3315
- this.dropStale(segmentKey(ref), pending);
4130
+ if (attempt === 1) throw err;
4131
+ if (!isNotFoundError(err) && !await this.replacedUnder(ref, pending, err)) throw err;
4132
+ this.dropStale(segmentKey(ref), snap);
3316
4133
  }
3317
4134
  }
3318
4135
  return ifGone;
3319
4136
  }
4137
+ /**
4138
+ * Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
4139
+ * reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
4140
+ * generations above a rolled-back pointer that held the id, a collection pass can stop part-way, and a load then
4141
+ * numbers `currentGen + 1` again), so a reader that still holds the old object's index reads the new object's
4142
+ * bytes at the old offsets: a chunk checksum fails, or a range runs past the end. The object's footer says which it
4143
+ * was, as it does for a pin ({@link CrbmReader.sameObject}): replaced, or gone, re-resolves; the same object means
4144
+ * the error is real, and it stands. Only these two errors pay for the footer read, never a read that succeeded. A
4145
+ * transient fault in the check is rethrown; a footer that cannot be read says nothing, and the error stands.
4146
+ */
4147
+ async replacedUnder(ref, pending, err) {
4148
+ if (!(isIntegrityError(err) || isValidationError(err))) return false;
4149
+ let reader;
4150
+ try {
4151
+ reader = await pending;
4152
+ } catch {
4153
+ return false;
4154
+ }
4155
+ if (reader === null) return false;
4156
+ const at = {
4157
+ namespace: ref.namespace,
4158
+ segment: ref.segment,
4159
+ generation: reader.generation
4160
+ };
4161
+ try {
4162
+ return !await CrbmReader.sameObject(storageBlobReader(this.driver, at), reader.fingerprint);
4163
+ } catch (checkErr) {
4164
+ if (isNotFoundError(checkErr)) return true;
4165
+ if (isTransientError(checkErr)) throw checkErr;
4166
+ return false;
4167
+ }
4168
+ }
3320
4169
  };
3321
- function writeCrbmGeneration(driver, key, chunks, options = {}) {
3322
- const sorted = [...chunks].sort((a, b) => a.chunkKey - b.chunkKey);
4170
+ function* encodeEach(chunks) {
4171
+ for (const { chunkKey, bitmap } of chunks) {
4172
+ if (bitmap.isEmpty) continue;
4173
+ bitmap.optimize?.();
4174
+ yield { chunkKey, payload: bitmap.serialize(), cardinality: bitmap.size };
4175
+ }
4176
+ }
4177
+ async function writeEncodedChunks(driver, key, chunks, options) {
3323
4178
  const tick = yieldEvery(options.clock);
3324
- return driver.putImmutable(key, async (sink) => {
3325
- const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
3326
- for (const { chunkKey, bitmap } of sorted) {
3327
- if (bitmap.isEmpty) continue;
3328
- bitmap.optimize?.();
3329
- await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
4179
+ let chunkCount = 0;
4180
+ let cardinality = 0;
4181
+ let identity;
4182
+ const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
4183
+ const writer = new CrbmWriter(sink, {
4184
+ generation: key.generation,
4185
+ crypto: options.crypto,
4186
+ metadata: options.metadata
4187
+ });
4188
+ for (const chunk of chunks) {
4189
+ await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
4190
+ chunkCount++;
4191
+ cardinality += chunk.cardinality;
3330
4192
  const pause = tick();
3331
4193
  if (pause !== null) await pause;
3332
4194
  }
3333
4195
  await writer.finish();
4196
+ identity = writer.identity;
3334
4197
  });
4198
+ if (identity === void 0) {
4199
+ throw new IntegrityError(
4200
+ `the driver committed ${key.segment}.${key.generation} without writing it`
4201
+ );
4202
+ }
4203
+ return {
4204
+ size,
4205
+ sha256,
4206
+ chunkCount,
4207
+ cardinality,
4208
+ fingerprint: fingerprintFor(identity.size, identity.footerCrc)
4209
+ };
4210
+ }
4211
+ async function holdsObject(storage, key, fingerprint) {
4212
+ try {
4213
+ return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
4214
+ } catch {
4215
+ return false;
4216
+ }
4217
+ }
4218
+ function objectIsEncrypted(storage, key) {
4219
+ return footerSaysEncrypted(storageBlobReader(storage, key));
4220
+ }
4221
+ function rememberingTail(blob) {
4222
+ let first;
4223
+ return {
4224
+ getRange: (offset, length) => blob.getRange(offset, length),
4225
+ async getTail(maxBytes) {
4226
+ if (first !== void 0 && (maxBytes <= first.bytes.length || first.bytes.length === first.size)) {
4227
+ const take = Math.min(maxBytes, first.bytes.length);
4228
+ return { bytes: first.bytes.subarray(first.bytes.length - take), size: first.size };
4229
+ }
4230
+ const read = await blob.getTail(maxBytes);
4231
+ first ??= read;
4232
+ return read;
4233
+ }
4234
+ };
4235
+ }
4236
+ async function openRollbackTarget(storage, key, crypto) {
4237
+ const blob = rememberingTail(storageBlobReader(storage, key));
4238
+ await blob.getTail(DEFAULT_TAIL_BYTES);
4239
+ const encrypted = await footerSaysEncrypted(blob);
4240
+ if (encrypted !== (crypto !== void 0)) return { encrypted, reader: void 0 };
4241
+ const reader = await CrbmReader.open(blob, { crypto });
4242
+ if (reader.generation !== key.generation) {
4243
+ throw new IntegrityError(
4244
+ `segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
4245
+ );
4246
+ }
4247
+ return { encrypted, reader };
3335
4248
  }
3336
4249
  async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
3337
4250
  const chunkKeys = [];
3338
4251
  let cardinality = 0;
3339
4252
  const tick = yieldEvery(options.clock);
4253
+ let identity;
3340
4254
  const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
3341
- const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
4255
+ const writer = new CrbmWriter(sink, {
4256
+ generation: key.generation,
4257
+ crypto: options.crypto,
4258
+ metadata: options.metadata
4259
+ });
3342
4260
  for await (const { chunkKey, bitmap } of chunks) {
3343
4261
  if (bitmap.isEmpty) continue;
3344
4262
  bitmap.optimize?.();
@@ -3349,12 +4267,57 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
3349
4267
  if (pause !== null) await pause;
3350
4268
  }
3351
4269
  await writer.finish();
4270
+ identity = writer.identity;
3352
4271
  });
3353
- return { size, sha256, chunkKeys, cardinality };
4272
+ if (identity === void 0) {
4273
+ throw new IntegrityError(
4274
+ `the driver committed ${key.segment}.${key.generation} without writing it`
4275
+ );
4276
+ }
4277
+ return {
4278
+ size,
4279
+ sha256,
4280
+ chunkKeys,
4281
+ cardinality,
4282
+ fingerprint: fingerprintFor(identity.size, identity.footerCrc)
4283
+ };
4284
+ }
4285
+ function refuseCleartextOntoKey(key, record, cleartext) {
4286
+ if (cleartext !== true || record.wrappedDeks === void 0 || record.wrappedDeks.length === 0) {
4287
+ return;
4288
+ }
4289
+ throw new KeyUnavailableError(
4290
+ `segment "${key.segment}" is encrypted, and generation ${key.generation} was written in cleartext: another writer created the segment with a key while this one was writing. Nothing was published. Re-run the write with the keystore that holds the segment's key.`
4291
+ );
3354
4292
  }
4293
+ var UNANSWERED_RESENDS = 3;
4294
+ var UNANSWERED_RESEND_BASE_MS = 500;
3355
4295
  async function publishGeneration(registry, key, options = {}) {
4296
+ let fresh;
4297
+ let failedOn;
4298
+ let unanswered;
4299
+ let resends = 0;
4300
+ let sawUnanswered = false;
4301
+ let lastRecord;
3356
4302
  for (let attempt = 0; attempt < 5; attempt++) {
3357
- const record = await registry.get(key);
4303
+ const reused = attempt === 0 && options.row !== void 0;
4304
+ const record = fresh !== void 0 ? fresh : reused ? options.row ?? null : await registry.get(key);
4305
+ fresh = void 0;
4306
+ unanswered = void 0;
4307
+ lastRecord = record;
4308
+ if (failedOn !== void 0) {
4309
+ let landed;
4310
+ try {
4311
+ landed = await landedHere(record, failedOn, key, options.holdsOwnObject);
4312
+ } catch (proofErr) {
4313
+ throw outcomeUnknown(key, proofErr);
4314
+ }
4315
+ if (landed) return true;
4316
+ if (record !== null && record.status === "active" && record.currentGen === key.generation) {
4317
+ return false;
4318
+ }
4319
+ failedOn = void 0;
4320
+ }
3358
4321
  try {
3359
4322
  if (options.expectFrom !== void 0 && record?.currentGen !== options.expectFrom) {
3360
4323
  return false;
@@ -3365,103 +4328,138 @@ async function publishGeneration(registry, key, options = {}) {
3365
4328
  if (options.expectToken !== void 0 && record?.token !== options.expectToken) {
3366
4329
  return false;
3367
4330
  }
4331
+ if (record === null && options.summary !== void 0 && "sealed" in options.summary && options.wrappedDeks === void 0) {
4332
+ if (reused) continue;
4333
+ return false;
4334
+ }
3368
4335
  if (record === null) {
3369
4336
  await registry.create(key, {
3370
4337
  currentGen: key.generation,
3371
- wrappedDeks: options.wrappedDeks
4338
+ wrappedDeks: options.wrappedDeks,
4339
+ ...options.summary === void 0 ? {} : { summary: options.summary }
3372
4340
  });
3373
4341
  } else if (record.status === "destroyed") {
3374
4342
  throw new ValidationError(
3375
4343
  `segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
3376
4344
  );
3377
4345
  } else if (record.currentGen === null) {
4346
+ refuseCleartextOntoKey(key, record, options.cleartext);
3378
4347
  await registry.compareAndSwap(key, record.token, {
3379
4348
  currentGen: key.generation,
3380
- ...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks }
4349
+ ...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks },
4350
+ ...options.summary === void 0 ? {} : { summary: options.summary }
3381
4351
  });
3382
4352
  } else if (record.currentGen > key.generation) {
3383
4353
  return false;
3384
4354
  } else if (record.currentGen === key.generation) {
4355
+ if (reused) continue;
3385
4356
  return true;
3386
4357
  } else {
3387
4358
  if (options.wrappedDeks !== void 0 && options.wrappedDeks.length > 0) {
3388
4359
  throw new ValidationError(
3389
- `publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen} \u2014 its existing generations are not encrypted under this key. Encrypt a new segment and load into that instead.`
4360
+ `publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen}: its generations are not encrypted under this key, and nothing was published. If another writer published to the segment while this one was writing, re-run the write, which then uses the segment's own key; if the segment's generations are cleartext, encrypt a new segment and load into that instead.`
3390
4361
  );
3391
4362
  }
3392
- await registry.compareAndSwap(key, record.token, { currentGen: key.generation });
4363
+ refuseCleartextOntoKey(key, record, options.cleartext);
4364
+ await registry.compareAndSwap(key, record.token, {
4365
+ currentGen: key.generation,
4366
+ ...options.summary === void 0 ? {} : { summary: options.summary }
4367
+ });
3393
4368
  }
3394
4369
  return true;
3395
4370
  } catch (err) {
3396
- if (isWriteConflictError(err)) continue;
3397
- throw err;
4371
+ if (isWriteConflictError(err)) {
4372
+ failedOn = record;
4373
+ continue;
4374
+ }
4375
+ if (!isTransientError(err)) throw err;
4376
+ options.onUnanswered?.();
4377
+ sawUnanswered = true;
4378
+ let now;
4379
+ try {
4380
+ now = await registry.get(key);
4381
+ } catch (readErr) {
4382
+ if (isIntegrityError(readErr)) throw readErr;
4383
+ throw outcomeUnknown(key, err);
4384
+ }
4385
+ if (sameRow(now, record)) {
4386
+ const sleep = options.clock?.sleep;
4387
+ if (sleep === void 0 || resends >= UNANSWERED_RESENDS) throw outcomeUnknown(key, err);
4388
+ const bound = UNANSWERED_RESEND_BASE_MS * 2 ** resends;
4389
+ resends += 1;
4390
+ await sleep.call(options.clock, Math.floor((options.rng?.next() ?? 1) * bound));
4391
+ unanswered = outcomeUnknown(key, err);
4392
+ fresh = now;
4393
+ continue;
4394
+ }
4395
+ failedOn = record;
4396
+ fresh = now;
3398
4397
  }
3399
4398
  }
3400
4399
  const final = await registry.get(key);
3401
- if (final !== null && final.currentGen !== null && final.currentGen >= key.generation)
3402
- return final.currentGen === key.generation;
4400
+ if (final !== null && final.currentGen !== null && final.currentGen >= key.generation) {
4401
+ if (final.currentGen !== key.generation) return false;
4402
+ if (!sawUnanswered) return true;
4403
+ try {
4404
+ return await landedHere(final, lastRecord ?? null, key, options.holdsOwnObject);
4405
+ } catch (proofErr) {
4406
+ throw outcomeUnknown(key, proofErr);
4407
+ }
4408
+ }
4409
+ if (unanswered !== void 0) throw unanswered;
3403
4410
  throw new WriteConflictError(
3404
4411
  `publishGeneration: contention setting currentGen for ${key.segment}`
3405
4412
  );
3406
4413
  }
4414
+ async function landedHere(now, before, key, holdsOwnObject) {
4415
+ if (now === null || now.status === "destroyed" || now.currentGen !== key.generation) return false;
4416
+ if (before !== null && !sameIncarnation(before, now)) return false;
4417
+ if (holdsOwnObject === void 0) {
4418
+ throw new TransientError(
4419
+ `publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled, since the caller gave no way to tell its own object from another's. Its object is kept; re-run the write, which numbers past it.`
4420
+ );
4421
+ }
4422
+ return holdsOwnObject();
4423
+ }
4424
+ function sameRow(a, b) {
4425
+ return a === null ? b === null : b !== null && a.token === b.token;
4426
+ }
4427
+ function outcomeUnknown(key, cause) {
4428
+ if (isTransientError(cause)) return cause;
4429
+ return new TransientError(
4430
+ `publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled (${cause?.name ?? "error"} while checking its object). Its object is kept, since the write may still land; re-run the write, which numbers past it.`,
4431
+ { cause }
4432
+ );
4433
+ }
4434
+ async function provesOwnObject(storage, key, fingerprint) {
4435
+ try {
4436
+ return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
4437
+ } catch (err) {
4438
+ if (isNotFoundError(err)) return false;
4439
+ throw err;
4440
+ }
4441
+ }
3407
4442
  async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
3408
4443
  if (options.keystore === void 0 && options.requireEncryption === true) {
3409
4444
  throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
3410
4445
  }
3411
4446
  const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
3412
- const byChunk = /* @__PURE__ */ new Map();
3413
- const pendingByChunk = /* @__PURE__ */ new Map();
3414
- let pending = 0;
3415
- const tickChunk = yieldEvery(options.clock);
3416
- const tickId = yieldEvery(options.clock, YIELD_EVERY_IDS);
3417
- const flushPending = async () => {
3418
- for (const [chunkKey, rems] of pendingByChunk) {
3419
- if (rems.length === 0) continue;
3420
- const existingBitmap = byChunk.get(chunkKey);
3421
- if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
3422
- else existingBitmap.addMany(rems);
3423
- rems.length = 0;
3424
- const pause = tickChunk();
3425
- if (pause !== null) await pause;
3426
- }
3427
- pending = 0;
3428
- };
3429
- const ingest = (id) => {
3430
- const { chunkKey, remainder } = splitId(id);
3431
- let bucket = pendingByChunk.get(chunkKey);
3432
- if (bucket === void 0) {
3433
- bucket = [];
3434
- pendingByChunk.set(chunkKey, bucket);
3435
- }
3436
- bucket.push(remainder);
3437
- return ++pending >= BULK_FLUSH_IDS;
3438
- };
3439
- if (Symbol.asyncIterator in ids) {
3440
- for await (const id of ids) {
3441
- if (ingest(id)) await flushPending();
3442
- const pause = tickId();
3443
- if (pause !== null) await pause;
3444
- }
4447
+ let chunks;
4448
+ if (ids instanceof DecodedLoadInput && ids.bitmap.encodeChunks !== void 0) {
4449
+ const pause = yieldEvery(options.clock, 1);
4450
+ await pause();
4451
+ chunks = ids.bitmap.encodeChunks();
4452
+ await pause();
3445
4453
  } else {
3446
- for (const id of ids) {
3447
- if (ingest(id)) await flushPending();
3448
- const pause = tickId();
3449
- if (pause !== null) await pause;
3450
- }
3451
- }
3452
- await flushPending();
3453
- let cardinality = 0;
3454
- const chunks = [];
3455
- for (const [chunkKey, bitmap] of byChunk) {
3456
- cardinality += bitmap.size;
3457
- chunks.push({ chunkKey, bitmap });
3458
- const pause = tickChunk();
3459
- if (pause !== null) await pause;
4454
+ chunks = encodeEach(
4455
+ await bucketIds(ids instanceof DecodedLoadInput ? ids.bitmap : ids, codec, options.clock)
4456
+ );
3460
4457
  }
3461
4458
  if (options.keystore !== void 0 && options.registry === void 0) {
3462
4459
  throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
3463
4460
  }
3464
- const existing = options.registry !== void 0 ? await options.registry.get(key) : null;
4461
+ const passed = options.row;
4462
+ const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(key) : null;
3465
4463
  if (existing?.status === "destroyed") {
3466
4464
  throw new ValidationError(
3467
4465
  `segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
@@ -3490,13 +4488,25 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
3490
4488
  crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
3491
4489
  }
3492
4490
  }
3493
- const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, {
3494
- crypto,
3495
- clock: options.clock
3496
- });
4491
+ const { size, sha256, chunkCount, cardinality, fingerprint } = await writeEncodedChunks(
4492
+ driver,
4493
+ key,
4494
+ chunks,
4495
+ { crypto, clock: options.clock, metadata: options.metadata }
4496
+ );
4497
+ const summary = summaryOf(
4498
+ key,
4499
+ key.generation,
4500
+ { cardinality, metadata: options.metadata },
4501
+ crypto?.aead
4502
+ );
3497
4503
  if (options.registry !== void 0 && options.publish !== false) {
3498
4504
  const becameCurrent = await publishGeneration(options.registry, key, {
3499
- wrappedDeks: newWrapped
4505
+ wrappedDeks: newWrapped,
4506
+ summary,
4507
+ cleartext: crypto === void 0,
4508
+ holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
4509
+ clock: options.clock
3500
4510
  });
3501
4511
  if (becameCurrent) {
3502
4512
  safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
@@ -3506,16 +4516,72 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
3506
4516
  generation: key.generation
3507
4517
  });
3508
4518
  }
3509
- return {
3510
- size,
3511
- sha256,
3512
- chunkCount: chunks.length,
3513
- cardinality,
3514
- becameCurrent,
3515
- wrappedDeks: newWrapped
3516
- };
4519
+ return {
4520
+ size,
4521
+ sha256,
4522
+ chunkCount,
4523
+ cardinality,
4524
+ summary,
4525
+ becameCurrent,
4526
+ wrappedDeks: newWrapped,
4527
+ encrypted: crypto !== void 0,
4528
+ fingerprint
4529
+ };
4530
+ }
4531
+ return {
4532
+ size,
4533
+ sha256,
4534
+ chunkCount,
4535
+ cardinality,
4536
+ summary,
4537
+ wrappedDeks: newWrapped,
4538
+ encrypted: crypto !== void 0,
4539
+ fingerprint
4540
+ };
4541
+ }
4542
+ async function bucketIds(ids, codec, clock) {
4543
+ const byChunk = /* @__PURE__ */ new Map();
4544
+ const pendingByChunk = /* @__PURE__ */ new Map();
4545
+ let pending = 0;
4546
+ const tickChunk = yieldEvery(clock);
4547
+ const tickId = yieldEvery(clock, YIELD_EVERY_IDS);
4548
+ const flushPending = async () => {
4549
+ for (const [chunkKey, rems] of pendingByChunk) {
4550
+ if (rems.length === 0) continue;
4551
+ const existingBitmap = byChunk.get(chunkKey);
4552
+ if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
4553
+ else existingBitmap.addMany(rems);
4554
+ rems.length = 0;
4555
+ const pause = tickChunk();
4556
+ if (pause !== null) await pause;
4557
+ }
4558
+ pending = 0;
4559
+ };
4560
+ const ingest = (id) => {
4561
+ const { chunkKey, remainder } = splitId(id);
4562
+ let bucket = pendingByChunk.get(chunkKey);
4563
+ if (bucket === void 0) {
4564
+ bucket = [];
4565
+ pendingByChunk.set(chunkKey, bucket);
4566
+ }
4567
+ bucket.push(remainder);
4568
+ return ++pending >= BULK_FLUSH_IDS;
4569
+ };
4570
+ if (Symbol.asyncIterator in ids) {
4571
+ for await (const id of ids) {
4572
+ if (ingest(id)) await flushPending();
4573
+ const pause = tickId();
4574
+ if (pause !== null) await pause;
4575
+ }
4576
+ } else {
4577
+ for (const id of ids) {
4578
+ if (ingest(id)) await flushPending();
4579
+ const pause = tickId();
4580
+ if (pause !== null) await pause;
4581
+ }
3517
4582
  }
3518
- return { size, sha256, chunkCount: chunks.length, cardinality, wrappedDeks: newWrapped };
4583
+ await flushPending();
4584
+ return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
3519
4585
  }
3520
4586
  function openGenerationReader(storage, key, crypto, options = {}) {
3521
4587
  return openChecked(storage, key, { ...options, crypto });
@@ -3544,6 +4610,11 @@ async function verifyGeneration(storage, key, expected, crypto) {
3544
4610
  `verify failed for ${key.segment}.${key.generation}: cardinality ${reader.count()} != ${expected.cardinality}`
3545
4611
  );
3546
4612
  }
4613
+ if (!summaryAgrees(expected, { cardinality: reader.count(), metadata: reader.metadata })) {
4614
+ throw new IntegrityError(
4615
+ `verify failed for ${key.segment}.${key.generation}: the object's metadata is not the metadata it was written with`
4616
+ );
4617
+ }
3547
4618
  }
3548
4619
 
3549
4620
  // src/core/pinned-storage-source.ts
@@ -3592,6 +4663,11 @@ var PinnedStorageChunkSource = class {
3592
4663
  if (pin === void 0) return this.inner.cardinalities(ref);
3593
4664
  return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
3594
4665
  }
4666
+ summary(ref) {
4667
+ const pin = this.pinFor(ref);
4668
+ if (pin === void 0) return this.inner.summary(ref);
4669
+ return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
4670
+ }
3595
4671
  currentGeneration(ref) {
3596
4672
  const pin = this.pinFor(ref);
3597
4673
  return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
@@ -3628,67 +4704,58 @@ var PinnedStorageChunkSource = class {
3628
4704
  }
3629
4705
  };
3630
4706
 
3631
- // src/core/generation-gc.ts
3632
- async function nextGeneration(ref, deps) {
3633
- const record = await deps.registry.get(ref);
3634
- let highest = record?.currentGen ?? -1;
3635
- for await (const key of deps.storage.list(ref)) {
3636
- if (key.generation > highest) highest = key.generation;
3637
- }
3638
- return highest + 1;
4707
+ // src/core/retry.ts
4708
+ var DEFAULT_RETRY_POLICY = {
4709
+ maxAttempts: 4,
4710
+ baseDelayMs: 50,
4711
+ maxDelayMs: 2e3,
4712
+ backoffFactor: 2,
4713
+ jitter: "full"
4714
+ };
4715
+ function backoffDelayMs(policy, attempt) {
4716
+ const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
4717
+ return Math.min(policy.maxDelayMs, raw);
3639
4718
  }
3640
- async function gcOrphanGenerations(ref, deps, options = {}) {
3641
- const keep = options.keep ?? 1;
3642
- if (!Number.isInteger(keep) || keep < 0) {
3643
- throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
3644
- }
3645
- const record = await deps.registry.get(ref);
3646
- if (record === null) return [];
3647
- const current = record.currentGen;
3648
- const seen = /* @__PURE__ */ new Set();
3649
- for await (const key of deps.storage.list(ref)) seen.add(key.generation);
3650
- const gens = [...seen];
3651
- const after = await deps.registry.get(ref);
3652
- if (after === null)
3653
- throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
3654
- if (record.status === "destroyed" && after.token !== record.token) {
3655
- throw new WriteConflictError(
3656
- `segment ${ref.segment} changed incarnation while its generations were being listed`
4719
+ function applyJitter(policy, delayMs, rng) {
4720
+ if (policy.jitter === "none") return delayMs;
4721
+ return rng.next() * delayMs;
4722
+ }
4723
+ function retryRead(op, retry) {
4724
+ return retry === void 0 ? op() : withRetry(op, retry.policy ?? DEFAULT_RETRY_POLICY, retry);
4725
+ }
4726
+ async function withRetry(op, policy, deps) {
4727
+ const retryable = deps.isRetryable ?? isTransientError;
4728
+ if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
4729
+ throw new ValidationError(
4730
+ `retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
3657
4731
  );
3658
4732
  }
3659
- const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
3660
- const stillCollectable = async () => {
3661
- const still = await deps.registry.get(ref);
3662
- if (still === null) {
3663
- throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
3664
- }
3665
- const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
3666
- if (!ok) {
3667
- throw new WriteConflictError(
3668
- `segment ${ref.segment} changed incarnation while its generations were being collected`
3669
- );
4733
+ const attempts = Math.floor(policy.maxAttempts);
4734
+ let lastErr;
4735
+ for (let attempt = 1; attempt <= attempts; attempt++) {
4736
+ try {
4737
+ return await op();
4738
+ } catch (err) {
4739
+ lastErr = err;
4740
+ if (attempt >= attempts || !retryable(err)) throw err;
4741
+ const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
4742
+ try {
4743
+ deps.onRetry?.({ attempt, delayMs, err });
4744
+ } catch {
4745
+ }
4746
+ await deps.clock.sleep(delayMs);
3670
4747
  }
3671
- };
3672
- const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
3673
- // No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
3674
- // here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
3675
- // that publish into a dangling pointer. It is collected once a pointer exists.
3676
- []
3677
- ) : (
3678
- // Delete generations below the cutoff, except the newest `keep` of them (the grace window).
3679
- gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
3680
- );
3681
- for (const generation of toDelete) {
3682
- await stillCollectable();
3683
- await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
3684
4748
  }
3685
- return toDelete;
4749
+ throw lastErr;
3686
4750
  }
3687
4751
 
3688
4752
  // src/core/load.ts
3689
4753
  async function currentCardinality(ref, deps, record) {
3690
- if (record === null || record.currentGen === null || record.status === "destroyed") return null;
4754
+ if (record === null || record.currentGen === null || record.status === "destroyed") {
4755
+ return { cardinality: null, fromSummary: false };
4756
+ }
3691
4757
  const generation = record.currentGen;
4758
+ let aead;
3692
4759
  let crypto;
3693
4760
  const wrapped = record.wrappedDeks;
3694
4761
  if (wrapped !== void 0 && wrapped.length > 0) {
@@ -3697,23 +4764,29 @@ async function currentCardinality(ref, deps, record) {
3697
4764
  `segment "${ref.segment}" is encrypted but load was given no keystore`
3698
4765
  );
3699
4766
  }
3700
- const aead = await deps.keystore.openDek(wrapped);
3701
- crypto = { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
4767
+ aead = await deps.keystore.openDek(wrapped);
4768
+ const opened = aead;
4769
+ crypto = { aead: opened, aadFor: (scope) => aadFor(ref, generation, scope) };
3702
4770
  }
4771
+ const described = usableSummary(ref, record, aead);
4772
+ if (described !== void 0) return { cardinality: described.cardinality, fromSummary: true };
3703
4773
  let reader;
3704
4774
  try {
3705
- reader = await openGenerationReader(deps.storage, { ...ref, generation }, crypto);
4775
+ reader = await retryRead(
4776
+ () => openGenerationReader(deps.storage, { ...ref, generation }, crypto),
4777
+ deps.readRetry
4778
+ );
3706
4779
  } catch (err) {
3707
4780
  if (!isNotFoundError(err)) throw err;
3708
- return null;
4781
+ return { cardinality: null, fromSummary: false };
3709
4782
  }
3710
4783
  let total = 0;
3711
4784
  for (const n of reader.cardinalities().values()) total += n;
3712
- return total;
4785
+ return { cardinality: total, fromSummary: false };
3713
4786
  }
3714
- async function loadSegment(ref, ids, deps, options = {}) {
4787
+ async function loadSegment(ref, input, deps, options = {}) {
3715
4788
  validateUserRef(ref);
3716
- requireCodec(deps.codec, "loadSegment");
4789
+ const codec = requireCodec(deps.codec, "loadSegment");
3717
4790
  const keep = options.keep ?? 1;
3718
4791
  if (!Number.isInteger(keep) || keep < 0) {
3719
4792
  throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
@@ -3733,13 +4806,19 @@ async function loadSegment(ref, ids, deps, options = {}) {
3733
4806
  throw new ValidationError(`guard.minRetained must be a fraction in 0..1; got ${String(s)}`);
3734
4807
  }
3735
4808
  }
4809
+ const metadata = copiedMetadata(options.metadata, (message) => {
4810
+ throw new ValidationError(message);
4811
+ });
3736
4812
  const audit = safeAudit(options.audit ?? NOOP_AUDIT);
4813
+ const ids = prepareLoadInput(input, codec);
4814
+ assertRegistryCanWrite(deps.registry, "load");
3737
4815
  const row = await deps.registry.get(ref);
3738
4816
  const fromToken = row?.token;
3739
4817
  const fromGeneration = row?.currentGen ?? void 0;
3740
4818
  const needsBefore = guard?.minRetained !== void 0 || options.allowEmpty !== true;
3741
- const before = needsBefore ? await currentCardinality(ref, deps, row) : null;
3742
- const generation = await nextGeneration(ref, deps);
4819
+ const current = needsBefore ? await currentCardinality(ref, deps, row) : { cardinality: null, fromSummary: false };
4820
+ const before = current.cardinality;
4821
+ const { generation, checked } = await nextLoadGeneration(ref, deps, row);
3743
4822
  const key = { namespace: ref.namespace, segment: ref.segment, generation };
3744
4823
  let written;
3745
4824
  try {
@@ -3749,7 +4828,9 @@ async function loadSegment(ref, ids, deps, options = {}) {
3749
4828
  keystore: deps.keystore,
3750
4829
  requireEncryption: deps.requireEncryption,
3751
4830
  codec: deps.codec,
3752
- clock: deps.clock
4831
+ clock: deps.clock,
4832
+ metadata,
4833
+ row
3753
4834
  });
3754
4835
  } catch (err) {
3755
4836
  if (!isWriteConflictError(err)) throw err;
@@ -3773,16 +4854,28 @@ async function loadSegment(ref, ids, deps, options = {}) {
3773
4854
  collected: []
3774
4855
  };
3775
4856
  }
3776
- const refuse = async (reason) => {
4857
+ const reclaim = async () => {
3777
4858
  const now = await deps.registry.get(ref);
3778
- if (now === null || now.token === fromToken) await deps.storage.delete(key);
4859
+ if (now !== null && (now.token === fromToken || now.status === "destroyed")) {
4860
+ await deps.storage.delete(key);
4861
+ return;
4862
+ }
4863
+ const keyed = now !== null && now.wrappedDeks !== void 0 && now.wrappedDeks.length > 0;
4864
+ if ((now === null || keyed && !written.encrypted) && await holdsObject(deps.storage, key, written.fingerprint)) {
4865
+ await deps.storage.delete(key);
4866
+ }
4867
+ };
4868
+ let unanswered = false;
4869
+ const refuse = async (reason) => {
4870
+ await reclaim();
3779
4871
  audit.onEvent({
3780
4872
  kind: "segment.load-refused",
3781
4873
  segment: ref.segment,
3782
4874
  namespace: ref.namespace,
3783
4875
  generation,
3784
4876
  reason,
3785
- cardinality: written.cardinality
4877
+ cardinality: written.cardinality,
4878
+ ...unanswered ? { unanswered: true } : {}
3786
4879
  });
3787
4880
  return {
3788
4881
  generation,
@@ -3805,19 +4898,37 @@ async function loadSegment(ref, ids, deps, options = {}) {
3805
4898
  if (guard?.minRetained !== void 0 && before !== null) {
3806
4899
  if (written.cardinality < before * guard.minRetained) return refuse("min-retained");
3807
4900
  }
3808
- const published = await publishGeneration(deps.registry, key, {
3809
- wrappedDeks: written.wrappedDeks,
3810
- ...fromToken === void 0 ? {} : { expectToken: fromToken },
3811
- ...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
3812
- // The third case, and the one the two fences above structurally cannot cover: the guard judged a segment
3813
- // that had NO ROW. Both `expectFrom` and `expectToken` compare against a value read from a row, so with no
3814
- // row both are omitted and the publish becomes a bare forward-only advance — which lands over anything
3815
- // that appeared in between. `before` was `null`, so the empty and `minRetained` bounds had nothing to
3816
- // judge and passed vacuously. Verified: an empty generation published over a thousand ids that a
3817
- // concurrent writer had created meanwhile, reporting success. A guarded write therefore has to fence on
3818
- // the ABSENCE it relied on, exactly as it fences on the pointer it relied on.
3819
- ...needsBefore && row === null ? { expectAbsent: true } : {}
3820
- });
4901
+ let published;
4902
+ try {
4903
+ published = await publishGeneration(deps.registry, key, {
4904
+ row,
4905
+ wrappedDeks: written.wrappedDeks,
4906
+ summary: written.summary,
4907
+ cleartext: !written.encrypted,
4908
+ ...fromToken === void 0 ? {} : { expectToken: fromToken },
4909
+ ...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
4910
+ // The third case, and the one the two fences above structurally cannot cover: the guard judged a segment
4911
+ // that had NO ROW. Both `expectFrom` and `expectToken` compare against a value read from a row, so with no
4912
+ // row both are omitted and the publish becomes a bare forward-only advance — which lands over anything
4913
+ // that appeared in between. `before` was `null`, so the empty and `minRetained` bounds had nothing to
4914
+ // judge and passed vacuously. Verified: an empty generation published over a thousand ids that a
4915
+ // concurrent writer had created meanwhile, reporting success. A guarded write therefore has to fence on
4916
+ // the ABSENCE it relied on, exactly as it fences on the pointer it relied on.
4917
+ ...needsBefore && row === null ? { expectAbsent: true } : {},
4918
+ // A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
4919
+ // is this load's publish only over the object this load wrote, which one footer read proves.
4920
+ holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
4921
+ // And the wait before a fresh write, when the first left the row as it was.
4922
+ clock: deps.clock,
4923
+ rng: deps.rng ?? deps.readRetry?.rng,
4924
+ onUnanswered: () => {
4925
+ unanswered = true;
4926
+ }
4927
+ });
4928
+ } catch (err) {
4929
+ if (isValidationError(err) || err instanceof KeyUnavailableError) await reclaim();
4930
+ throw err;
4931
+ }
3821
4932
  if (!published) return refuse("superseded");
3822
4933
  audit.onEvent({
3823
4934
  kind: "segment.publish",
@@ -3825,7 +4936,13 @@ async function loadSegment(ref, ids, deps, options = {}) {
3825
4936
  namespace: ref.namespace,
3826
4937
  generation
3827
4938
  });
3828
- const collected = await gcOrphanGenerations(ref, deps, { keep });
4939
+ const currentObjectGone = needsBefore && before === null;
4940
+ const collected = await collectAfterLoad(ref, deps, {
4941
+ generation,
4942
+ keep,
4943
+ byName: checked && !currentObjectGone && deps.collectByListing !== true,
4944
+ ...current.fromSummary && fromGeneration !== void 0 ? { proveCurrent: { ...ref, generation: fromGeneration } } : {}
4945
+ });
3829
4946
  return {
3830
4947
  generation,
3831
4948
  published: true,
@@ -3845,7 +4962,25 @@ async function listGenerations(ref, deps) {
3845
4962
  const current = record?.currentGen ?? null;
3846
4963
  const seen = /* @__PURE__ */ new Set();
3847
4964
  for await (const key of deps.storage.list(ref)) seen.add(key.generation);
3848
- return [...seen].sort((a, b) => a - b).map((generation) => ({ generation, current: generation === current }));
4965
+ const described = record === null ? void 0 : await currentDescription(ref, record, deps);
4966
+ return [...seen].sort((a, b) => a - b).map(
4967
+ (generation) => generation === current ? { generation, current: true, ...described } : { generation, current: false }
4968
+ );
4969
+ }
4970
+ async function currentDescription(ref, record, deps) {
4971
+ if (record.summary === void 0) return void 0;
4972
+ let aead;
4973
+ if ("sealed" in record.summary && record.wrappedDeks !== void 0 && deps.keystore !== void 0) {
4974
+ try {
4975
+ aead = await deps.keystore.openDek(record.wrappedDeks);
4976
+ } catch {
4977
+ return void 0;
4978
+ }
4979
+ }
4980
+ const usable = usableSummary(ref, record, aead);
4981
+ if (usable === void 0) return void 0;
4982
+ const metadata = usable.metadata;
4983
+ return metadata === void 0 || Object.keys(metadata).length === 0 ? { cardinality: usable.cardinality } : { cardinality: usable.cardinality, metadata: Object.freeze({ ...metadata }) };
3849
4984
  }
3850
4985
  async function rollbackSegment(ref, toGeneration, deps, options = {}) {
3851
4986
  validateUserRef(ref);
@@ -3879,8 +5014,34 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
3879
5014
  `rollback: generation ${toGeneration} of "${ref.segment}" is above the current pointer (${record.currentGen}) \u2014 it may never have been published. Pass { allowForward: true } if you are undoing an earlier rollback.`
3880
5015
  );
3881
5016
  }
5017
+ const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
5018
+ let aead;
5019
+ if (keyed && deps.keystore !== void 0) {
5020
+ try {
5021
+ aead = await deps.keystore.openDek(record.wrappedDeks);
5022
+ } catch {
5023
+ }
5024
+ }
5025
+ const crypto = aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, toGeneration, scope) };
5026
+ const { encrypted, reader } = await openRollbackTarget(
5027
+ deps.storage,
5028
+ { ...ref, generation: toGeneration },
5029
+ crypto
5030
+ );
5031
+ if (encrypted !== keyed) {
5032
+ throw new IntegrityError(
5033
+ `rollback: generation ${toGeneration} of "${ref.segment}" is ${encrypted ? "encrypted" : "cleartext"}, but the segment is ${keyed ? "encrypted" : "cleartext"}, so no read of the segment could use it; it was never one of its generations. Roll back to another generation.`
5034
+ );
5035
+ }
5036
+ const summary = reader === void 0 ? void 0 : summaryOf(
5037
+ ref,
5038
+ toGeneration,
5039
+ { cardinality: reader.count(), metadata: reader.metadata },
5040
+ aead
5041
+ );
3882
5042
  const { token } = await deps.registry.compareAndSwap(ref, record.token, {
3883
- currentGen: toGeneration
5043
+ currentGen: toGeneration,
5044
+ summary
3884
5045
  });
3885
5046
  let stillThere = false;
3886
5047
  for await (const key of deps.storage.list(ref)) {
@@ -3889,15 +5050,31 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
3889
5050
  break;
3890
5051
  }
3891
5052
  }
3892
- if (!stillThere) {
5053
+ let replaced = false;
5054
+ if (stillThere && reader !== void 0 && toGeneration > (record.currentGen ?? -1)) {
5055
+ try {
5056
+ replaced = !await provesOwnObject(
5057
+ deps.storage,
5058
+ { ...ref, generation: toGeneration },
5059
+ reader.fingerprint
5060
+ );
5061
+ } catch (err) {
5062
+ if (!isIntegrityError(err)) throw err;
5063
+ replaced = true;
5064
+ }
5065
+ }
5066
+ if (!stillThere || replaced) {
3893
5067
  let undone = false;
3894
5068
  try {
3895
- await deps.registry.compareAndSwap(ref, token, { currentGen: record.currentGen });
5069
+ await deps.registry.compareAndSwap(ref, token, {
5070
+ currentGen: record.currentGen,
5071
+ summary: describingSummary(record)
5072
+ });
3896
5073
  undone = true;
3897
5074
  } catch {
3898
5075
  }
3899
5076
  throw new NotFoundError(
3900
- `rollback: generation ${toGeneration} of "${ref.segment}" was collected while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
5077
+ `rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
3901
5078
  );
3902
5079
  }
3903
5080
  safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
@@ -3909,6 +5086,12 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
3909
5086
  });
3910
5087
  return { fromGeneration: record.currentGen, generation: toGeneration };
3911
5088
  }
5089
+ function describingSummary(record) {
5090
+ const { summary } = record;
5091
+ if (summary === void 0 || summary.generation !== record.currentGen) return void 0;
5092
+ const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
5093
+ return "sealed" in summary === keyed ? summary : void 0;
5094
+ }
3912
5095
 
3913
5096
  // src/core/erase-id.ts
3914
5097
  var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
@@ -3918,6 +5101,7 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
3918
5101
  const codec = requireCodec(deps.codec, "eraseIdFromSegment");
3919
5102
  const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
3920
5103
  const base = { segment: ref.segment, namespace: ref.namespace };
5104
+ const read = (op) => retryRead(op, deps.readRetry);
3921
5105
  const record = await deps.registry.get(ref);
3922
5106
  if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
3923
5107
  if (record.status === "destroyed")
@@ -3954,13 +5138,18 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
3954
5138
  return row.currentGen === from && row.token === fromToken ? null : "superseded";
3955
5139
  };
3956
5140
  const holds = async (generation2) => {
5141
+ const key2 = { ...base, generation: generation2 };
5142
+ const chunkIn = (crypto) => read(async () => (await openGenerationReader(deps.storage, key2, crypto)).getChunk(chunkKey));
3957
5143
  try {
3958
- const reader = await openGenerationReader(
3959
- deps.storage,
3960
- { ...base, generation: generation2 },
3961
- cryptoAt(generation2)
3962
- );
3963
- const bytes = await reader.getChunk(chunkKey);
5144
+ let bytes;
5145
+ try {
5146
+ bytes = await chunkIn(cryptoAt(generation2));
5147
+ } catch (err) {
5148
+ if (!isIntegrityError(err) || cryptoAt(generation2) === void 0 || await read(() => objectIsEncrypted(deps.storage, key2))) {
5149
+ throw err;
5150
+ }
5151
+ bytes = await chunkIn(void 0);
5152
+ }
3964
5153
  return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
3965
5154
  } catch (err) {
3966
5155
  if (isNotFoundError(err)) return null;
@@ -4030,27 +5219,40 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
4030
5219
  const stage = async () => {
4031
5220
  let written;
4032
5221
  try {
4033
- const reader = await openGenerationReader(deps.storage, fromKey, cryptoAt(from));
4034
- const bytes = await reader.getChunk(chunkKey);
5222
+ const reader = await read(() => openGenerationReader(deps.storage, fromKey, cryptoAt(from)));
5223
+ const bytes = await read(() => reader.getChunk(chunkKey));
4035
5224
  if (bytes === null) return await notInCurrent();
4036
5225
  const target = codec.safeDeserialize(bytes, maxBytes);
4037
5226
  assertRemaindersInRange(target, chunkKey);
4038
5227
  if (!target.has(remainder)) return await notInCurrent();
4039
5228
  target.remove(remainder);
5229
+ assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
4040
5230
  const generation2 = await nextGeneration(ref, deps);
4041
5231
  const key2 = { ...base, generation: generation2 };
5232
+ const metadata = metadataToCarry(
5233
+ reader.metadata,
5234
+ aead === void 0 ? void 0 : usableSummary(ref, record, aead)
5235
+ );
4042
5236
  const tally = await writeCrbmGenerationStream(
4043
5237
  deps.storage,
4044
5238
  key2,
4045
- rewrite(reader, chunkKey, target, codec, maxBytes),
4046
- { crypto: cryptoAt(generation2), clock: deps.clock }
5239
+ rewrite(reader, chunkKey, target, codec, maxBytes, read),
5240
+ { crypto: cryptoAt(generation2), clock: deps.clock, metadata }
4047
5241
  );
4048
5242
  written = generation2;
4049
5243
  const beforeVerify = await deps.registry.get(ref);
4050
5244
  const early = rowVerdict(beforeVerify);
4051
5245
  if (early !== null) return refused(early, written);
4052
- await verifyGeneration(deps.storage, key2, tally, cryptoAt(generation2));
4053
- return { generation: generation2, key: key2 };
5246
+ await read(
5247
+ () => verifyGeneration(deps.storage, key2, { ...tally, metadata }, cryptoAt(generation2))
5248
+ );
5249
+ const summary2 = summaryOf(
5250
+ ref,
5251
+ generation2,
5252
+ { cardinality: tally.cardinality, metadata },
5253
+ aead
5254
+ );
5255
+ return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
4054
5256
  } catch (err) {
4055
5257
  if (!isNotFoundError(err)) throw err;
4056
5258
  let now;
@@ -4075,10 +5277,16 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
4075
5277
  if (!staged.erased && staged.generation !== void 0) await discardRefused(staged.generation);
4076
5278
  return staged;
4077
5279
  }
4078
- const { generation, key } = staged;
5280
+ const { generation, key, fingerprint, summary } = staged;
4079
5281
  const published = await publishGeneration(deps.registry, key, {
4080
5282
  expectFrom: from,
4081
- expectToken: fromToken
5283
+ expectToken: fromToken,
5284
+ summary,
5285
+ // A write that ends without an answer is settled by reading the row, and a pointer at this number is this
5286
+ // rewrite's only over the object it wrote: the footer proves it, so another incarnation's cannot pass for it.
5287
+ holdsOwnObject: () => provesOwnObject(deps.storage, key, fingerprint),
5288
+ clock: deps.clock,
5289
+ rng: deps.rng ?? deps.readRetry?.rng
4082
5290
  });
4083
5291
  if (!published) {
4084
5292
  await discardRefused(generation);
@@ -4103,14 +5311,14 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
4103
5311
  if (left !== void 0) throw cannotRemove(left);
4104
5312
  return { ...base, erased: true, fromGeneration: from, generation, collected };
4105
5313
  }
4106
- async function* rewrite(reader, chunkKey, replacement, codec, maxBytes) {
5314
+ async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
4107
5315
  const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
4108
5316
  for (const k of keys) {
4109
5317
  if (k === chunkKey) {
4110
5318
  yield { chunkKey: k, bitmap: replacement };
4111
5319
  continue;
4112
5320
  }
4113
- const bytes = await reader.getChunk(k);
5321
+ const bytes = await read(() => reader.getChunk(k));
4114
5322
  if (bytes === null) continue;
4115
5323
  const bitmap = codec.safeDeserialize(bytes, maxBytes);
4116
5324
  assertRemaindersInRange(bitmap, k);
@@ -4129,8 +5337,8 @@ function assertRemaindersInRange(bitmap, chunkKey) {
4129
5337
  // src/drivers/crypto.ts
4130
5338
  import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
4131
5339
  var KEY_BYTES = 32;
4132
- var NONCE_BYTES = 12;
4133
- var TAG_BYTES = 16;
5340
+ var NONCE_BYTES2 = 12;
5341
+ var TAG_BYTES2 = 16;
4134
5342
  var NodeAead = class {
4135
5343
  key;
4136
5344
  constructor(key) {
@@ -4139,14 +5347,14 @@ var NodeAead = class {
4139
5347
  this.key = Buffer.from(key);
4140
5348
  }
4141
5349
  seal(plaintext, aad) {
4142
- const nonce = randomBytes(NONCE_BYTES);
5350
+ const nonce = randomBytes(NONCE_BYTES2);
4143
5351
  const cipher = createCipheriv("aes-256-gcm", this.key, nonce);
4144
5352
  cipher.setAAD(aad);
4145
5353
  const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
4146
5354
  return { nonce, ciphertext, tag: cipher.getAuthTag() };
4147
5355
  }
4148
5356
  open(sealed, aad) {
4149
- if (sealed.nonce.length !== NONCE_BYTES || sealed.tag.length !== TAG_BYTES)
5357
+ if (sealed.nonce.length !== NONCE_BYTES2 || sealed.tag.length !== TAG_BYTES2)
4150
5358
  throw new IntegrityError("AEAD nonce/tag have the wrong length");
4151
5359
  const decipher = createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
4152
5360
  decipher.setAAD(aad);
@@ -4166,11 +5374,11 @@ function wrapKey(dek, kek, keyId) {
4166
5374
  }
4167
5375
  function unwrapKey(wrapped, kek, keyId) {
4168
5376
  const blob = Buffer.from(wrapped, "base64");
4169
- if (blob.length < NONCE_BYTES + TAG_BYTES)
5377
+ if (blob.length < NONCE_BYTES2 + TAG_BYTES2)
4170
5378
  throw new IntegrityError("wrapped DEK is too short to be valid");
4171
- const nonce = blob.subarray(0, NONCE_BYTES);
4172
- const tag = blob.subarray(blob.length - TAG_BYTES);
4173
- const ciphertext = blob.subarray(NONCE_BYTES, blob.length - TAG_BYTES);
5379
+ const nonce = blob.subarray(0, NONCE_BYTES2);
5380
+ const tag = blob.subarray(blob.length - TAG_BYTES2);
5381
+ const ciphertext = blob.subarray(NONCE_BYTES2, blob.length - TAG_BYTES2);
4174
5382
  return new NodeAead(kek).open({ nonce, ciphertext, tag }, new TextEncoder().encode(keyId));
4175
5383
  }
4176
5384
  var InProcessKeystore = class {
@@ -4411,7 +5619,10 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
4411
5619
  status: "destroyed",
4412
5620
  wrappedDeks: void 0,
4413
5621
  // ← the crypto-shred: the only copy of the DEK wrappings is gone
4414
- keyId: void 0
5622
+ keyId: void 0,
5623
+ // The current generation's cached count and metadata go with it: sealed, they cannot be opened without
5624
+ // the wrappings; clear, they would outlive the segment on its tombstone.
5625
+ summary: void 0
4415
5626
  });
4416
5627
  return { ...base, destroyed: true, cryptoShredded: encrypted };
4417
5628
  } catch (err) {
@@ -4550,39 +5761,108 @@ function assertNotDestroyed(record, ref) {
4550
5761
 
4551
5762
  // src/core/retention-sweep.ts
4552
5763
  var DEFAULT_RETIRE_LIMIT = 100;
5764
+ var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
4553
5765
  var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
4554
5766
  var DEFAULT_LOOKBACK_BUCKETS = 7;
4555
5767
  async function rowsFromDueIndex(registry, options) {
4556
5768
  const rows = [];
4557
- const seen = /* @__PURE__ */ new Set();
5769
+ const pointers = /* @__PURE__ */ new Map();
5770
+ const litter = [];
5771
+ const gone = /* @__PURE__ */ new Set();
4558
5772
  for (const bucket of dueBucketsAt(options.now, options.lookbackBuckets)) {
4559
5773
  for await (const pointer of registry.list(dueNamespace(bucket))) {
4560
5774
  const ref = decodeDueName(pointer.segment);
4561
5775
  if (ref === null) continue;
4562
5776
  if (isReservedNamespace(ref.namespace)) continue;
4563
5777
  if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
4564
- const key = `${ref.namespace ?? ""}\0${ref.segment}`;
4565
- if (seen.has(key)) continue;
4566
- seen.add(key);
5778
+ const found = {
5779
+ ref: { namespace: pointer.namespace, segment: pointer.segment },
5780
+ target: ref,
5781
+ token: pointer.token
5782
+ };
5783
+ const key = segmentKey(ref);
5784
+ const known = pointers.get(key);
5785
+ if (known !== void 0) {
5786
+ known.push(found);
5787
+ continue;
5788
+ }
5789
+ if (gone.has(key)) {
5790
+ litter.push(found);
5791
+ continue;
5792
+ }
4567
5793
  if (rows.length >= options.maxScanSegments) {
4568
5794
  throw new BudgetExceededError(
4569
5795
  `retireExpired: the due index yielded more than ${options.maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Raise \`maxScanSegments\`, narrow with \`namespace\`, or reduce \`lookbackBuckets\`.`
4570
5796
  );
4571
5797
  }
4572
5798
  const live = await registry.get(ref);
4573
- if (live === null) continue;
5799
+ if (live === null) {
5800
+ gone.add(key);
5801
+ litter.push(found);
5802
+ continue;
5803
+ }
4574
5804
  rows.push(live);
5805
+ pointers.set(key, [found]);
4575
5806
  }
4576
5807
  }
4577
- return rows;
5808
+ return { rows, pointers, litter };
5809
+ }
5810
+ async function rowsFromFleet(registry, options) {
5811
+ const held = [];
5812
+ const rows = await drainRegistry(registry, {
5813
+ namespace: options.namespace,
5814
+ maxScanSegments: options.maxScanSegments,
5815
+ op: "retireExpired",
5816
+ onReserved: (pointer) => {
5817
+ if (held.length >= options.maxScanSegments) return;
5818
+ const target = decodeDueName(pointer.segment);
5819
+ if (target === null || isReservedNamespace(target.namespace)) return;
5820
+ held.push({
5821
+ ref: { namespace: pointer.namespace, segment: pointer.segment },
5822
+ target,
5823
+ token: pointer.token
5824
+ });
5825
+ }
5826
+ });
5827
+ const pointers = /* @__PURE__ */ new Map();
5828
+ const litter = [];
5829
+ if (held.length === 0) return { rows, pointers, litter };
5830
+ const present = new Set(rows.map((rec) => segmentKey(rec)));
5831
+ for (const found of held) {
5832
+ const key = segmentKey(found.target);
5833
+ if (!present.has(key)) litter.push(found);
5834
+ else if (pointers.has(key)) pointers.get(key).push(found);
5835
+ else pointers.set(key, [found]);
5836
+ }
5837
+ return { rows, pointers, litter };
4578
5838
  }
4579
- async function forgetDuePointer(registry, ref, expiresAt) {
5839
+ async function forgetDuePointer(registry, ref, expiresAt, onFault) {
4580
5840
  if (!canIndex(ref)) return;
4581
5841
  try {
4582
5842
  await registry.delete(dueIndexRef(dueBucket(expiresAt), ref));
5843
+ } catch (err) {
5844
+ onFault(err);
5845
+ }
5846
+ }
5847
+ async function filePurgePointer(registry, ref, purgeAt) {
5848
+ if (!canIndex(ref)) return;
5849
+ try {
5850
+ await registry.create(dueIndexRef(dueBucket(purgeAt), ref), { currentGen: null });
4583
5851
  } catch {
4584
5852
  }
4585
5853
  }
5854
+ async function forgetPurgePointers(registry, ref, purgeAt, found, onFault) {
5855
+ if (!canIndex(ref)) return;
5856
+ const filed = dueIndexRef(dueBucket(purgeAt), ref);
5857
+ const all = found.some((p) => p.ref.namespace === filed.namespace) ? found : [...found, { ref: filed, target: ref }];
5858
+ for (const pointer of all) {
5859
+ try {
5860
+ await registry.delete(pointer.ref, pointer.token);
5861
+ } catch (err) {
5862
+ onFault(err);
5863
+ }
5864
+ }
5865
+ }
4586
5866
  async function retireExpired(deps, options) {
4587
5867
  if (options.namespace !== void 0) validateUserNamespace(options.namespace);
4588
5868
  const now = options.now;
@@ -4615,32 +5895,40 @@ async function retireExpired(deps, options) {
4615
5895
  }
4616
5896
  const dryRun = options.dryRun === true;
4617
5897
  const scan = options.scan ?? "fleet";
4618
- const rows = scan === "index" ? await rowsFromDueIndex(deps.registry, {
5898
+ const scanned = scan === "index" ? await rowsFromDueIndex(deps.registry, {
4619
5899
  now,
4620
5900
  lookbackBuckets: options.lookbackBuckets ?? DEFAULT_LOOKBACK_BUCKETS,
4621
5901
  namespace: options.namespace,
4622
5902
  maxScanSegments
4623
- }) : await drainRegistry(deps.registry, {
4624
- namespace: options.namespace,
4625
- maxScanSegments,
4626
- op: "retireExpired"
4627
- });
5903
+ }) : await rowsFromFleet(deps.registry, { namespace: options.namespace, maxScanSegments });
5904
+ const rows = scanned.rows;
5905
+ const removesRows = deps.registry.capabilities().conditionalDelete === true;
4628
5906
  const shards = options.shards;
4629
5907
  const totalShards = options.totalShards ?? 0;
4630
- const mine = shards === void 0 || totalShards <= 1 ? rows : rows.filter((r) => shards.includes(shardOf(segmentKey(r), totalShards)));
5908
+ const owned = (ref) => shards === void 0 || totalShards <= 1 || shards.includes(shardOf(segmentKey(ref), totalShards));
5909
+ const mine = rows.filter(owned);
4631
5910
  const entries = [];
4632
5911
  let eligible = 0;
4633
5912
  let retired = 0;
4634
5913
  let wouldRetire = 0;
4635
5914
  let tombstonesPurged = 0;
4636
5915
  let limited = false;
5916
+ let purgeFaults = 0;
5917
+ let firstPurgeFault;
5918
+ const noteFault = (err) => {
5919
+ if (isWriteConflictError(err)) return;
5920
+ purgeFaults += 1;
5921
+ firstPurgeFault ??= failureReason(err);
5922
+ };
5923
+ let purgeRun = 0;
5924
+ let purging = true;
4637
5925
  let attempted = 0;
4638
5926
  for (const rec of mine) {
4639
5927
  const ref = { namespace: rec.namespace, segment: rec.segment };
4640
5928
  const base = { segment: rec.segment, namespace: rec.namespace };
4641
5929
  const policy = readRetentionPolicy(rec.retention);
4642
5930
  if (rec.status === "destroyed") {
4643
- if (!purgeTombstones) continue;
5931
+ if (!purgeTombstones || !purging) continue;
4644
5932
  const retiredAt = retirementStamp(rec.retention);
4645
5933
  if (retiredAt === null) continue;
4646
5934
  if (now - retiredAt < grace) continue;
@@ -4657,13 +5945,31 @@ async function retireExpired(deps, options) {
4657
5945
  continue;
4658
5946
  }
4659
5947
  }
4660
- if (!dryRun) await deps.registry.delete(ref, rec.token);
4661
- if (!dryRun) tombstonesPurged += 1;
5948
+ if (!dryRun) {
5949
+ await deps.registry.delete(ref, rec.token);
5950
+ tombstonesPurged += 1;
5951
+ purgeRun = 0;
5952
+ if (removesRows) {
5953
+ await forgetPurgePointers(
5954
+ deps.registry,
5955
+ ref,
5956
+ retiredAt + grace,
5957
+ scanned.pointers.get(segmentKey(ref)) ?? [],
5958
+ noteFault
5959
+ );
5960
+ }
5961
+ }
4662
5962
  entries.push({
4663
5963
  ...base,
4664
5964
  action: dryRun ? "would-purge-tombstone" : "purged-tombstone"
4665
5965
  });
4666
5966
  } catch (err) {
5967
+ if (!isWriteConflictError(err)) {
5968
+ attempted -= 1;
5969
+ purgeRun += 1;
5970
+ if (purgeRun >= MAX_CONSECUTIVE_PURGE_FAULTS) purging = false;
5971
+ noteFault(err);
5972
+ }
4667
5973
  entries.push({ ...base, action: "skipped", reason: failureReason(err) });
4668
5974
  }
4669
5975
  continue;
@@ -4707,22 +6013,27 @@ async function retireExpired(deps, options) {
4707
6013
  }
4708
6014
  retired += 1;
4709
6015
  entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
4710
- await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt);
6016
+ await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt, noteFault);
4711
6017
  if (purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
4712
6018
  const tombstone = await deps.registry.get(ref).catch(() => null);
4713
6019
  if (tombstone?.status === "destroyed") {
4714
6020
  const deleted = await deps.registry.delete(ref, tombstone.token).then(
4715
6021
  () => true,
4716
- () => false
6022
+ (err) => {
6023
+ noteFault(err);
6024
+ return false;
6025
+ }
4717
6026
  );
4718
6027
  if (deleted) continue;
4719
6028
  }
4720
6029
  }
4721
- await stampRetirement(deps.registry, ref, now).catch(() => void 0);
6030
+ const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
6031
+ if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
4722
6032
  } catch (err) {
4723
6033
  const after = await deps.registry.get(ref).catch(() => null);
4724
6034
  if (after?.status === "destroyed") {
4725
- await stampRetirement(deps.registry, ref, now).catch(() => void 0);
6035
+ const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
6036
+ if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
4726
6037
  retired += 1;
4727
6038
  entries.push({
4728
6039
  ...base,
@@ -4743,6 +6054,20 @@ async function retireExpired(deps, options) {
4743
6054
  entries.push({ ...base, action: "skipped", reason: failureReason(err) });
4744
6055
  }
4745
6056
  }
6057
+ if (!dryRun && removesRows) {
6058
+ let attemptedLitter = 0;
6059
+ for (const pointer of scanned.litter) {
6060
+ if (!owned(pointer.target)) continue;
6061
+ if (attemptedLitter >= limit) break;
6062
+ attemptedLitter += 1;
6063
+ try {
6064
+ if (await deps.registry.get(pointer.target) !== null) continue;
6065
+ await deps.registry.delete(pointer.ref, pointer.token);
6066
+ } catch (err) {
6067
+ noteFault(err);
6068
+ }
6069
+ }
6070
+ }
4746
6071
  return {
4747
6072
  scanned: mine.length,
4748
6073
  eligible,
@@ -4750,6 +6075,8 @@ async function retireExpired(deps, options) {
4750
6075
  wouldRetire,
4751
6076
  tombstonesPurged,
4752
6077
  limited,
6078
+ purgeFaults,
6079
+ ...firstPurgeFault === void 0 ? {} : { firstPurgeFault },
4753
6080
  dryRun,
4754
6081
  entries
4755
6082
  };
@@ -4765,16 +6092,17 @@ function retirementStamp(meta) {
4765
6092
  async function stampRetirement(registry, ref, now) {
4766
6093
  for (let attempt = 0; attempt < 3; attempt += 1) {
4767
6094
  const rec = await registry.get(ref);
4768
- if (rec === null || rec.status !== "destroyed") return;
6095
+ if (rec === null || rec.status !== "destroyed") return false;
4769
6096
  try {
4770
6097
  await registry.compareAndSwap(ref, rec.token, {
4771
6098
  retention: { ...rec.retention, [RETIRED_AT]: now }
4772
6099
  });
4773
- return;
6100
+ return true;
4774
6101
  } catch (err) {
4775
6102
  if (!isWriteConflictError(err)) throw err;
4776
6103
  }
4777
6104
  }
6105
+ return false;
4778
6106
  }
4779
6107
  async function isFullyReclaimed(deps, ref) {
4780
6108
  for await (const key of deps.storage.list(ref)) {
@@ -4795,8 +6123,45 @@ async function generationsPresent(storage, ref) {
4795
6123
  for await (const key of storage.list(ref)) present.add(key.generation);
4796
6124
  return present;
4797
6125
  }
6126
+ async function checkSummary(ref, live, deps) {
6127
+ const generation = live.currentGen;
6128
+ const keyed = live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0;
6129
+ let aead;
6130
+ if (keyed) {
6131
+ if (deps.keystore === void 0) return { kind: "ok", unchecked: true };
6132
+ try {
6133
+ aead = await deps.keystore.openDek(live.wrappedDeks);
6134
+ } catch (err) {
6135
+ if (isIntegrityError(err) || isCloudRoaringError(err) && err.name === "KeyUnavailableError")
6136
+ return { kind: "ok", unchecked: true };
6137
+ throw err;
6138
+ }
6139
+ }
6140
+ const described = usableSummary(ref, live, aead);
6141
+ if (described === void 0) return { kind: "ok", unchecked: keyed };
6142
+ const reader = await openGenerationReader(
6143
+ deps.storage,
6144
+ { namespace: ref.namespace, segment: ref.segment, generation },
6145
+ aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) }
6146
+ );
6147
+ let cardinality = 0;
6148
+ for (const n of reader.cardinalities().values()) cardinality += n;
6149
+ if (summaryAgrees(described, { cardinality, metadata: reader.metadata })) return { kind: "ok" };
6150
+ return {
6151
+ kind: "issue",
6152
+ issue: {
6153
+ segment: ref.segment,
6154
+ namespace: ref.namespace,
6155
+ currentGen: generation,
6156
+ issue: "summary-mismatch"
6157
+ }
6158
+ };
6159
+ }
4798
6160
  async function runConsistencyCheck(deps, options = {}) {
4799
6161
  if (options.namespace !== void 0) validateUserNamespace(options.namespace);
6162
+ if (options.summaries !== void 0 && typeof options.summaries !== "boolean") {
6163
+ throw new ValidationError(`summaries must be a boolean; got ${String(options.summaries)}`);
6164
+ }
4800
6165
  const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
4801
6166
  if (!Number.isInteger(concurrency) || concurrency < 1) {
4802
6167
  throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
@@ -4818,7 +6183,10 @@ async function runConsistencyCheck(deps, options = {}) {
4818
6183
  if (!live || live.status === "destroyed") return { kind: "ok" };
4819
6184
  if (live.currentGen === null) return { kind: "ok" };
4820
6185
  const present = await generationsPresent(deps.storage, ref);
4821
- if (present.has(live.currentGen)) return { kind: "ok" };
6186
+ if (present.has(live.currentGen)) {
6187
+ if (options.summaries !== true || live.summary === void 0) return { kind: "ok" };
6188
+ return await checkSummary(ref, live, deps);
6189
+ }
4822
6190
  return {
4823
6191
  kind: "issue",
4824
6192
  issue: {
@@ -4841,53 +6209,13 @@ async function runConsistencyCheck(deps, options = {}) {
4841
6209
  });
4842
6210
  const inconsistent = [];
4843
6211
  const errored = [];
6212
+ let summariesUnchecked = 0;
4844
6213
  for (const r of results) {
4845
6214
  if (r.kind === "issue") inconsistent.push(r.issue);
4846
6215
  else if (r.kind === "error") errored.push(r.error);
6216
+ else if (r.unchecked === true) summariesUnchecked += 1;
4847
6217
  }
4848
- return { checked: recs.length, inconsistent, errored };
4849
- }
4850
-
4851
- // src/core/retry.ts
4852
- var DEFAULT_RETRY_POLICY = {
4853
- maxAttempts: 4,
4854
- baseDelayMs: 50,
4855
- maxDelayMs: 2e3,
4856
- backoffFactor: 2,
4857
- jitter: "full"
4858
- };
4859
- function backoffDelayMs(policy, attempt) {
4860
- const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
4861
- return Math.min(policy.maxDelayMs, raw);
4862
- }
4863
- function applyJitter(policy, delayMs, rng) {
4864
- if (policy.jitter === "none") return delayMs;
4865
- return rng.next() * delayMs;
4866
- }
4867
- async function withRetry(op, policy, deps) {
4868
- const retryable = deps.isRetryable ?? isTransientError;
4869
- if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
4870
- throw new ValidationError(
4871
- `retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
4872
- );
4873
- }
4874
- const attempts = Math.floor(policy.maxAttempts);
4875
- let lastErr;
4876
- for (let attempt = 1; attempt <= attempts; attempt++) {
4877
- try {
4878
- return await op();
4879
- } catch (err) {
4880
- lastErr = err;
4881
- if (attempt >= attempts || !retryable(err)) throw err;
4882
- const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
4883
- try {
4884
- deps.onRetry?.({ attempt, delayMs, err });
4885
- } catch {
4886
- }
4887
- await deps.clock.sleep(delayMs);
4888
- }
4889
- }
4890
- throw lastErr;
6218
+ return options.summaries === true ? { checked: recs.length, inconsistent, errored, summariesUnchecked } : { checked: recs.length, inconsistent, errored };
4891
6219
  }
4892
6220
 
4893
6221
  // src/drivers/retry/retrying-chunk-source.ts
@@ -4909,6 +6237,7 @@ var RetryingStorageChunkSource = class {
4909
6237
  /** Present only when the inner source supports it — so capability detection stays honest. */
4910
6238
  sizeOf;
4911
6239
  cardinalities;
6240
+ summary;
4912
6241
  currentGeneration;
4913
6242
  invalidate;
4914
6243
  exists;
@@ -4933,6 +6262,10 @@ var RetryingStorageChunkSource = class {
4933
6262
  if (innerCardinalities) {
4934
6263
  this.cardinalities = (ref) => withRetry(() => innerCardinalities.call(inner, ref), this.policy, this.deps);
4935
6264
  }
6265
+ const innerSummary = inner.summary;
6266
+ if (innerSummary) {
6267
+ this.summary = (ref) => withRetry(() => innerSummary.call(inner, ref), this.policy, this.deps);
6268
+ }
4936
6269
  const innerCurrentGeneration = inner.currentGeneration;
4937
6270
  if (innerCurrentGeneration) {
4938
6271
  this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);