@cloudbitmaps/core 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-5OMC5U7U.js → chunk-5W3XS2IU.js} +292 -22
- package/dist/chunk-5W3XS2IU.js.map +7 -0
- package/dist/core/audit.d.ts +6 -0
- package/dist/core/codec.d.ts +33 -1
- package/dist/core/consistency.d.ts +21 -2
- package/dist/core/cost.d.ts +91 -10
- package/dist/core/crbm/format.d.ts +31 -1
- package/dist/core/crbm/reader.d.ts +77 -18
- package/dist/core/crbm/writer.d.ts +32 -3
- package/dist/core/crbm-storage-source.d.ts +204 -23
- package/dist/core/crypto.d.ts +11 -7
- package/dist/core/determinism.d.ts +7 -0
- package/dist/core/engine.d.ts +22 -7
- package/dist/core/erase-id.d.ts +20 -0
- package/dist/core/erasure.d.ts +8 -4
- package/dist/core/generation-gc.d.ts +102 -4
- package/dist/core/load-input.d.ts +55 -0
- package/dist/core/load.d.ts +65 -10
- package/dist/core/metadata.d.ts +42 -0
- package/dist/core/pinned-storage-source.d.ts +2 -1
- package/dist/core/ports.d.ts +119 -13
- package/dist/core/registry-scan.d.ts +6 -0
- package/dist/core/retention-sweep.d.ts +47 -2
- package/dist/core/retry.d.ts +14 -0
- package/dist/core/rollback.d.ts +34 -4
- package/dist/core/summary.d.ts +61 -0
- package/dist/core/token.d.ts +36 -0
- package/dist/core/validate.d.ts +2 -0
- package/dist/driver-kit.js +39 -20
- package/dist/driver-kit.js.map +2 -2
- package/dist/drivers/_shared/entropy.d.ts +5 -0
- package/dist/drivers/_shared/object-registry.d.ts +36 -6
- package/dist/drivers/_shared/registry.d.ts +47 -19
- package/dist/drivers/localfs/registry.d.ts +10 -0
- package/dist/drivers/memory.d.ts +9 -3
- package/dist/drivers/retry/retrying-chunk-source.d.ts +2 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1922 -512
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +6 -5
- package/package.json +1 -1
- package/dist/chunk-5OMC5U7U.js.map +0 -7
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
DEFAULT_NAMESPACE,
|
|
6
6
|
IntegrityError,
|
|
7
7
|
KeyUnavailableError,
|
|
8
|
+
MAX_METADATA_BYTES,
|
|
8
9
|
MAX_NAME_LENGTH,
|
|
9
10
|
NotFoundError,
|
|
10
11
|
RESERVED_NAMESPACE_PREFIX,
|
|
@@ -13,11 +14,18 @@ import {
|
|
|
13
14
|
ValidationError,
|
|
14
15
|
WriteConflictError,
|
|
15
16
|
applyRegistryPatch,
|
|
17
|
+
assertRegistryCanWrite,
|
|
16
18
|
brandAsBackend,
|
|
19
|
+
canonicalMetadataJson,
|
|
20
|
+
copiedMetadata,
|
|
17
21
|
decodeNameFromPath,
|
|
22
|
+
drawIncarnation,
|
|
23
|
+
drawWrite,
|
|
18
24
|
encodeNameForKey,
|
|
19
25
|
encodeNameForPath,
|
|
20
26
|
encodedNameLength,
|
|
27
|
+
entropyIsAvailable,
|
|
28
|
+
incarnationOf,
|
|
21
29
|
isCloudRoaringError,
|
|
22
30
|
isIntegrityError,
|
|
23
31
|
isNotFoundError,
|
|
@@ -27,18 +35,23 @@ import {
|
|
|
27
35
|
isValidationError,
|
|
28
36
|
isWriteConflictError,
|
|
29
37
|
mapWithConcurrency,
|
|
38
|
+
metadataBytes,
|
|
39
|
+
metadataFromBytes,
|
|
30
40
|
namespacePathPart,
|
|
41
|
+
newIncarnationToken,
|
|
42
|
+
nextRegistryToken,
|
|
31
43
|
parseRegistryEnvelope,
|
|
32
44
|
recordFromNew,
|
|
33
|
-
|
|
45
|
+
sameIncarnation,
|
|
34
46
|
serializeRegistryEnvelope,
|
|
35
47
|
validateChunkRef,
|
|
36
48
|
validateNewRegistryRecord,
|
|
37
49
|
validateRegistryPatch,
|
|
38
50
|
validateSegmentRef,
|
|
39
51
|
validateUserNamespace,
|
|
40
|
-
validateUserRef
|
|
41
|
-
|
|
52
|
+
validateUserRef,
|
|
53
|
+
webCryptoEntropy
|
|
54
|
+
} from "./chunk-5W3XS2IU.js";
|
|
42
55
|
|
|
43
56
|
// src/core/bit-route.ts
|
|
44
57
|
var U32_MAX = 4294967295;
|
|
@@ -255,6 +268,37 @@ function keysWithin(keys, w) {
|
|
|
255
268
|
};
|
|
256
269
|
return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
|
|
257
270
|
}
|
|
271
|
+
var ChunkWindow = class {
|
|
272
|
+
constructor(keys, fetch, max, ramp) {
|
|
273
|
+
this.keys = keys;
|
|
274
|
+
this.fetch = fetch;
|
|
275
|
+
this.max = max;
|
|
276
|
+
this.ramp = ramp;
|
|
277
|
+
}
|
|
278
|
+
keys;
|
|
279
|
+
fetch;
|
|
280
|
+
max;
|
|
281
|
+
ramp;
|
|
282
|
+
open = [];
|
|
283
|
+
launched = 0;
|
|
284
|
+
taken = 0;
|
|
285
|
+
/** The next chunk in key order (null if the source holds none), or the error its read raised. */
|
|
286
|
+
async take() {
|
|
287
|
+
const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
|
|
288
|
+
while (this.launched < this.keys.length && this.launched - this.taken < width) {
|
|
289
|
+
this.open.push(
|
|
290
|
+
this.fetch(this.keys[this.launched++]).then(
|
|
291
|
+
(chunk) => ({ chunk }),
|
|
292
|
+
(cause) => ({ chunk: null, error: { cause } })
|
|
293
|
+
)
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const slot = await this.open.shift();
|
|
297
|
+
this.taken += 1;
|
|
298
|
+
if (slot.error) throw slot.error.cause;
|
|
299
|
+
return slot.chunk;
|
|
300
|
+
}
|
|
301
|
+
};
|
|
258
302
|
var SegmentEngine = class {
|
|
259
303
|
storage;
|
|
260
304
|
cache;
|
|
@@ -282,15 +326,19 @@ var SegmentEngine = class {
|
|
|
282
326
|
return chunk ? chunk.has(remainder) : false;
|
|
283
327
|
}
|
|
284
328
|
/**
|
|
285
|
-
* Cardinality. When the Storage source can
|
|
286
|
-
*
|
|
287
|
-
*
|
|
329
|
+
* Cardinality. When the Storage source can describe the current generation (the `.crbm` source can), the count is
|
|
330
|
+
* that description's: the registry row's summary when the row has one it can use, which is one request when cold
|
|
331
|
+
* and no read of the object, and otherwise the sum of the object's index. Either way, **zero payload reads**. A
|
|
332
|
+
* source that can only serve per-chunk cardinality sums those, and one with neither (the in-memory source) falls back
|
|
333
|
+
* to fetching every chunk.
|
|
288
334
|
*
|
|
289
|
-
* The index sum
|
|
335
|
+
* The row's summary and the index sum are trusted, not confirmed: the source holds the summary against the object
|
|
336
|
+
* whenever it opens it, and the reader checked the index for internal consistency when it opened
|
|
290
337
|
* the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
|
|
291
338
|
* combines decode the payloads.
|
|
292
339
|
*/
|
|
293
340
|
async count(seg) {
|
|
341
|
+
if (this.storage.summary) return (await this.storage.summary(seg))?.cardinality ?? 0;
|
|
294
342
|
const cardinalities = this.storage.cardinalities ? await this.storage.cardinalities(seg) : null;
|
|
295
343
|
if (cardinalities) {
|
|
296
344
|
let total2 = 0;
|
|
@@ -303,12 +351,22 @@ var SegmentEngine = class {
|
|
|
303
351
|
const chunkKeys = await this.chunkKeys(seg);
|
|
304
352
|
checkBudget(this.budget, chunkKeys.length, "count");
|
|
305
353
|
const gen = await this.cacheVersion(seg);
|
|
354
|
+
const window = this.chunkWindow(seg, chunkKeys, gen, false);
|
|
306
355
|
let total = 0;
|
|
307
|
-
for (
|
|
308
|
-
total += (await this.storageChunk({ ...seg, chunkKey }, gen))?.size ?? 0;
|
|
309
|
-
}
|
|
356
|
+
for (let i = 0; i < chunkKeys.length; i++) total += (await window.take())?.size ?? 0;
|
|
310
357
|
return total;
|
|
311
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* What the segment's current generation is: its number, its id count and its metadata, from one resolution, so the
|
|
361
|
+
* three cannot straddle a publish. A source with no summary answers from `count()` and the generation it resolves.
|
|
362
|
+
*/
|
|
363
|
+
async stat(seg) {
|
|
364
|
+
if (this.storage.summary) {
|
|
365
|
+
return await this.storage.summary(seg) ?? { generation: null, cardinality: 0 };
|
|
366
|
+
}
|
|
367
|
+
const generation = this.storage.currentGeneration ? await this.storage.currentGeneration(seg) : null;
|
|
368
|
+
return { generation, cardinality: await this.count(seg) };
|
|
369
|
+
}
|
|
312
370
|
/** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
|
|
313
371
|
get supportsStorageSize() {
|
|
314
372
|
return typeof this.storage.sizeOf === "function";
|
|
@@ -321,8 +379,18 @@ var SegmentEngine = class {
|
|
|
321
379
|
segmentSize(seg) {
|
|
322
380
|
return this.storage.sizeOf ? this.storage.sizeOf(seg) : Promise.resolve(null);
|
|
323
381
|
}
|
|
382
|
+
/** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
|
|
383
|
+
chunkWindow(seg, chunkKeys, gen, ramp) {
|
|
384
|
+
return new ChunkWindow(
|
|
385
|
+
chunkKeys,
|
|
386
|
+
(chunkKey) => this.storageChunk({ ...seg, chunkKey }, gen),
|
|
387
|
+
DEFAULT_INTERSECT_CONCURRENCY,
|
|
388
|
+
ramp
|
|
389
|
+
);
|
|
390
|
+
}
|
|
324
391
|
/**
|
|
325
|
-
* Every id, ascending,
|
|
392
|
+
* Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
|
|
393
|
+
* read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
|
|
326
394
|
* chunks the range overlaps (see {@link IdRange}).
|
|
327
395
|
*
|
|
328
396
|
* Two generators, not one with a branch: a second `yield` site in the full read's generator grows its frame, and
|
|
@@ -336,8 +404,9 @@ var SegmentEngine = class {
|
|
|
336
404
|
const chunkKeys = await this.chunkKeys(seg);
|
|
337
405
|
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
338
406
|
const gen = await this.cacheVersion(seg);
|
|
407
|
+
const window = this.chunkWindow(seg, chunkKeys, gen, true);
|
|
339
408
|
for (const chunkKey of chunkKeys) {
|
|
340
|
-
const chunk = await
|
|
409
|
+
const chunk = await window.take();
|
|
341
410
|
if (chunk === null) continue;
|
|
342
411
|
for (const remainder of chunk) yield joinId(chunkKey, remainder);
|
|
343
412
|
}
|
|
@@ -348,8 +417,9 @@ var SegmentEngine = class {
|
|
|
348
417
|
const chunkKeys = keysWithin(await this.chunkKeys(seg), w);
|
|
349
418
|
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
350
419
|
const gen = await this.cacheVersion(seg);
|
|
420
|
+
const window = this.chunkWindow(seg, chunkKeys, gen, true);
|
|
351
421
|
for (const chunkKey of chunkKeys) {
|
|
352
|
-
const chunk = await
|
|
422
|
+
const chunk = await window.take();
|
|
353
423
|
if (chunk === null) continue;
|
|
354
424
|
if (isEdge(chunkKey, w)) {
|
|
355
425
|
for (const id of edgeIds(chunk, chunkKey, w)) yield id;
|
|
@@ -382,7 +452,7 @@ var SegmentEngine = class {
|
|
|
382
452
|
* up front (before the fan-out) and threaded into every chunk read, so a concurrent load can't corrupt or tear
|
|
383
453
|
* the result — every chunk read is a whole, checksum-verified, immutable generation. The edge a *long* call can
|
|
384
454
|
* hit is invariant 3's: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
|
|
385
|
-
* operand's not-yet-
|
|
455
|
+
* operand's not-yet-requested chunks may re-resolve forward to the newer generation (a generation hop within one long
|
|
386
456
|
* call) — the call never crashes or returns a torn object, but may mix generations. Three things hop it without
|
|
387
457
|
* waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
|
|
388
458
|
* (`maxOpenSegments`), whose re-read re-resolves fresh; a sweep deleting the generation it was reading, which
|
|
@@ -829,6 +899,111 @@ var BoundedLru = class {
|
|
|
829
899
|
}
|
|
830
900
|
};
|
|
831
901
|
|
|
902
|
+
// src/core/generation-gc.ts
|
|
903
|
+
async function nextGeneration(ref, deps) {
|
|
904
|
+
return aboveEverything(ref, deps, await deps.registry.get(ref));
|
|
905
|
+
}
|
|
906
|
+
async function aboveEverything(ref, deps, record) {
|
|
907
|
+
let highest = record?.currentGen ?? -1;
|
|
908
|
+
for await (const key of deps.storage.list(ref)) {
|
|
909
|
+
if (key.generation > highest) highest = key.generation;
|
|
910
|
+
}
|
|
911
|
+
return highest + 1;
|
|
912
|
+
}
|
|
913
|
+
async function nextLoadGeneration(ref, deps, row) {
|
|
914
|
+
const generation = (row?.currentGen ?? -1) + 1;
|
|
915
|
+
try {
|
|
916
|
+
await deps.storage.getTail({ namespace: ref.namespace, segment: ref.segment, generation }, 0);
|
|
917
|
+
} catch (err) {
|
|
918
|
+
if (isNotFoundError(err)) return { generation, checked: true };
|
|
919
|
+
}
|
|
920
|
+
return { generation: await aboveEverything(ref, deps, row), checked: false };
|
|
921
|
+
}
|
|
922
|
+
var LIST_COLLECTION_CADENCE = 16;
|
|
923
|
+
async function gcOrphanGenerations(ref, deps, options = {}) {
|
|
924
|
+
const keep = options.keep ?? 1;
|
|
925
|
+
if (!Number.isInteger(keep) || keep < 0) {
|
|
926
|
+
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
927
|
+
}
|
|
928
|
+
const record = await deps.registry.get(ref);
|
|
929
|
+
if (record === null) return [];
|
|
930
|
+
const current = record.currentGen;
|
|
931
|
+
const seen = /* @__PURE__ */ new Set();
|
|
932
|
+
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
933
|
+
const gens = [...seen];
|
|
934
|
+
const after = await deps.registry.get(ref);
|
|
935
|
+
if (after === null)
|
|
936
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
937
|
+
if (record.status === "destroyed" && after.token !== record.token) {
|
|
938
|
+
throw new WriteConflictError(
|
|
939
|
+
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
const cutoff = current === null || after.currentGen === null ? null : Math.min(current, after.currentGen);
|
|
943
|
+
const stillCollectable = async () => {
|
|
944
|
+
const still = await deps.registry.get(ref);
|
|
945
|
+
if (still === null) {
|
|
946
|
+
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
947
|
+
}
|
|
948
|
+
const ok = record.status === "destroyed" ? still.token === after.token : cutoff === null || still.currentGen !== null && still.currentGen >= cutoff;
|
|
949
|
+
if (!ok) {
|
|
950
|
+
throw new WriteConflictError(
|
|
951
|
+
`segment ${ref.segment} changed incarnation while its generations were being collected`
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
};
|
|
955
|
+
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
956
|
+
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
957
|
+
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
958
|
+
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
959
|
+
[]
|
|
960
|
+
) : (
|
|
961
|
+
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
962
|
+
gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
|
|
963
|
+
);
|
|
964
|
+
for (const generation of toDelete) {
|
|
965
|
+
await stillCollectable();
|
|
966
|
+
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
967
|
+
}
|
|
968
|
+
return toDelete;
|
|
969
|
+
}
|
|
970
|
+
async function collectAfterLoad(ref, deps, options) {
|
|
971
|
+
const { generation, keep, byName, proveCurrent } = options;
|
|
972
|
+
if (keep >= generation) return [];
|
|
973
|
+
const periodic = generation % LIST_COLLECTION_CADENCE === 0;
|
|
974
|
+
if (byName && keep <= 1 && !periodic) {
|
|
975
|
+
if (keep === 1 && proveCurrent !== void 0 && !await objectIsThere(deps.storage, proveCurrent)) {
|
|
976
|
+
return gcOrphanGenerations(ref, deps, { keep });
|
|
977
|
+
}
|
|
978
|
+
return collectByName(ref, deps, { generation, keep });
|
|
979
|
+
}
|
|
980
|
+
return gcOrphanGenerations(ref, deps, { keep });
|
|
981
|
+
}
|
|
982
|
+
async function objectIsThere(storage, key) {
|
|
983
|
+
try {
|
|
984
|
+
await storage.getTail(key, 0);
|
|
985
|
+
return true;
|
|
986
|
+
} catch {
|
|
987
|
+
return false;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
async function collectByName(ref, deps, options) {
|
|
991
|
+
const { generation, keep } = options;
|
|
992
|
+
if (!Number.isInteger(keep) || keep < 0 || keep > 1) {
|
|
993
|
+
throw new ValidationError(`collecting by name needs a keep of 0 or 1; got ${String(keep)}`);
|
|
994
|
+
}
|
|
995
|
+
const evicted = generation - keep - 1;
|
|
996
|
+
if (evicted < 0) return [];
|
|
997
|
+
const still = await deps.registry.get(ref);
|
|
998
|
+
if (still === null || still.currentGen === null || still.currentGen < generation) return [];
|
|
999
|
+
await deps.storage.delete({
|
|
1000
|
+
namespace: ref.namespace,
|
|
1001
|
+
segment: ref.segment,
|
|
1002
|
+
generation: evicted
|
|
1003
|
+
});
|
|
1004
|
+
return [evicted];
|
|
1005
|
+
}
|
|
1006
|
+
|
|
832
1007
|
// src/core/reader-defaults.ts
|
|
833
1008
|
var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
|
|
834
1009
|
var REFRESH_RETRY_MS = 500;
|
|
@@ -876,9 +1051,20 @@ var AWS_US_EAST_1_ONDEMAND = deepFreeze({
|
|
|
876
1051
|
var HOURS_PER_MONTH = 730;
|
|
877
1052
|
var SECONDS_PER_MONTH = HOURS_PER_MONTH * 3600;
|
|
878
1053
|
var GIB = 1024 ** 3;
|
|
879
|
-
var
|
|
880
|
-
var
|
|
881
|
-
var
|
|
1054
|
+
var STORE_LOAD_PUT_CLASS = 1 + 1 / LIST_COLLECTION_CADENCE;
|
|
1055
|
+
var STORE_LOAD_POINTER_READS = 3 + 2 / LIST_COLLECTION_CADENCE;
|
|
1056
|
+
var STORE_LOAD_EXISTENCE_CHECKS = 2 - 1 / LIST_COLLECTION_CADENCE;
|
|
1057
|
+
var RETENTION_SWEEP_REQUESTS = deepFreeze({
|
|
1058
|
+
conditionalDelete: {
|
|
1059
|
+
retirement: { reads: 9, writes: 3, deletes: 1 },
|
|
1060
|
+
purge: { reads: 4, writes: 0, deletes: 2 }
|
|
1061
|
+
},
|
|
1062
|
+
tombstoning: {
|
|
1063
|
+
retirement: { reads: 8, writes: 3, deletes: 0 },
|
|
1064
|
+
purge: { reads: 3, writes: 1, deletes: 0 }
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
var shown = (n) => String(Number(n.toFixed(2)));
|
|
882
1068
|
function requireFiniteNonNeg(n, field) {
|
|
883
1069
|
if (n === void 0 || !Number.isFinite(n) || n < 0) {
|
|
884
1070
|
throw new ValidationError(`${field} must be a finite number >= 0; got ${n}`);
|
|
@@ -1015,8 +1201,8 @@ function bytesWords(bytes) {
|
|
|
1015
1201
|
const units = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB"];
|
|
1016
1202
|
let value = bytes;
|
|
1017
1203
|
let unit = 0;
|
|
1018
|
-
const
|
|
1019
|
-
while (unit < units.length - 1 &&
|
|
1204
|
+
const shown2 = (v, u) => u === 0 ? Math.round(v) : Number(v.toFixed(2));
|
|
1205
|
+
while (unit < units.length - 1 && shown2(value, unit) >= 1024) {
|
|
1020
1206
|
value /= 1024;
|
|
1021
1207
|
unit += 1;
|
|
1022
1208
|
}
|
|
@@ -1043,6 +1229,10 @@ function buildReport(input) {
|
|
|
1043
1229
|
requireFiniteNonNeg(storage.getPerMillion, "pricing.storage.getPerMillion");
|
|
1044
1230
|
requireFiniteNonNeg(storage.putPerMillion, "pricing.storage.putPerMillion");
|
|
1045
1231
|
requireFiniteNonNeg(storage.storagePerGiBMonth, "pricing.storage.storagePerGiBMonth");
|
|
1232
|
+
const pointerRead = requireFiniteNonNeg(
|
|
1233
|
+
storage.requestsPerPointerRead ?? 1,
|
|
1234
|
+
"pricing.storage.requestsPerPointerRead"
|
|
1235
|
+
);
|
|
1046
1236
|
const sizedRead = requireFiniteNonNeg(
|
|
1047
1237
|
storage.requestsPerSizedRead ?? 1,
|
|
1048
1238
|
"pricing.storage.requestsPerSizedRead"
|
|
@@ -1089,6 +1279,15 @@ function buildReport(input) {
|
|
|
1089
1279
|
input.workload.genTtlMs ?? DEFAULT_CURRENT_GEN_TTL_MS,
|
|
1090
1280
|
"genTtlMs"
|
|
1091
1281
|
);
|
|
1282
|
+
const retirementsPerMonth = requireFiniteNonNeg(
|
|
1283
|
+
input.workload.retirementsPerMonth ?? 0,
|
|
1284
|
+
"retirementsPerMonth"
|
|
1285
|
+
);
|
|
1286
|
+
const purgesPerMonth = requireFiniteNonNeg(input.workload.purgesPerMonth ?? 0, "purgesPerMonth");
|
|
1287
|
+
const conditionalDelete = input.workload.conditionalDelete ?? true;
|
|
1288
|
+
if (typeof conditionalDelete !== "boolean") {
|
|
1289
|
+
throw new ValidationError(`conditionalDelete must be a boolean; got ${conditionalDelete}`);
|
|
1290
|
+
}
|
|
1092
1291
|
const storageGetUSD = storage.getPerMillion / 1e6;
|
|
1093
1292
|
const putUSD = storage.putPerMillion / 1e6;
|
|
1094
1293
|
const missFraction = 1 - cacheHitRate;
|
|
@@ -1097,16 +1296,26 @@ function buildReport(input) {
|
|
|
1097
1296
|
const refreshes = genTtlMs > 0 ? Math.min(readerProcesses * hotSegments * S * 1e3 / genTtlMs, readsPerSec * S) : 0;
|
|
1098
1297
|
const storageUSD = storedGiB * storage.storagePerGiBMonth;
|
|
1099
1298
|
const readsUSD = readMisses * storageGetUSD;
|
|
1100
|
-
const
|
|
1299
|
+
const getsPerColdOperand = pointerRead + sizedRead;
|
|
1300
|
+
const intersectGets = chunksPerIntersect + getsPerColdOperand * operandsPerIntersect;
|
|
1101
1301
|
const intersectsUSD = intersects * intersectGets * storageGetUSD;
|
|
1102
|
-
const loadGets =
|
|
1302
|
+
const loadGets = STORE_LOAD_POINTER_READS * pointerRead + STORE_LOAD_EXISTENCE_CHECKS;
|
|
1103
1303
|
const loadsUSD = loadsPerMonth * ((requestsPerLoad + STORE_LOAD_PUT_CLASS) * putUSD + loadGets * storageGetUSD);
|
|
1104
|
-
const refreshUSD = refreshes *
|
|
1105
|
-
const
|
|
1304
|
+
const refreshUSD = refreshes * pointerRead * storageGetUSD;
|
|
1305
|
+
const sweep = RETENTION_SWEEP_REQUESTS[conditionalDelete ? "conditionalDelete" : "tombstoning"];
|
|
1306
|
+
const retentionUSD = retirementsPerMonth * (sweep.retirement.reads * storageGetUSD + sweep.retirement.writes * putUSD) + purgesPerMonth * (sweep.purge.reads * storageGetUSD + sweep.purge.writes * putUSD);
|
|
1307
|
+
const total = readsUSD + intersectsUSD + storageUSD + loadsUSD + refreshUSD + retentionUSD;
|
|
1106
1308
|
const headroom = Math.max(0, baselineUSD - storageUSD - refreshUSD);
|
|
1107
1309
|
const readsCross = storageGetUSD > 0 && missFraction > 0 ? headroom / (S * storageGetUSD * missFraction) : Infinity;
|
|
1108
1310
|
const verdict = total <= baselineUSD * 0.1 ? "win-big" : total < baselineUSD ? "win" : "lose-zone";
|
|
1109
|
-
const dominant = Math.max(
|
|
1311
|
+
const dominant = Math.max(
|
|
1312
|
+
readsUSD,
|
|
1313
|
+
intersectsUSD,
|
|
1314
|
+
storageUSD,
|
|
1315
|
+
loadsUSD,
|
|
1316
|
+
refreshUSD,
|
|
1317
|
+
retentionUSD
|
|
1318
|
+
);
|
|
1110
1319
|
let driver = "storage";
|
|
1111
1320
|
if (dominant === readsUSD && readsUSD > 0) driver = "point reads (object GETs)";
|
|
1112
1321
|
else if (dominant === intersectsUSD && intersectsUSD > 0) {
|
|
@@ -1114,6 +1323,8 @@ function buildReport(input) {
|
|
|
1114
1323
|
} else if (dominant === loadsUSD && loadsUSD > 0)
|
|
1115
1324
|
driver = "loads (object, listing and pointer requests)";
|
|
1116
1325
|
else if (dominant === refreshUSD && refreshUSD > 0) driver = "the pointer refresh (object GETs)";
|
|
1326
|
+
else if (dominant === retentionUSD && retentionUSD > 0)
|
|
1327
|
+
driver = "the retention sweep (registry reads and writes)";
|
|
1117
1328
|
const sized = baseline.sized;
|
|
1118
1329
|
const noBytes = Math.round(storageBytes) === 0;
|
|
1119
1330
|
const redisWords = sized === void 0 ? void 0 : noBytes ? `$${baselineUSD.toFixed(2)}/mo cheapest Redis cluster in the catalogue, as no bytes are counted (${clusterWords(sized.cluster)})` : `$${baselineUSD.toFixed(2)}/mo Redis that would hold ${bytesWords(storageBytes)} (${clusterWords(sized.cluster)})`;
|
|
@@ -1123,12 +1334,15 @@ function buildReport(input) {
|
|
|
1123
1334
|
const notes = [
|
|
1124
1335
|
"Same-region egress treated as free; internet egress is not modeled.",
|
|
1125
1336
|
"Request cost is from the supplied workload rates (live-metrics-derived request cost is a later phase).",
|
|
1126
|
-
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${STORE_LOAD_PUT_CLASS} PUT-class and ${loadGets} GETs that store.load() adds (
|
|
1337
|
+
loadsPerMonth > 0 ? `Loads modeled: ${loadsPerMonth}/mo, each ${requestsPerLoad} PUT-class request(s) for the object plus ${shown(STORE_LOAD_PUT_CLASS)} PUT-class and ${shown(loadGets)} GETs that store.load() adds (the pointer, a check that the next generation number is free, a check that the current generation is there, and a listing every ${LIST_COLLECTION_CADENCE}th load).` : "Loads are NOT modeled \u2014 set workload.loadsPerMonth (+ requestsPerLoad for multipart) to include them.",
|
|
1127
1338
|
...intersects > 0 ? [
|
|
1128
|
-
`Intersections priced cold: ${intersectGets} GETs each, ${
|
|
1339
|
+
`Intersections priced cold: ${intersectGets} GETs each, ${getsPerColdOperand} for each of ${operandsPerIntersect} operand(s) plus ${chunksPerIntersect} chunk read(s); cacheHitRate does not apply.`
|
|
1129
1340
|
] : [],
|
|
1130
1341
|
hotSegments > 0 ? genTtlMs > 0 ? `Pointer refresh modeled: ${hotSegments} hot segment(s) in each of ${readerProcesses} reader process(es), each re-reading its pointer at most every ${genTtlMs} ms, and at most once a point read.` : "Pointer refresh: none \u2014 genTtlMs 0 turns the timed refresh off. A reader still re-reads a pointer on an eviction, a swept generation or an invalidation, which this does not price." : readsPerSec > 0 ? "The pointer refresh is NOT modeled \u2014 set workload.hotSegments to the segments each long-lived reader keeps reading, and workload.readerProcesses to how many readers there are." : null,
|
|
1131
1342
|
hotSegments > DEFAULT_MAX_OPEN_SEGMENTS ? `More hot segments in a reader than a store keeps open by default (${DEFAULT_MAX_OPEN_SEGMENTS}): a read of a segment the reader evicted opens it again, a pointer and a tail read, which this does not price. Raise cache.readerMax, and cache.readerMaxBytes, to keep them open.` : null,
|
|
1343
|
+
...retirementsPerMonth > 0 || purgesPerMonth > 0 ? [
|
|
1344
|
+
`Retention sweep modeled: ${retirementsPerMonth}/mo retirements at ${sweep.retirement.reads} reads, ${sweep.retirement.writes} writes and ${sweep.retirement.deletes} delete(s) each, ${purgesPerMonth}/mo purges at ${sweep.purge.reads} reads, ${sweep.purge.writes} writes and ${sweep.purge.deletes} delete(s) each; reads are priced at the GET rate and writes at the PUT rate, deletes at nothing, as S3 bills none` + (conditionalDelete ? "." : "; a registry that only tombstones is read again, two reads per purged segment, by every later full sweep, which is not priced.")
|
|
1345
|
+
] : [],
|
|
1132
1346
|
...input.extraNotes ?? [],
|
|
1133
1347
|
// Last, so the notes a report already carried keep their places.
|
|
1134
1348
|
redisNote
|
|
@@ -1140,7 +1354,8 @@ function buildReport(input) {
|
|
|
1140
1354
|
intersects: intersectsUSD,
|
|
1141
1355
|
storage: storageUSD,
|
|
1142
1356
|
loads: loadsUSD,
|
|
1143
|
-
pointerRefresh: refreshUSD
|
|
1357
|
+
pointerRefresh: refreshUSD,
|
|
1358
|
+
retention: retentionUSD
|
|
1144
1359
|
},
|
|
1145
1360
|
total
|
|
1146
1361
|
},
|
|
@@ -1249,7 +1464,10 @@ async function drainRegistry(registry, options) {
|
|
|
1249
1464
|
const advice = options.narrowable === false ? `${raise[0].toUpperCase()}${raise.slice(1)} if the namespace really is that large and the memory is ` : `Narrow it with \`namespace\`, or ${raise} if the fleet really is that large and the memory is `;
|
|
1250
1465
|
const rows = [];
|
|
1251
1466
|
for await (const rec of registry.list(options.namespace)) {
|
|
1252
|
-
if (options.namespace === void 0 && isReservedRow(rec))
|
|
1467
|
+
if (options.namespace === void 0 && isReservedRow(rec)) {
|
|
1468
|
+
options.onReserved?.(rec);
|
|
1469
|
+
continue;
|
|
1470
|
+
}
|
|
1253
1471
|
if (rows.length >= maxScanSegments) {
|
|
1254
1472
|
throw new BudgetExceededError(
|
|
1255
1473
|
`${op} would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. ${advice}available (a record is a few hundred bytes resident).`
|
|
@@ -1421,16 +1639,19 @@ var BufferReader = class {
|
|
|
1421
1639
|
var MemoryRegistryDriver = class {
|
|
1422
1640
|
rows = /* @__PURE__ */ new Map();
|
|
1423
1641
|
now;
|
|
1642
|
+
entropy;
|
|
1424
1643
|
seq = 0;
|
|
1425
1644
|
constructor(options = {}) {
|
|
1426
1645
|
this.now = options.now ?? (() => Date.now());
|
|
1646
|
+
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
1427
1647
|
}
|
|
1428
|
-
nextToken() {
|
|
1648
|
+
nextToken(incarnation) {
|
|
1429
1649
|
this.seq += 1;
|
|
1430
|
-
return
|
|
1650
|
+
return `${incarnation}.${this.seq}.${drawWrite(this.entropy)}`;
|
|
1431
1651
|
}
|
|
1652
|
+
/** `conditionalDelete`: a delete checks the token and removes the row in one step, with no await between. */
|
|
1432
1653
|
capabilities() {
|
|
1433
|
-
return { strongRead: true };
|
|
1654
|
+
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
1434
1655
|
}
|
|
1435
1656
|
async get(ref) {
|
|
1436
1657
|
validateSegmentRef(ref);
|
|
@@ -1439,25 +1660,25 @@ var MemoryRegistryDriver = class {
|
|
|
1439
1660
|
}
|
|
1440
1661
|
async create(ref, record) {
|
|
1441
1662
|
validateSegmentRef(ref);
|
|
1442
|
-
validateNewRegistryRecord(record);
|
|
1663
|
+
const checked = validateNewRegistryRecord(record);
|
|
1443
1664
|
const key = segmentKey(ref);
|
|
1444
1665
|
if (this.rows.has(key)) {
|
|
1445
1666
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
1446
1667
|
}
|
|
1447
|
-
const token = this.nextToken();
|
|
1448
|
-
this.rows.set(key, structuredClone(recordFromNew(ref,
|
|
1668
|
+
const token = this.nextToken(drawIncarnation(this.entropy));
|
|
1669
|
+
this.rows.set(key, structuredClone(recordFromNew(ref, checked, this.now(), token)));
|
|
1449
1670
|
return { token };
|
|
1450
1671
|
}
|
|
1451
1672
|
async compareAndSwap(ref, expected, patch) {
|
|
1452
1673
|
validateSegmentRef(ref);
|
|
1453
|
-
validateRegistryPatch(patch);
|
|
1674
|
+
const checked = validateRegistryPatch(patch);
|
|
1454
1675
|
const key = segmentKey(ref);
|
|
1455
1676
|
const existing = this.rows.get(key);
|
|
1456
1677
|
if (!existing || existing.token !== expected) {
|
|
1457
1678
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1458
1679
|
}
|
|
1459
|
-
const token = this.nextToken();
|
|
1460
|
-
this.rows.set(key, structuredClone(applyRegistryPatch(existing,
|
|
1680
|
+
const token = this.nextToken(incarnationOf(existing.token) ?? drawIncarnation(this.entropy));
|
|
1681
|
+
this.rows.set(key, structuredClone(applyRegistryPatch(existing, checked, this.now(), token)));
|
|
1461
1682
|
return { token };
|
|
1462
1683
|
}
|
|
1463
1684
|
async *list(namespace) {
|
|
@@ -1774,43 +1995,48 @@ var LocalFsRegistryDriver = class {
|
|
|
1774
1995
|
constructor(root, options = {}) {
|
|
1775
1996
|
this.root = root;
|
|
1776
1997
|
this.now = options.now ?? (() => Date.now());
|
|
1998
|
+
this.entropy = options.entropy ?? webCryptoEntropy;
|
|
1777
1999
|
}
|
|
1778
2000
|
root;
|
|
1779
2001
|
now;
|
|
2002
|
+
entropy;
|
|
2003
|
+
/**
|
|
2004
|
+
* `conditionalDelete`: a delete reads the row, checks it, and unlinks it under the row's lock, which every writer on
|
|
2005
|
+
* the root in this process takes, so no write can land between the check and the unlink.
|
|
2006
|
+
*/
|
|
1780
2007
|
capabilities() {
|
|
1781
|
-
return { strongRead: true };
|
|
2008
|
+
return entropyIsAvailable(this.entropy) ? { strongRead: true, conditionalDelete: true } : { strongRead: true, canWrite: false, conditionalDelete: true };
|
|
1782
2009
|
}
|
|
1783
2010
|
async get(ref) {
|
|
1784
2011
|
const env = await this.readRow(registryRowPath(this.root, ref));
|
|
1785
2012
|
return env && !env.deleted ? env.record : null;
|
|
1786
2013
|
}
|
|
1787
2014
|
async create(ref, record) {
|
|
1788
|
-
validateNewRegistryRecord(record);
|
|
2015
|
+
const checked = validateNewRegistryRecord(record);
|
|
1789
2016
|
const path = registryRowPath(this.root, ref);
|
|
1790
2017
|
return this.withRowLock(path, async () => {
|
|
1791
2018
|
const current = await this.readRow(path);
|
|
1792
2019
|
if (current !== null && !current.deleted) {
|
|
1793
2020
|
throw new WriteConflictError(`registry row already exists for segment ${ref.segment}`);
|
|
1794
2021
|
}
|
|
1795
|
-
const
|
|
1796
|
-
|
|
1797
|
-
await this.writeRow(path, false, recordFromNew(ref, record, this.now(), token));
|
|
2022
|
+
const token = newIncarnationToken(this.entropy, current?.record);
|
|
2023
|
+
await this.writeRow(path, false, recordFromNew(ref, checked, this.now(), token));
|
|
1798
2024
|
return { token };
|
|
1799
2025
|
});
|
|
1800
2026
|
}
|
|
1801
2027
|
async compareAndSwap(ref, expected, patch) {
|
|
1802
|
-
validateRegistryPatch(patch);
|
|
2028
|
+
const checked = validateRegistryPatch(patch);
|
|
1803
2029
|
const path = registryRowPath(this.root, ref);
|
|
1804
2030
|
return this.withRowLock(path, async () => {
|
|
1805
2031
|
const current = await this.readRow(path);
|
|
1806
2032
|
if (current === null || current.deleted || current.record.token !== expected) {
|
|
1807
2033
|
throw new WriteConflictError(`OCC token mismatch for registry row ${ref.segment}`);
|
|
1808
2034
|
}
|
|
1809
|
-
const token =
|
|
2035
|
+
const token = nextRegistryToken(current.record, this.entropy);
|
|
1810
2036
|
await this.writeRow(
|
|
1811
2037
|
path,
|
|
1812
2038
|
false,
|
|
1813
|
-
applyRegistryPatch(current.record,
|
|
2039
|
+
applyRegistryPatch(current.record, checked, this.now(), token)
|
|
1814
2040
|
);
|
|
1815
2041
|
return { token };
|
|
1816
2042
|
});
|
|
@@ -1844,10 +2070,23 @@ var LocalFsRegistryDriver = class {
|
|
|
1844
2070
|
} else if (current === null || current.deleted) {
|
|
1845
2071
|
return;
|
|
1846
2072
|
}
|
|
1847
|
-
|
|
2073
|
+
if (incarnationOf(current.record.token) !== void 0) {
|
|
2074
|
+
await this.unlinkRow(path);
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
const token = nextRegistryToken(current.record, this.entropy);
|
|
1848
2078
|
await this.writeRow(path, true, { ...current.record, token, updatedAt: this.now() });
|
|
1849
2079
|
});
|
|
1850
2080
|
}
|
|
2081
|
+
/** Remove a row's file and make the removal durable (fsync the directory). */
|
|
2082
|
+
async unlinkRow(path) {
|
|
2083
|
+
try {
|
|
2084
|
+
await unlink2(path);
|
|
2085
|
+
} catch (err) {
|
|
2086
|
+
if (!isCode(err, "ENOENT")) throw mapFsError(err);
|
|
2087
|
+
}
|
|
2088
|
+
await fsyncDir(dirname2(path));
|
|
2089
|
+
}
|
|
1851
2090
|
/** Namespaces to scan: just the one requested, or every namespace dir under the root. */
|
|
1852
2091
|
async namespaceDirs(namespace) {
|
|
1853
2092
|
if (namespace !== void 0) return [namespace];
|
|
@@ -2003,6 +2242,8 @@ function yieldEvery(clock, everyN = YIELD_EVERY) {
|
|
|
2003
2242
|
var AAD_VERSION = 1;
|
|
2004
2243
|
var SCOPE_CHUNK = 0;
|
|
2005
2244
|
var SCOPE_INDEX = 1;
|
|
2245
|
+
var SCOPE_METADATA = 2;
|
|
2246
|
+
var SCOPE_SUMMARY = 3;
|
|
2006
2247
|
function aadFor(ref, generation, scope) {
|
|
2007
2248
|
const enc = new TextEncoder();
|
|
2008
2249
|
const ns = enc.encode(ref.namespace ?? "");
|
|
@@ -2022,12 +2263,82 @@ function aadFor(ref, generation, scope) {
|
|
|
2022
2263
|
o += seg.length;
|
|
2023
2264
|
view.setBigUint64(o, BigInt(generation), true);
|
|
2024
2265
|
o += 8;
|
|
2025
|
-
out[o] = scope === "index" ? SCOPE_INDEX : SCOPE_CHUNK;
|
|
2266
|
+
out[o] = scope === "index" ? SCOPE_INDEX : scope === "metadata" ? SCOPE_METADATA : scope === "summary" ? SCOPE_SUMMARY : SCOPE_CHUNK;
|
|
2026
2267
|
o += 1;
|
|
2027
|
-
view.setUint32(o, scope === "
|
|
2268
|
+
view.setUint32(o, typeof scope === "number" ? scope : 0, true);
|
|
2028
2269
|
return out;
|
|
2029
2270
|
}
|
|
2030
2271
|
|
|
2272
|
+
// src/core/crbm/format.ts
|
|
2273
|
+
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
2274
|
+
var VERSION_MAJOR = 1;
|
|
2275
|
+
var VERSION_MINOR = 0;
|
|
2276
|
+
var PREAMBLE_BYTES = 8;
|
|
2277
|
+
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
2278
|
+
var FOOTER_BYTES = 104;
|
|
2279
|
+
var FOOTER = {
|
|
2280
|
+
indexOffset: 0,
|
|
2281
|
+
// u64
|
|
2282
|
+
indexLength: 8,
|
|
2283
|
+
// u64
|
|
2284
|
+
indexCrc32c: 16,
|
|
2285
|
+
// u32
|
|
2286
|
+
flags: 20,
|
|
2287
|
+
// u32
|
|
2288
|
+
payloadCodecId: 24,
|
|
2289
|
+
// u16 — which codec wrote the chunk payloads
|
|
2290
|
+
elementWidth: 26,
|
|
2291
|
+
// u8
|
|
2292
|
+
containerCodec: 27,
|
|
2293
|
+
// u8
|
|
2294
|
+
versionMajor: 28,
|
|
2295
|
+
// u8
|
|
2296
|
+
versionMinor: 29,
|
|
2297
|
+
// u8
|
|
2298
|
+
reserved2: 30,
|
|
2299
|
+
// u16
|
|
2300
|
+
generation: 32,
|
|
2301
|
+
// u64
|
|
2302
|
+
indexNonce: 40,
|
|
2303
|
+
// 12 B
|
|
2304
|
+
indexTag: 52,
|
|
2305
|
+
// 16 B
|
|
2306
|
+
keyId: 68,
|
|
2307
|
+
// 16 B
|
|
2308
|
+
chunkCount: 84,
|
|
2309
|
+
// u32
|
|
2310
|
+
totalCardinality: 88,
|
|
2311
|
+
// u64
|
|
2312
|
+
footerCrc32c: 96,
|
|
2313
|
+
// u32 — covers footer bytes [0, 96)
|
|
2314
|
+
endMagic: 100
|
|
2315
|
+
// char[4]
|
|
2316
|
+
};
|
|
2317
|
+
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
2318
|
+
var FLAG_ENCRYPTED = 1 << 0;
|
|
2319
|
+
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
2320
|
+
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
2321
|
+
var FLAG_EXTENSION = 1 << 3;
|
|
2322
|
+
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
|
|
2323
|
+
var ELEMENT_WIDTH_32 = 32;
|
|
2324
|
+
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
2325
|
+
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
2326
|
+
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
2327
|
+
]);
|
|
2328
|
+
var CONTAINER_CODEC_NONE = 0;
|
|
2329
|
+
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
2330
|
+
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
2331
|
+
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
2332
|
+
var MAX_CHUNK_CARDINALITY = 65536;
|
|
2333
|
+
var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
|
|
2334
|
+
var EXT_TRAILER_BYTES = 12;
|
|
2335
|
+
var EXT_SECTION_HEADER_BYTES = 5;
|
|
2336
|
+
var MAX_EXT_BYTES = 4 * 1024;
|
|
2337
|
+
var EXT_SECTION_METADATA = 1;
|
|
2338
|
+
var CRC32C_BYTES = 4;
|
|
2339
|
+
var AEAD_NONCE_BYTES = 12;
|
|
2340
|
+
var AEAD_TAG_BYTES = 16;
|
|
2341
|
+
|
|
2031
2342
|
// src/core/crbm/crc32c.ts
|
|
2032
2343
|
var TABLES = (() => {
|
|
2033
2344
|
const POLY = 2197175160;
|
|
@@ -2097,7 +2408,7 @@ function readVarint(bytes, offset) {
|
|
|
2097
2408
|
if (i === 4 && byte > 15) {
|
|
2098
2409
|
throw new IntegrityError("varint exceeds u32 range");
|
|
2099
2410
|
}
|
|
2100
|
-
result
|
|
2411
|
+
result = (result | (byte & 127) << shift) >>> 0;
|
|
2101
2412
|
if ((byte & 128) === 0) {
|
|
2102
2413
|
return { value: result, next: pos };
|
|
2103
2414
|
}
|
|
@@ -2106,72 +2417,13 @@ function readVarint(bytes, offset) {
|
|
|
2106
2417
|
throw new IntegrityError("varint exceeds u32 range");
|
|
2107
2418
|
}
|
|
2108
2419
|
|
|
2109
|
-
// src/core/crbm/format.ts
|
|
2110
|
-
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
2111
|
-
var VERSION_MAJOR = 1;
|
|
2112
|
-
var VERSION_MINOR = 0;
|
|
2113
|
-
var PREAMBLE_BYTES = 8;
|
|
2114
|
-
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
2115
|
-
var FOOTER_BYTES = 104;
|
|
2116
|
-
var FOOTER = {
|
|
2117
|
-
indexOffset: 0,
|
|
2118
|
-
// u64
|
|
2119
|
-
indexLength: 8,
|
|
2120
|
-
// u64
|
|
2121
|
-
indexCrc32c: 16,
|
|
2122
|
-
// u32
|
|
2123
|
-
flags: 20,
|
|
2124
|
-
// u32
|
|
2125
|
-
payloadCodecId: 24,
|
|
2126
|
-
// u16 — which codec wrote the chunk payloads
|
|
2127
|
-
elementWidth: 26,
|
|
2128
|
-
// u8
|
|
2129
|
-
containerCodec: 27,
|
|
2130
|
-
// u8
|
|
2131
|
-
versionMajor: 28,
|
|
2132
|
-
// u8
|
|
2133
|
-
versionMinor: 29,
|
|
2134
|
-
// u8
|
|
2135
|
-
reserved2: 30,
|
|
2136
|
-
// u16
|
|
2137
|
-
generation: 32,
|
|
2138
|
-
// u64
|
|
2139
|
-
indexNonce: 40,
|
|
2140
|
-
// 12 B
|
|
2141
|
-
indexTag: 52,
|
|
2142
|
-
// 16 B
|
|
2143
|
-
keyId: 68,
|
|
2144
|
-
// 16 B
|
|
2145
|
-
chunkCount: 84,
|
|
2146
|
-
// u32
|
|
2147
|
-
totalCardinality: 88,
|
|
2148
|
-
// u64
|
|
2149
|
-
footerCrc32c: 96,
|
|
2150
|
-
// u32 — covers footer bytes [0, 96)
|
|
2151
|
-
endMagic: 100
|
|
2152
|
-
// char[4]
|
|
2153
|
-
};
|
|
2154
|
-
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
2155
|
-
var FLAG_ENCRYPTED = 1 << 0;
|
|
2156
|
-
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
2157
|
-
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
2158
|
-
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN;
|
|
2159
|
-
var ELEMENT_WIDTH_32 = 32;
|
|
2160
|
-
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
2161
|
-
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
2162
|
-
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
2163
|
-
]);
|
|
2164
|
-
var CONTAINER_CODEC_NONE = 0;
|
|
2165
|
-
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
2166
|
-
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
2167
|
-
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
2168
|
-
var MAX_CHUNK_CARDINALITY = 65536;
|
|
2169
|
-
var CRC32C_BYTES = 4;
|
|
2170
|
-
var AEAD_NONCE_BYTES = 12;
|
|
2171
|
-
var AEAD_TAG_BYTES = 16;
|
|
2172
|
-
|
|
2173
2420
|
// src/core/crbm/reader.ts
|
|
2174
|
-
var
|
|
2421
|
+
var RETAINED_BYTES_PER_METADATA_BYTE = 2;
|
|
2422
|
+
var RETAINED_BYTES_PER_METADATA_KEY = 160;
|
|
2423
|
+
var MIN_INDEX_RECORD_BYTES = 4 + CRC32C_BYTES;
|
|
2424
|
+
function indexCapacity(indexLength) {
|
|
2425
|
+
return Math.min(65536, Math.floor(indexLength / MIN_INDEX_RECORD_BYTES));
|
|
2426
|
+
}
|
|
2175
2427
|
function magicMatches(bytes, offset) {
|
|
2176
2428
|
return bytes[offset] === MAGIC[0] && bytes[offset + 1] === MAGIC[1] && bytes[offset + 2] === MAGIC[2] && bytes[offset + 3] === MAGIC[3];
|
|
2177
2429
|
}
|
|
@@ -2198,6 +2450,11 @@ function checkedFooter(tail, size) {
|
|
|
2198
2450
|
}
|
|
2199
2451
|
return { footer, fview, storedFooterCrc };
|
|
2200
2452
|
}
|
|
2453
|
+
async function footerSaysEncrypted(blob) {
|
|
2454
|
+
const { bytes: tail, size } = await blob.getTail(FOOTER_BYTES);
|
|
2455
|
+
const { fview } = checkedFooter(tail, size);
|
|
2456
|
+
return (fview.getUint32(FOOTER.flags, true) & FLAG_ENCRYPTED) !== 0;
|
|
2457
|
+
}
|
|
2201
2458
|
function readU64(view, offset, field) {
|
|
2202
2459
|
const big = view.getBigUint64(offset, true);
|
|
2203
2460
|
if (big > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
@@ -2206,28 +2463,30 @@ function readU64(view, offset, field) {
|
|
|
2206
2463
|
return Number(big);
|
|
2207
2464
|
}
|
|
2208
2465
|
var CrbmReader = class _CrbmReader {
|
|
2209
|
-
constructor(blob, objectSize, generation, lineage, totalCardinality,
|
|
2466
|
+
constructor(blob, objectSize, generation, lineage, totalCardinality, index, servedFromTail, crypto, footerCrc, metadata, metadataWeight) {
|
|
2210
2467
|
this.blob = blob;
|
|
2211
2468
|
this.objectSize = objectSize;
|
|
2212
2469
|
this.generation = generation;
|
|
2213
2470
|
this.lineage = lineage;
|
|
2214
2471
|
this.totalCardinality = totalCardinality;
|
|
2215
|
-
this.
|
|
2216
|
-
this.orderedKeys = orderedKeys;
|
|
2472
|
+
this.index = index;
|
|
2217
2473
|
this.servedFromTail = servedFromTail;
|
|
2218
2474
|
this.crypto = crypto;
|
|
2219
2475
|
this.footerCrc = footerCrc;
|
|
2476
|
+
this.metadata = metadata;
|
|
2477
|
+
this.metadataWeight = metadataWeight;
|
|
2220
2478
|
}
|
|
2221
2479
|
blob;
|
|
2222
2480
|
objectSize;
|
|
2223
2481
|
generation;
|
|
2224
2482
|
lineage;
|
|
2225
2483
|
totalCardinality;
|
|
2226
|
-
|
|
2227
|
-
orderedKeys;
|
|
2484
|
+
index;
|
|
2228
2485
|
servedFromTail;
|
|
2229
2486
|
crypto;
|
|
2230
2487
|
footerCrc;
|
|
2488
|
+
metadata;
|
|
2489
|
+
metadataWeight;
|
|
2231
2490
|
/**
|
|
2232
2491
|
* Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
|
|
2233
2492
|
* objects with the same fingerprint hold the same chunks, as far as a CRC can tell; a pin compares it to know that
|
|
@@ -2241,17 +2500,28 @@ var CrbmReader = class _CrbmReader {
|
|
|
2241
2500
|
return this.objectSize;
|
|
2242
2501
|
}
|
|
2243
2502
|
/**
|
|
2244
|
-
*
|
|
2245
|
-
*
|
|
2246
|
-
*
|
|
2503
|
+
* Retained JS heap of this reader's parsed index — the weight the storage reader cache bounds on (a wide
|
|
2504
|
+
* segment's parsed index, not its payloads, dominates the reader's footprint): the exact byte length of the
|
|
2505
|
+
* arrays that hold it, {@link RETAINED_BYTES_PER_INDEX_ENTRY} per entry.
|
|
2247
2506
|
*/
|
|
2248
2507
|
get retainedIndexBytes() {
|
|
2249
|
-
|
|
2508
|
+
const { keys, cardinalityMinusOne, lengths, crcs, offsets } = this.index;
|
|
2509
|
+
return keys.byteLength + cardinalityMinusOne.byteLength + lengths.byteLength + crcs.byteLength + offsets.byteLength;
|
|
2510
|
+
}
|
|
2511
|
+
/**
|
|
2512
|
+
* What this reader holds, the weight the storage reader cache bounds on: its parsed index
|
|
2513
|
+
* ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
|
|
2514
|
+
* of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
|
|
2515
|
+
* how the metadata is weighed may change in any release.
|
|
2516
|
+
*/
|
|
2517
|
+
get retainedBytes() {
|
|
2518
|
+
return this.retainedIndexBytes + this.metadataWeight;
|
|
2250
2519
|
}
|
|
2251
2520
|
/** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
|
|
2252
2521
|
cardinalities() {
|
|
2522
|
+
const { keys, cardinalityMinusOne } = this.index;
|
|
2253
2523
|
const out = /* @__PURE__ */ new Map();
|
|
2254
|
-
for (
|
|
2524
|
+
for (let i = 0; i < keys.length; i++) out.set(keys[i], cardinalityMinusOne[i] + 1);
|
|
2255
2525
|
return out;
|
|
2256
2526
|
}
|
|
2257
2527
|
/**
|
|
@@ -2313,6 +2583,13 @@ var CrbmReader = class _CrbmReader {
|
|
|
2313
2583
|
const chunkCount = fview.getUint32(FOOTER.chunkCount, true);
|
|
2314
2584
|
const totalCardinality = readU64(fview, FOOTER.totalCardinality, "total_cardinality");
|
|
2315
2585
|
const generation = readU64(fview, FOOTER.generation, "generation");
|
|
2586
|
+
if (!encrypted && options.crypto !== void 0) {
|
|
2587
|
+
throw new IntegrityError(
|
|
2588
|
+
`.crbm generation ${generation} is not encrypted, but it was opened with a key: a cleartext object where an encrypted one belongs is forged, corrupt, or a write that never published (a crash, before the segment's key was made); a load's collection, dropSegment or deleting the object removes it`
|
|
2589
|
+
);
|
|
2590
|
+
}
|
|
2591
|
+
const versionMinor = footer[FOOTER.versionMinor];
|
|
2592
|
+
const hasExtension = (flags & FLAG_EXTENSION) !== 0;
|
|
2316
2593
|
if (indexOffset < PAYLOAD_START || indexOffset + indexLength > size - FOOTER_BYTES) {
|
|
2317
2594
|
throw new IntegrityError(
|
|
2318
2595
|
`.crbm index region [${indexOffset}, +${indexLength}) out of bounds`
|
|
@@ -2323,20 +2600,27 @@ var CrbmReader = class _CrbmReader {
|
|
|
2323
2600
|
}
|
|
2324
2601
|
const tailCoversFrom = size - tail.length;
|
|
2325
2602
|
if (tailCoversFrom === 0) {
|
|
2326
|
-
if (!magicMatches(tail, 0) || tail[4] !== versionMajor) {
|
|
2603
|
+
if (!magicMatches(tail, 0) || tail[4] !== versionMajor || tail[5] !== versionMinor) {
|
|
2327
2604
|
throw new IntegrityError(".crbm preamble magic/version mismatch with footer");
|
|
2328
2605
|
}
|
|
2329
2606
|
}
|
|
2330
|
-
|
|
2607
|
+
const lowest = hasExtension ? Math.max(PAYLOAD_START, indexOffset - EXT_TRAILER_BYTES - MAX_EXT_BYTES) : indexOffset;
|
|
2608
|
+
let region;
|
|
2609
|
+
let regionStart;
|
|
2331
2610
|
let servedFromTail;
|
|
2332
2611
|
if (indexOffset >= tailCoversFrom) {
|
|
2333
|
-
|
|
2334
|
-
|
|
2612
|
+
region = tail;
|
|
2613
|
+
regionStart = tailCoversFrom;
|
|
2335
2614
|
servedFromTail = true;
|
|
2336
2615
|
} else {
|
|
2337
|
-
|
|
2616
|
+
region = await blob.getRange(lowest, indexOffset + indexLength - lowest);
|
|
2617
|
+
regionStart = lowest;
|
|
2338
2618
|
servedFromTail = false;
|
|
2339
2619
|
}
|
|
2620
|
+
const indexBytes = region.subarray(
|
|
2621
|
+
indexOffset - regionStart,
|
|
2622
|
+
indexOffset - regionStart + indexLength
|
|
2623
|
+
);
|
|
2340
2624
|
if (crc32c(indexBytes) !== indexCrc) {
|
|
2341
2625
|
throw new IntegrityError(".crbm index CRC mismatch");
|
|
2342
2626
|
}
|
|
@@ -2348,9 +2632,56 @@ var CrbmReader = class _CrbmReader {
|
|
|
2348
2632
|
},
|
|
2349
2633
|
options.crypto.aadFor("index")
|
|
2350
2634
|
) : indexBytes;
|
|
2351
|
-
|
|
2635
|
+
let payloadEnd = indexOffset;
|
|
2636
|
+
let metadata;
|
|
2637
|
+
let metadataLength = 0;
|
|
2638
|
+
if (hasExtension) {
|
|
2639
|
+
const trailerStart = indexOffset - EXT_TRAILER_BYTES;
|
|
2640
|
+
if (trailerStart < PAYLOAD_START) {
|
|
2641
|
+
throw new IntegrityError(".crbm flags an extension block, but has no room for one");
|
|
2642
|
+
}
|
|
2643
|
+
let window = { bytes: region, start: regionStart };
|
|
2644
|
+
if (trailerStart < regionStart) {
|
|
2645
|
+
window = { bytes: await blob.getRange(lowest, indexOffset - lowest), start: lowest };
|
|
2646
|
+
servedFromTail = false;
|
|
2647
|
+
}
|
|
2648
|
+
const trailer = within(window.bytes, window.start, trailerStart, indexOffset);
|
|
2649
|
+
const tview = new DataView(trailer.buffer, trailer.byteOffset, trailer.byteLength);
|
|
2650
|
+
if (!EXT_MAGIC.every((b, i) => trailer[8 + i] === b)) {
|
|
2651
|
+
throw new IntegrityError(".crbm extension block trailer magic mismatch");
|
|
2652
|
+
}
|
|
2653
|
+
const sectionsLength = tview.getUint32(0, true);
|
|
2654
|
+
if (sectionsLength > MAX_EXT_BYTES) {
|
|
2655
|
+
throw new IntegrityError(
|
|
2656
|
+
`.crbm extension block ${sectionsLength}B exceeds cap ${MAX_EXT_BYTES}B`
|
|
2657
|
+
);
|
|
2658
|
+
}
|
|
2659
|
+
const extStart = trailerStart - sectionsLength;
|
|
2660
|
+
if (extStart < PAYLOAD_START) {
|
|
2661
|
+
throw new IntegrityError(`.crbm extension block of ${sectionsLength}B out of bounds`);
|
|
2662
|
+
}
|
|
2663
|
+
let covered;
|
|
2664
|
+
if (extStart >= window.start) {
|
|
2665
|
+
covered = within(window.bytes, window.start, extStart, trailerStart + 4);
|
|
2666
|
+
} else {
|
|
2667
|
+
covered = await blob.getRange(extStart, sectionsLength + 4);
|
|
2668
|
+
servedFromTail = false;
|
|
2669
|
+
if (covered.length !== sectionsLength + 4) {
|
|
2670
|
+
throw new IntegrityError(".crbm extension block read short");
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
if (crc32c(covered) !== tview.getUint32(4, true)) {
|
|
2674
|
+
throw new IntegrityError(".crbm extension block CRC mismatch");
|
|
2675
|
+
}
|
|
2676
|
+
({ metadata, metadataLength } = extensionContents(
|
|
2677
|
+
covered.subarray(0, sectionsLength),
|
|
2678
|
+
options.crypto
|
|
2679
|
+
));
|
|
2680
|
+
payloadEnd = extStart;
|
|
2681
|
+
}
|
|
2682
|
+
const index = parseIndex(
|
|
2352
2683
|
indexForParse,
|
|
2353
|
-
|
|
2684
|
+
payloadEnd,
|
|
2354
2685
|
maxPayloadBytes,
|
|
2355
2686
|
encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
|
|
2356
2687
|
);
|
|
@@ -2361,14 +2692,14 @@ var CrbmReader = class _CrbmReader {
|
|
|
2361
2692
|
);
|
|
2362
2693
|
}
|
|
2363
2694
|
} else {
|
|
2364
|
-
if (
|
|
2695
|
+
if (index.keys.length !== chunkCount) {
|
|
2365
2696
|
throw new IntegrityError(
|
|
2366
|
-
`.crbm chunk_count ${chunkCount} != ${
|
|
2697
|
+
`.crbm chunk_count ${chunkCount} != ${index.keys.length} index entries`
|
|
2367
2698
|
);
|
|
2368
2699
|
}
|
|
2369
|
-
if (cardinalitySum !== totalCardinality) {
|
|
2700
|
+
if (index.cardinalitySum !== totalCardinality) {
|
|
2370
2701
|
throw new IntegrityError(
|
|
2371
|
-
`.crbm total_cardinality ${totalCardinality} != \u03A3 index cardinality ${cardinalitySum}`
|
|
2702
|
+
`.crbm total_cardinality ${totalCardinality} != \u03A3 index cardinality ${index.cardinalitySum}`
|
|
2372
2703
|
);
|
|
2373
2704
|
}
|
|
2374
2705
|
}
|
|
@@ -2377,24 +2708,39 @@ var CrbmReader = class _CrbmReader {
|
|
|
2377
2708
|
size,
|
|
2378
2709
|
generation,
|
|
2379
2710
|
options.lineage,
|
|
2380
|
-
cardinalitySum,
|
|
2381
|
-
|
|
2382
|
-
orderedKeys,
|
|
2711
|
+
index.cardinalitySum,
|
|
2712
|
+
index,
|
|
2383
2713
|
servedFromTail,
|
|
2384
2714
|
options.crypto,
|
|
2385
|
-
storedFooterCrc
|
|
2715
|
+
storedFooterCrc,
|
|
2716
|
+
metadata,
|
|
2717
|
+
metadata === void 0 ? 0 : RETAINED_BYTES_PER_METADATA_BYTE * metadataLength + RETAINED_BYTES_PER_METADATA_KEY * Object.keys(metadata).length
|
|
2386
2718
|
);
|
|
2387
2719
|
}
|
|
2388
2720
|
/** Chunk keys present in this generation, ascending. */
|
|
2389
2721
|
chunkKeys() {
|
|
2390
|
-
return this.
|
|
2722
|
+
return Array.from(this.index.keys);
|
|
2391
2723
|
}
|
|
2392
2724
|
/** Segment cardinality from the footer (no payload reads) — the cheap `count()` path. */
|
|
2393
2725
|
count() {
|
|
2394
2726
|
return this.totalCardinality;
|
|
2395
2727
|
}
|
|
2396
|
-
|
|
2397
|
-
|
|
2728
|
+
/** The slot holding `chunkKey` (binary search over the ascending keys), or -1 when this generation has none. */
|
|
2729
|
+
slotOf(chunkKey) {
|
|
2730
|
+
const keys = this.index.keys;
|
|
2731
|
+
let lo = 0;
|
|
2732
|
+
let hi = keys.length - 1;
|
|
2733
|
+
while (lo <= hi) {
|
|
2734
|
+
const mid = lo + hi >>> 1;
|
|
2735
|
+
const k = keys[mid];
|
|
2736
|
+
if (k === chunkKey) return mid;
|
|
2737
|
+
if (k < chunkKey) lo = mid + 1;
|
|
2738
|
+
else hi = mid - 1;
|
|
2739
|
+
}
|
|
2740
|
+
return -1;
|
|
2741
|
+
}
|
|
2742
|
+
has(chunkKey) {
|
|
2743
|
+
return this.slotOf(chunkKey) >= 0;
|
|
2398
2744
|
}
|
|
2399
2745
|
/**
|
|
2400
2746
|
* Range-GET one chunk's payload, verifying its CRC32C before returning. Returns `null` if the chunk is
|
|
@@ -2403,13 +2749,15 @@ var CrbmReader = class _CrbmReader {
|
|
|
2403
2749
|
* not mutate it.
|
|
2404
2750
|
*/
|
|
2405
2751
|
async getChunk(chunkKey) {
|
|
2406
|
-
const
|
|
2407
|
-
if (
|
|
2408
|
-
|
|
2752
|
+
const slot = this.slotOf(chunkKey);
|
|
2753
|
+
if (slot < 0) return null;
|
|
2754
|
+
const offset = this.index.offsets[slot];
|
|
2755
|
+
const length = this.index.lengths[slot];
|
|
2756
|
+
if (offset < PAYLOAD_START || offset + length > this.objectSize - FOOTER_BYTES) {
|
|
2409
2757
|
throw new IntegrityError(`chunk ${chunkKey} payload out of bounds`);
|
|
2410
2758
|
}
|
|
2411
|
-
const bytes = await this.blob.getRange(
|
|
2412
|
-
if (crc32c(bytes) !==
|
|
2759
|
+
const bytes = await this.blob.getRange(offset, length);
|
|
2760
|
+
if (crc32c(bytes) !== this.index.crcs[slot]) {
|
|
2413
2761
|
throw new IntegrityError(`chunk ${chunkKey} payload CRC mismatch`);
|
|
2414
2762
|
}
|
|
2415
2763
|
if (this.crypto === void 0) return bytes;
|
|
@@ -2426,52 +2774,130 @@ var CrbmReader = class _CrbmReader {
|
|
|
2426
2774
|
);
|
|
2427
2775
|
}
|
|
2428
2776
|
};
|
|
2777
|
+
function within(bytes, start, from, to) {
|
|
2778
|
+
if (from < start || to > start + bytes.length) {
|
|
2779
|
+
throw new IntegrityError(".crbm extension block read short");
|
|
2780
|
+
}
|
|
2781
|
+
return bytes.subarray(from - start, to - start);
|
|
2782
|
+
}
|
|
2783
|
+
function extensionCorrupt(message) {
|
|
2784
|
+
throw new IntegrityError(`.crbm extension block: ${message}`);
|
|
2785
|
+
}
|
|
2786
|
+
function extensionContents(sections, crypto) {
|
|
2787
|
+
const view = new DataView(sections.buffer, sections.byteOffset, sections.byteLength);
|
|
2788
|
+
let metadata;
|
|
2789
|
+
let metadataLength = 0;
|
|
2790
|
+
let lastType = 0;
|
|
2791
|
+
let pos = 0;
|
|
2792
|
+
while (pos < sections.length) {
|
|
2793
|
+
if (sections.length - pos < EXT_SECTION_HEADER_BYTES)
|
|
2794
|
+
extensionCorrupt("a section header is cut off");
|
|
2795
|
+
const type = sections[pos];
|
|
2796
|
+
const length = view.getUint32(pos + 1, true);
|
|
2797
|
+
pos += EXT_SECTION_HEADER_BYTES;
|
|
2798
|
+
if (type <= lastType) {
|
|
2799
|
+
extensionCorrupt(
|
|
2800
|
+
type === 0 ? "section type 0 is not a section type" : `section type ${type} after ${lastType}`
|
|
2801
|
+
);
|
|
2802
|
+
}
|
|
2803
|
+
if (length > sections.length - pos) extensionCorrupt(`section ${type} runs past the block`);
|
|
2804
|
+
const body = sections.subarray(pos, pos + length);
|
|
2805
|
+
pos += length;
|
|
2806
|
+
lastType = type;
|
|
2807
|
+
if (type === EXT_SECTION_METADATA)
|
|
2808
|
+
({ metadata, metadataLength } = metadataSection(body, crypto));
|
|
2809
|
+
}
|
|
2810
|
+
return { metadata, metadataLength };
|
|
2811
|
+
}
|
|
2812
|
+
function metadataSection(body, crypto) {
|
|
2813
|
+
if (crypto === void 0) {
|
|
2814
|
+
return { metadata: metadataFromBytes(body, extensionCorrupt), metadataLength: body.length };
|
|
2815
|
+
}
|
|
2816
|
+
const framing = AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
|
|
2817
|
+
if (body.length <= framing || body.length > framing + MAX_METADATA_BYTES) {
|
|
2818
|
+
extensionCorrupt(`sealed metadata of ${body.length}B is not a nonce, up to 1 KiB and a tag`);
|
|
2819
|
+
}
|
|
2820
|
+
let plain;
|
|
2821
|
+
try {
|
|
2822
|
+
plain = crypto.aead.open(
|
|
2823
|
+
{
|
|
2824
|
+
nonce: body.subarray(0, AEAD_NONCE_BYTES),
|
|
2825
|
+
ciphertext: body.subarray(AEAD_NONCE_BYTES, body.length - AEAD_TAG_BYTES),
|
|
2826
|
+
tag: body.subarray(body.length - AEAD_TAG_BYTES)
|
|
2827
|
+
},
|
|
2828
|
+
crypto.aadFor("metadata")
|
|
2829
|
+
);
|
|
2830
|
+
} catch (err) {
|
|
2831
|
+
if (!isIntegrityError(err)) throw err;
|
|
2832
|
+
extensionCorrupt(
|
|
2833
|
+
"the sealed metadata does not open: tampered bytes, a block moved from another object, or a CrbmCrypto whose aadFor does not map the 'metadata' scope"
|
|
2834
|
+
);
|
|
2835
|
+
}
|
|
2836
|
+
return { metadata: metadataFromBytes(plain, extensionCorrupt), metadataLength: plain.length };
|
|
2837
|
+
}
|
|
2429
2838
|
function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1) {
|
|
2430
|
-
const
|
|
2431
|
-
const
|
|
2839
|
+
const capacity = indexCapacity(indexBytes.length);
|
|
2840
|
+
const keys = new Uint16Array(capacity);
|
|
2841
|
+
const cardinalityMinusOne = new Uint16Array(capacity);
|
|
2842
|
+
const lengths = new Uint32Array(capacity);
|
|
2843
|
+
const crcs = new Uint32Array(capacity);
|
|
2844
|
+
const offsets = new Float64Array(capacity);
|
|
2845
|
+
let count = 0;
|
|
2432
2846
|
let pos = 0;
|
|
2433
2847
|
let prevKey = 0;
|
|
2434
2848
|
let prevEnd = PAYLOAD_START;
|
|
2435
2849
|
let cardinalitySum = 0;
|
|
2436
2850
|
for (let i = 0; pos < indexBytes.length; i++) {
|
|
2437
|
-
const
|
|
2438
|
-
const
|
|
2439
|
-
const
|
|
2440
|
-
const
|
|
2441
|
-
|
|
2851
|
+
const keyDeltaRead = readVarint(indexBytes, pos);
|
|
2852
|
+
const offDeltaRead = readVarint(indexBytes, keyDeltaRead.next);
|
|
2853
|
+
const lenRead = readVarint(indexBytes, offDeltaRead.next);
|
|
2854
|
+
const cardRead = readVarint(indexBytes, lenRead.next);
|
|
2855
|
+
const keyDelta = keyDeltaRead.value;
|
|
2856
|
+
const offDelta = offDeltaRead.value;
|
|
2857
|
+
const len = lenRead.value;
|
|
2858
|
+
const card = cardRead.value;
|
|
2859
|
+
pos = cardRead.next;
|
|
2442
2860
|
if (pos + CRC32C_BYTES > indexBytes.length) {
|
|
2443
2861
|
throw new IntegrityError(".crbm index truncated (missing payload CRC)");
|
|
2444
2862
|
}
|
|
2445
2863
|
const crc = (indexBytes[pos] | indexBytes[pos + 1] << 8 | indexBytes[pos + 2] << 16 | indexBytes[pos + 3] << 24) >>> 0;
|
|
2446
2864
|
pos += CRC32C_BYTES;
|
|
2447
|
-
const chunkKey = prevKey + keyDelta
|
|
2448
|
-
const offset = prevEnd + offDelta
|
|
2449
|
-
if (i > 0 && keyDelta
|
|
2865
|
+
const chunkKey = prevKey + keyDelta;
|
|
2866
|
+
const offset = prevEnd + offDelta;
|
|
2867
|
+
if (i > 0 && keyDelta === 0) {
|
|
2450
2868
|
throw new IntegrityError(`.crbm index has a duplicate chunkKey ${chunkKey}`);
|
|
2451
2869
|
}
|
|
2452
2870
|
if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
|
|
2453
|
-
if (len
|
|
2454
|
-
throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len
|
|
2871
|
+
if (len < minPayloadBytes || len > maxPayloadBytes) {
|
|
2872
|
+
throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len} invalid`);
|
|
2455
2873
|
}
|
|
2456
|
-
if (card
|
|
2457
|
-
throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card
|
|
2874
|
+
if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
|
|
2875
|
+
throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
|
|
2458
2876
|
}
|
|
2459
|
-
if (offset + len
|
|
2877
|
+
if (offset + len > payloadEnd) {
|
|
2460
2878
|
throw new IntegrityError(`.crbm chunk ${chunkKey} payload out of bounds`);
|
|
2461
2879
|
}
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
orderedKeys.push(chunkKey);
|
|
2470
|
-
cardinalitySum += card.value;
|
|
2880
|
+
keys[count] = chunkKey;
|
|
2881
|
+
cardinalityMinusOne[count] = card - 1;
|
|
2882
|
+
lengths[count] = len;
|
|
2883
|
+
crcs[count] = crc;
|
|
2884
|
+
offsets[count] = offset;
|
|
2885
|
+
count++;
|
|
2886
|
+
cardinalitySum += card;
|
|
2471
2887
|
prevKey = chunkKey;
|
|
2472
|
-
prevEnd = offset + len
|
|
2888
|
+
prevEnd = offset + len;
|
|
2889
|
+
}
|
|
2890
|
+
if (count === capacity) {
|
|
2891
|
+
return { keys, cardinalityMinusOne, lengths, crcs, offsets, cardinalitySum };
|
|
2473
2892
|
}
|
|
2474
|
-
return {
|
|
2893
|
+
return {
|
|
2894
|
+
keys: keys.slice(0, count),
|
|
2895
|
+
cardinalityMinusOne: cardinalityMinusOne.slice(0, count),
|
|
2896
|
+
lengths: lengths.slice(0, count),
|
|
2897
|
+
crcs: crcs.slice(0, count),
|
|
2898
|
+
offsets: offsets.slice(0, count),
|
|
2899
|
+
cardinalitySum
|
|
2900
|
+
};
|
|
2475
2901
|
}
|
|
2476
2902
|
|
|
2477
2903
|
// src/core/crbm/writer.ts
|
|
@@ -2484,6 +2910,9 @@ var CrbmWriter = class {
|
|
|
2484
2910
|
`generation must be a non-negative integer; got ${options.generation}`
|
|
2485
2911
|
);
|
|
2486
2912
|
}
|
|
2913
|
+
this.metadata = metadataBytes(options.metadata, (message) => {
|
|
2914
|
+
throw new ValidationError(message);
|
|
2915
|
+
});
|
|
2487
2916
|
}
|
|
2488
2917
|
sink;
|
|
2489
2918
|
options;
|
|
@@ -2493,6 +2922,11 @@ var CrbmWriter = class {
|
|
|
2493
2922
|
totalCardinality = 0;
|
|
2494
2923
|
preambleWritten = false;
|
|
2495
2924
|
finished = false;
|
|
2925
|
+
/** The metadata's canonical JSON, copied at construction; `undefined` leaves the footer's extension flag clear. */
|
|
2926
|
+
metadata;
|
|
2927
|
+
/** Bytes written so far, the footer included once {@link finish} has run. */
|
|
2928
|
+
written = 0;
|
|
2929
|
+
footerCrc32c;
|
|
2496
2930
|
/**
|
|
2497
2931
|
* Append one chunk's payload (the roaring-serialized bytes) with its cardinality. `chunkKey` must be
|
|
2498
2932
|
* strictly greater than the previous one (ascending, no duplicates); `cardinality` is `[1, 65536]`.
|
|
@@ -2514,7 +2948,8 @@ var CrbmWriter = class {
|
|
|
2514
2948
|
}
|
|
2515
2949
|
if (payload.length === 0) throw new ValidationError("chunk payload must be non-empty");
|
|
2516
2950
|
await this.ensurePreamble();
|
|
2517
|
-
const
|
|
2951
|
+
const crypto = this.options.crypto;
|
|
2952
|
+
const stored = crypto ? framed(crypto.aead.seal(payload, crypto.aadFor(chunkKey))) : payload;
|
|
2518
2953
|
await this.sink.write(stored);
|
|
2519
2954
|
this.entries.push({
|
|
2520
2955
|
chunkKey,
|
|
@@ -2527,20 +2962,18 @@ var CrbmWriter = class {
|
|
|
2527
2962
|
this.totalCardinality += cardinality;
|
|
2528
2963
|
this.lastKey = chunkKey;
|
|
2529
2964
|
}
|
|
2530
|
-
/**
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
|
|
2535
|
-
out.set(sealed.nonce, 0);
|
|
2536
|
-
out.set(sealed.ciphertext, sealed.nonce.length);
|
|
2537
|
-
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
2538
|
-
return out;
|
|
2539
|
-
}
|
|
2540
|
-
/** Write the index region + the fixed footer. After this the object is complete and immutable. */
|
|
2965
|
+
/**
|
|
2966
|
+
* Write the extension block (when there is metadata), the index region and the fixed footer. After this the object
|
|
2967
|
+
* is complete and immutable.
|
|
2968
|
+
*/
|
|
2541
2969
|
async finish() {
|
|
2542
2970
|
if (this.finished) throw new ValidationError("CrbmWriter already finished");
|
|
2543
2971
|
await this.ensurePreamble();
|
|
2972
|
+
if (this.metadata !== void 0) {
|
|
2973
|
+
const block = this.extensionBlock(this.metadata);
|
|
2974
|
+
await this.sink.write(block);
|
|
2975
|
+
this.offset += block.length;
|
|
2976
|
+
}
|
|
2544
2977
|
const indexPlain = this.encodeIndex();
|
|
2545
2978
|
let indexRegion = indexPlain;
|
|
2546
2979
|
let indexNonce;
|
|
@@ -2556,8 +2989,22 @@ var CrbmWriter = class {
|
|
|
2556
2989
|
this.offset += indexRegion.length;
|
|
2557
2990
|
const footer = this.buildFooter(indexOffset, indexRegion, indexNonce, indexTag);
|
|
2558
2991
|
await this.sink.write(footer);
|
|
2992
|
+
this.written = this.offset + footer.length;
|
|
2993
|
+
this.footerCrc32c = new DataView(footer.buffer, footer.byteOffset).getUint32(
|
|
2994
|
+
FOOTER.footerCrc32c,
|
|
2995
|
+
true
|
|
2996
|
+
);
|
|
2559
2997
|
this.finished = true;
|
|
2560
2998
|
}
|
|
2999
|
+
/**
|
|
3000
|
+
* The finished object's size and its footer's CRC: what names it, as a reader's fingerprint does. A writer that
|
|
3001
|
+
* has to tell its own object from another stored later under the same key compares these with the object's
|
|
3002
|
+
* footer. Throws before {@link finish}.
|
|
3003
|
+
*/
|
|
3004
|
+
get identity() {
|
|
3005
|
+
if (this.footerCrc32c === void 0) throw new ValidationError("CrbmWriter not finished");
|
|
3006
|
+
return { size: this.written, footerCrc: this.footerCrc32c };
|
|
3007
|
+
}
|
|
2561
3008
|
async ensurePreamble() {
|
|
2562
3009
|
if (this.preambleWritten) return;
|
|
2563
3010
|
const preamble = new Uint8Array(PREAMBLE_BYTES);
|
|
@@ -2567,6 +3014,25 @@ var CrbmWriter = class {
|
|
|
2567
3014
|
await this.sink.write(preamble);
|
|
2568
3015
|
this.preambleWritten = true;
|
|
2569
3016
|
}
|
|
3017
|
+
/**
|
|
3018
|
+
* The extension block: one metadata section (`u8 type ‖ u32 length ‖ bytes`), then the trailer
|
|
3019
|
+
* (`u32 sectionsLength ‖ u32 crc32c(sections ‖ sectionsLength) ‖ "CRBX"`). Encrypted, the section holds
|
|
3020
|
+
* `nonce ‖ ciphertext ‖ tag`, sealed with the metadata scope of the associated data.
|
|
3021
|
+
*/
|
|
3022
|
+
extensionBlock(metadata) {
|
|
3023
|
+
const crypto = this.options.crypto;
|
|
3024
|
+
const body = crypto ? framed(crypto.aead.seal(metadata, crypto.aadFor("metadata"))) : metadata;
|
|
3025
|
+
const sectionsLength = EXT_SECTION_HEADER_BYTES + body.length;
|
|
3026
|
+
const block = new Uint8Array(sectionsLength + EXT_TRAILER_BYTES);
|
|
3027
|
+
const view = new DataView(block.buffer);
|
|
3028
|
+
block[0] = EXT_SECTION_METADATA;
|
|
3029
|
+
view.setUint32(1, body.length, true);
|
|
3030
|
+
block.set(body, EXT_SECTION_HEADER_BYTES);
|
|
3031
|
+
view.setUint32(sectionsLength, sectionsLength, true);
|
|
3032
|
+
view.setUint32(sectionsLength + 4, crc32c(block.subarray(0, sectionsLength + 4)), true);
|
|
3033
|
+
block.set(EXT_MAGIC, sectionsLength + 8);
|
|
3034
|
+
return block;
|
|
3035
|
+
}
|
|
2570
3036
|
encodeIndex() {
|
|
2571
3037
|
const out = [];
|
|
2572
3038
|
let prevKey = 0;
|
|
@@ -2594,7 +3060,11 @@ var CrbmWriter = class {
|
|
|
2594
3060
|
view.setBigUint64(FOOTER.indexOffset, BigInt(indexOffset), true);
|
|
2595
3061
|
view.setBigUint64(FOOTER.indexLength, BigInt(indexRegion.length), true);
|
|
2596
3062
|
view.setUint32(FOOTER.indexCrc32c, crc32c(indexRegion), true);
|
|
2597
|
-
view.setUint32(
|
|
3063
|
+
view.setUint32(
|
|
3064
|
+
FOOTER.flags,
|
|
3065
|
+
FLAG_LITTLE_ENDIAN | (encrypted ? FLAG_ENCRYPTED : 0) | (this.metadata === void 0 ? 0 : FLAG_EXTENSION),
|
|
3066
|
+
true
|
|
3067
|
+
);
|
|
2598
3068
|
view.setUint16(
|
|
2599
3069
|
FOOTER.payloadCodecId,
|
|
2600
3070
|
this.options.payloadCodecId ?? PAYLOAD_CODEC_ROARING_PORTABLE,
|
|
@@ -2614,6 +3084,226 @@ var CrbmWriter = class {
|
|
|
2614
3084
|
return footer;
|
|
2615
3085
|
}
|
|
2616
3086
|
};
|
|
3087
|
+
function framed(sealed) {
|
|
3088
|
+
const out = new Uint8Array(sealed.nonce.length + sealed.ciphertext.length + sealed.tag.length);
|
|
3089
|
+
out.set(sealed.nonce, 0);
|
|
3090
|
+
out.set(sealed.ciphertext, sealed.nonce.length);
|
|
3091
|
+
out.set(sealed.tag, sealed.nonce.length + sealed.ciphertext.length);
|
|
3092
|
+
return out;
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
// src/core/load-input.ts
|
|
3096
|
+
var MAX_SERIALIZED_LOAD_BYTES = 4 + 8192 + 8 * 65536 + 65536 * 8192;
|
|
3097
|
+
var DecodedLoadInput = class {
|
|
3098
|
+
constructor(bitmap) {
|
|
3099
|
+
this.bitmap = bitmap;
|
|
3100
|
+
}
|
|
3101
|
+
bitmap;
|
|
3102
|
+
};
|
|
3103
|
+
function tagOf(value) {
|
|
3104
|
+
return Object.prototype.toString.call(value).slice(8, -1);
|
|
3105
|
+
}
|
|
3106
|
+
var VIEW = Object.getPrototypeOf(Uint8Array.prototype);
|
|
3107
|
+
var viewGetter = (name) => Object.getOwnPropertyDescriptor(VIEW, name)?.get;
|
|
3108
|
+
var bufferOf = viewGetter("buffer");
|
|
3109
|
+
var byteOffsetOf = viewGetter("byteOffset");
|
|
3110
|
+
var byteLengthOf = viewGetter("byteLength");
|
|
3111
|
+
function plainView(bytes, what) {
|
|
3112
|
+
let buffer;
|
|
3113
|
+
let offset;
|
|
3114
|
+
let length;
|
|
3115
|
+
try {
|
|
3116
|
+
buffer = bufferOf.call(bytes);
|
|
3117
|
+
offset = byteOffsetOf.call(bytes);
|
|
3118
|
+
length = byteLengthOf.call(bytes);
|
|
3119
|
+
} catch {
|
|
3120
|
+
throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
|
|
3121
|
+
}
|
|
3122
|
+
return length === 0 ? new Uint8Array(0) : new Uint8Array(buffer, offset, length);
|
|
3123
|
+
}
|
|
3124
|
+
var sharedByteLength = typeof SharedArrayBuffer === "function" ? Object.getOwnPropertyDescriptor(SharedArrayBuffer.prototype, "byteLength")?.get : void 0;
|
|
3125
|
+
function isShared(buffer) {
|
|
3126
|
+
if (sharedByteLength === void 0) return false;
|
|
3127
|
+
try {
|
|
3128
|
+
sharedByteLength.call(buffer);
|
|
3129
|
+
return true;
|
|
3130
|
+
} catch {
|
|
3131
|
+
return false;
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
function overCap(what, length) {
|
|
3135
|
+
return new ValidationError(
|
|
3136
|
+
`${what} is ${length} bytes, more than any canonical 32-bit bitmap serializes to (${MAX_SERIALIZED_LOAD_BYTES}). Call runOptimize() on the bitmap before serializing it.`
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
3139
|
+
function prepareLoadInput(input, codec) {
|
|
3140
|
+
if (typeof input !== "object" || input === null) {
|
|
3141
|
+
throw new ValidationError(
|
|
3142
|
+
`a load takes ids, { serialized } or { bitmap }; got ${String(input)}`
|
|
3143
|
+
);
|
|
3144
|
+
}
|
|
3145
|
+
const tag = tagOf(input);
|
|
3146
|
+
if (tag === "Uint8Array" || tag === "Uint8ClampedArray") {
|
|
3147
|
+
throw new ValidationError(
|
|
3148
|
+
`a load's ids cannot be a ${tag} or Buffer: each byte would be loaded as an id. Pass portable Roaring bytes as { serialized }, and ids as a Uint32Array or an array of numbers.`
|
|
3149
|
+
);
|
|
3150
|
+
}
|
|
3151
|
+
if (Symbol.iterator in input || Symbol.asyncIterator in input) return input;
|
|
3152
|
+
const keys = Object.keys(input);
|
|
3153
|
+
if (keys.length === 1 && keys[0] === "serialized") {
|
|
3154
|
+
return decode(input.serialized, "{ serialized }", codec);
|
|
3155
|
+
}
|
|
3156
|
+
if (keys.length === 1 && keys[0] === "bitmap") {
|
|
3157
|
+
const bitmap = input.bitmap;
|
|
3158
|
+
if (typeof bitmap !== "object" || bitmap === null || typeof bitmap.serialize !== "function") {
|
|
3159
|
+
throw new ValidationError(`{ bitmap } needs an object with serialize('portable')`);
|
|
3160
|
+
}
|
|
3161
|
+
const what = "{ bitmap }'s serialize('portable')";
|
|
3162
|
+
if (typeof bitmap.getSerializationSizeInBytes === "function") {
|
|
3163
|
+
const size = bitmap.getSerializationSizeInBytes("portable");
|
|
3164
|
+
if (typeof size === "number" && size > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, size);
|
|
3165
|
+
}
|
|
3166
|
+
return decode(bitmap.serialize("portable"), what, codec);
|
|
3167
|
+
}
|
|
3168
|
+
throw new ValidationError(
|
|
3169
|
+
`a load takes ids (an iterable of integers), { serialized } or { bitmap }, as the only key; got an object with keys [${keys.join(", ")}]. Load options go in the next argument.`
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
function decode(bytes, what, codec) {
|
|
3173
|
+
if (typeof bytes !== "object" || bytes === null || tagOf(bytes) !== "Uint8Array") {
|
|
3174
|
+
throw new ValidationError(`${what} must be a Uint8Array of portable Roaring bytes`);
|
|
3175
|
+
}
|
|
3176
|
+
const view = plainView(bytes, what);
|
|
3177
|
+
if (view.byteLength > MAX_SERIALIZED_LOAD_BYTES) throw overCap(what, view.byteLength);
|
|
3178
|
+
const own = isShared(view.buffer) ? new Uint8Array(view) : view;
|
|
3179
|
+
try {
|
|
3180
|
+
return new DecodedLoadInput(
|
|
3181
|
+
codec.safeDeserialize(own, MAX_SERIALIZED_LOAD_BYTES, { whole: true })
|
|
3182
|
+
);
|
|
3183
|
+
} catch (err) {
|
|
3184
|
+
if (!isIntegrityError(err)) throw err;
|
|
3185
|
+
throw new ValidationError(`${what} is not a valid bitmap: ${err.message}`);
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
// src/core/summary.ts
|
|
3190
|
+
var NONCE_BYTES = 12;
|
|
3191
|
+
var TAG_BYTES = 16;
|
|
3192
|
+
var COUNT_BYTES = 8;
|
|
3193
|
+
var MAX_CARDINALITY = 2 ** 32;
|
|
3194
|
+
var BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
3195
|
+
function toBase64(bytes) {
|
|
3196
|
+
let out = "";
|
|
3197
|
+
for (let i = 0; i < bytes.length; i += 3) {
|
|
3198
|
+
const a = bytes[i];
|
|
3199
|
+
const b = bytes[i + 1];
|
|
3200
|
+
const c = bytes[i + 2];
|
|
3201
|
+
out += BASE64[a >> 2] + BASE64[(a & 3) << 4 | (b ?? 0) >> 4];
|
|
3202
|
+
out += b === void 0 ? "=" : BASE64[(b & 15) << 2 | (c ?? 0) >> 6];
|
|
3203
|
+
out += c === void 0 ? "=" : BASE64[c & 63];
|
|
3204
|
+
}
|
|
3205
|
+
return out;
|
|
3206
|
+
}
|
|
3207
|
+
function fromBase64(text) {
|
|
3208
|
+
if (text.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(text)) return void 0;
|
|
3209
|
+
const pad = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
3210
|
+
const out = new Uint8Array(text.length / 4 * 3 - pad);
|
|
3211
|
+
let o = 0;
|
|
3212
|
+
for (let i = 0; i < text.length; i += 4) {
|
|
3213
|
+
const n = BASE64.indexOf(text[i]) << 18 | BASE64.indexOf(text[i + 1]) << 12 | Math.max(BASE64.indexOf(text[i + 2]), 0) << 6 | Math.max(BASE64.indexOf(text[i + 3]), 0);
|
|
3214
|
+
if (o < out.length) out[o++] = n >> 16 & 255;
|
|
3215
|
+
if (o < out.length) out[o++] = n >> 8 & 255;
|
|
3216
|
+
if (o < out.length) out[o++] = n & 255;
|
|
3217
|
+
}
|
|
3218
|
+
return out;
|
|
3219
|
+
}
|
|
3220
|
+
function clearSummary(generation, cardinality, metadata) {
|
|
3221
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata };
|
|
3222
|
+
}
|
|
3223
|
+
function sealSummary(aead, ref, generation, cardinality, metadata) {
|
|
3224
|
+
const meta = metadataBytes(metadata, (message) => {
|
|
3225
|
+
throw new IntegrityError(`summary metadata: ${message}`);
|
|
3226
|
+
});
|
|
3227
|
+
const plain = new Uint8Array(COUNT_BYTES + (meta?.length ?? 0));
|
|
3228
|
+
new DataView(plain.buffer).setBigUint64(0, BigInt(cardinality), true);
|
|
3229
|
+
if (meta !== void 0) plain.set(meta, COUNT_BYTES);
|
|
3230
|
+
const { nonce, ciphertext, tag } = aead.seal(plain, aadFor(ref, generation, "summary"));
|
|
3231
|
+
const framed2 = new Uint8Array(nonce.length + ciphertext.length + tag.length);
|
|
3232
|
+
framed2.set(nonce, 0);
|
|
3233
|
+
framed2.set(ciphertext, nonce.length);
|
|
3234
|
+
framed2.set(tag, nonce.length + ciphertext.length);
|
|
3235
|
+
return { generation, sealed: toBase64(framed2) };
|
|
3236
|
+
}
|
|
3237
|
+
function openSummary(aead, ref, summary) {
|
|
3238
|
+
const fail = (message) => {
|
|
3239
|
+
throw new IntegrityError(`registry record summary: ${message}`);
|
|
3240
|
+
};
|
|
3241
|
+
const framed2 = fromBase64(summary.sealed);
|
|
3242
|
+
const framing = NONCE_BYTES + TAG_BYTES;
|
|
3243
|
+
if (framed2 === void 0 || framed2.length < framing + COUNT_BYTES || framed2.length > framing + COUNT_BYTES + MAX_METADATA_BYTES) {
|
|
3244
|
+
fail("the sealed summary is not a nonce, a count, up to 1 KiB of metadata and a tag");
|
|
3245
|
+
}
|
|
3246
|
+
const bytes = framed2;
|
|
3247
|
+
let plain;
|
|
3248
|
+
try {
|
|
3249
|
+
plain = aead.open(
|
|
3250
|
+
{
|
|
3251
|
+
nonce: bytes.subarray(0, NONCE_BYTES),
|
|
3252
|
+
ciphertext: bytes.subarray(NONCE_BYTES, bytes.length - TAG_BYTES),
|
|
3253
|
+
tag: bytes.subarray(bytes.length - TAG_BYTES)
|
|
3254
|
+
},
|
|
3255
|
+
aadFor(ref, summary.generation, "summary")
|
|
3256
|
+
);
|
|
3257
|
+
} catch (err) {
|
|
3258
|
+
if (!isIntegrityError(err)) throw err;
|
|
3259
|
+
return fail(
|
|
3260
|
+
"the sealed summary does not open: tampered bytes, or a summary moved from another generation"
|
|
3261
|
+
);
|
|
3262
|
+
}
|
|
3263
|
+
if (plain.length < COUNT_BYTES) fail("the sealed summary is shorter than its count");
|
|
3264
|
+
const count = new DataView(plain.buffer, plain.byteOffset, plain.byteLength).getBigUint64(
|
|
3265
|
+
0,
|
|
3266
|
+
true
|
|
3267
|
+
);
|
|
3268
|
+
if (count > BigInt(MAX_CARDINALITY)) fail(`cardinality ${count} is outside 0 to 2^32`);
|
|
3269
|
+
const metadata = plain.length === COUNT_BYTES ? void 0 : metadataFromBytes(plain.subarray(COUNT_BYTES), fail);
|
|
3270
|
+
return { cardinality: Number(count), metadata };
|
|
3271
|
+
}
|
|
3272
|
+
function usableSummary(ref, row, aead) {
|
|
3273
|
+
const summary = row.summary;
|
|
3274
|
+
if (summary === void 0 || row.status !== "active" || summary.generation !== row.currentGen) {
|
|
3275
|
+
return void 0;
|
|
3276
|
+
}
|
|
3277
|
+
const keyed = row.wrappedDeks !== void 0 && row.wrappedDeks.length > 0;
|
|
3278
|
+
if ("sealed" in summary) {
|
|
3279
|
+
if (!keyed || aead === void 0) return void 0;
|
|
3280
|
+
try {
|
|
3281
|
+
return openSummary(aead, ref, summary);
|
|
3282
|
+
} catch (err) {
|
|
3283
|
+
if (isIntegrityError(err)) return void 0;
|
|
3284
|
+
throw err;
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
if (keyed) return void 0;
|
|
3288
|
+
return { cardinality: summary.cardinality, metadata: summary.metadata };
|
|
3289
|
+
}
|
|
3290
|
+
function canonicalOrNone(metadata) {
|
|
3291
|
+
if (metadata === void 0) return void 0;
|
|
3292
|
+
const json = canonicalMetadataJson(metadata, (message) => {
|
|
3293
|
+
throw new IntegrityError(`metadata: ${message}`);
|
|
3294
|
+
});
|
|
3295
|
+
return json === "{}" ? void 0 : json;
|
|
3296
|
+
}
|
|
3297
|
+
function summaryAgrees(summary, object) {
|
|
3298
|
+
return summary.cardinality === object.cardinality && canonicalOrNone(summary.metadata) === canonicalOrNone(object.metadata);
|
|
3299
|
+
}
|
|
3300
|
+
function metadataToCarry(object, authenticated) {
|
|
3301
|
+
const empty = (metadata) => metadata === void 0 || Object.keys(metadata).length === 0;
|
|
3302
|
+
return empty(object) && authenticated !== void 0 && !empty(authenticated.metadata) ? authenticated.metadata : object;
|
|
3303
|
+
}
|
|
3304
|
+
function summaryOf(ref, generation, description, aead) {
|
|
3305
|
+
return aead === void 0 ? clearSummary(generation, description.cardinality, description.metadata) : sealSummary(aead, ref, generation, description.cardinality, description.metadata);
|
|
3306
|
+
}
|
|
2617
3307
|
|
|
2618
3308
|
// src/core/crbm-storage-source.ts
|
|
2619
3309
|
function storageBlobReader(driver, key) {
|
|
@@ -2632,6 +3322,88 @@ var notThePinned = (ref, generation) => new NotFoundError(
|
|
|
2632
3322
|
function versionOf(generation, lineage) {
|
|
2633
3323
|
return lineage === void 0 ? String(generation) : `${generation}:${String(lineage)}`;
|
|
2634
3324
|
}
|
|
3325
|
+
function summaryOfGeneration(generation, description) {
|
|
3326
|
+
const { cardinality, metadata } = description;
|
|
3327
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { generation, cardinality } : { generation, cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
3328
|
+
}
|
|
3329
|
+
var sameKeys = (a, b) => a !== void 0 && JSON.stringify(a) === JSON.stringify(b);
|
|
3330
|
+
function once(fn) {
|
|
3331
|
+
let pending;
|
|
3332
|
+
return () => {
|
|
3333
|
+
if (pending === void 0) {
|
|
3334
|
+
const mine = fn();
|
|
3335
|
+
pending = mine;
|
|
3336
|
+
mine.catch(() => {
|
|
3337
|
+
if (pending === mine) pending = void 0;
|
|
3338
|
+
});
|
|
3339
|
+
}
|
|
3340
|
+
return pending;
|
|
3341
|
+
};
|
|
3342
|
+
}
|
|
3343
|
+
var readerIsOf = (reader, live) => reader.generation === live.target.generation && reader.lineage === live.target.lineage;
|
|
3344
|
+
function describe(reader) {
|
|
3345
|
+
let cardinality = 0;
|
|
3346
|
+
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
3347
|
+
return { cardinality, metadata: reader.metadata };
|
|
3348
|
+
}
|
|
3349
|
+
var Snapshot = class _Snapshot {
|
|
3350
|
+
constructor(target, open4, prior) {
|
|
3351
|
+
this.target = target;
|
|
3352
|
+
this.open = open4;
|
|
3353
|
+
this.prior = prior;
|
|
3354
|
+
}
|
|
3355
|
+
target;
|
|
3356
|
+
open;
|
|
3357
|
+
prior;
|
|
3358
|
+
/** When the snapshot's refresh clock started. A refresh that failed transiently moves it back, so it lapses sooner. */
|
|
3359
|
+
installedAtMs = 0;
|
|
3360
|
+
/** Set by the cache, to hear of a reader as it is opened. */
|
|
3361
|
+
onReader;
|
|
3362
|
+
opened;
|
|
3363
|
+
static live(target, open4, prior) {
|
|
3364
|
+
return new _Snapshot(target, open4, prior);
|
|
3365
|
+
}
|
|
3366
|
+
static eager(reader) {
|
|
3367
|
+
const snap = new _Snapshot(void 0, void 0, void 0);
|
|
3368
|
+
snap.opened = reader;
|
|
3369
|
+
return snap;
|
|
3370
|
+
}
|
|
3371
|
+
/** The reader, opened by the first call. */
|
|
3372
|
+
get reader() {
|
|
3373
|
+
if (this.opened === void 0) {
|
|
3374
|
+
const target = this.target;
|
|
3375
|
+
this.opened = target.then(async (live) => {
|
|
3376
|
+
if (live === null) return null;
|
|
3377
|
+
const kept = await this.reuse(live);
|
|
3378
|
+
this.prior = void 0;
|
|
3379
|
+
return kept ?? this.open(live);
|
|
3380
|
+
});
|
|
3381
|
+
this.onReader?.(this.opened);
|
|
3382
|
+
}
|
|
3383
|
+
return this.opened;
|
|
3384
|
+
}
|
|
3385
|
+
/** The reader if one has been asked for, without asking for one. */
|
|
3386
|
+
get openedReader() {
|
|
3387
|
+
return this.opened;
|
|
3388
|
+
}
|
|
3389
|
+
/** The prior snapshot's reader, if it is of what this snapshot resolved. */
|
|
3390
|
+
async reuse(live) {
|
|
3391
|
+
const prior = this.prior;
|
|
3392
|
+
const reader = prior === void 0 ? null : await prior.catch(() => null);
|
|
3393
|
+
return reader !== null && readerIsOf(reader, live) ? reader : null;
|
|
3394
|
+
}
|
|
3395
|
+
/** Keep the prior snapshot's reader when the refresh found its generation, so a count does not drop it. */
|
|
3396
|
+
async adopt(live) {
|
|
3397
|
+
if (this.opened !== void 0 || this.prior === void 0) return;
|
|
3398
|
+
const reader = await this.reuse(live);
|
|
3399
|
+
this.prior = void 0;
|
|
3400
|
+
if (reader !== null && this.opened === void 0) {
|
|
3401
|
+
this.opened = Promise.resolve(reader);
|
|
3402
|
+
this.onReader?.(this.opened);
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
};
|
|
3406
|
+
var SNAPSHOT_BASE_BYTES = 256;
|
|
2635
3407
|
var BULK_FLUSH_IDS = 1 << 20;
|
|
2636
3408
|
var YIELD_EVERY_IDS = 1 << 14;
|
|
2637
3409
|
var CrbmStorageChunkSource = class {
|
|
@@ -2669,6 +3441,10 @@ var CrbmStorageChunkSource = class {
|
|
|
2669
3441
|
maxBytes: maxOpenIndexBytes ?? DEFAULT_MAX_OPEN_INDEX_BYTES,
|
|
2670
3442
|
clock: clock ?? { now: () => 0 }
|
|
2671
3443
|
});
|
|
3444
|
+
this.distrusted = new BoundedLru({
|
|
3445
|
+
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
3446
|
+
clock: { now: () => 0 }
|
|
3447
|
+
});
|
|
2672
3448
|
this.replacedPins = new BoundedLru({
|
|
2673
3449
|
maxEntries: maxOpenSegments ?? DEFAULT_MAX_OPEN_SEGMENTS,
|
|
2674
3450
|
clock: { now: () => 0 }
|
|
@@ -2710,23 +3486,46 @@ var CrbmStorageChunkSource = class {
|
|
|
2710
3486
|
* check again.
|
|
2711
3487
|
*/
|
|
2712
3488
|
invalidations = 0;
|
|
3489
|
+
/**
|
|
3490
|
+
* Generations whose row summary this process found to disagree with their object, by segment and generation. A
|
|
3491
|
+
* summary here is not used again: a count of that generation reads the object. Bounded as the readers are, so a
|
|
3492
|
+
* long-lived process cannot grow it without limit; one that falls out is trusted again until its next open finds
|
|
3493
|
+
* the disagreement again.
|
|
3494
|
+
*/
|
|
3495
|
+
distrusted;
|
|
2713
3496
|
registry;
|
|
2714
3497
|
keystore;
|
|
2715
3498
|
requireEncryption;
|
|
2716
3499
|
readerOptions;
|
|
2717
3500
|
clock;
|
|
2718
3501
|
currentGenTtlMs;
|
|
2719
|
-
/**
|
|
2720
|
-
|
|
3502
|
+
/**
|
|
3503
|
+
* The segment's live snapshot, refreshing on the TTL. Cheap within the TTL window. The snapshot holds the resolved
|
|
3504
|
+
* target, and its reader once a read has asked for one.
|
|
3505
|
+
*/
|
|
3506
|
+
liveSnapshot(ref) {
|
|
2721
3507
|
const key = segmentKey(ref);
|
|
2722
3508
|
const existing = this.snapshots.get(key);
|
|
2723
|
-
if (existing === void 0)
|
|
2724
|
-
|
|
3509
|
+
if (existing === void 0) {
|
|
3510
|
+
return this.install(
|
|
3511
|
+
key,
|
|
3512
|
+
Snapshot.live(this.resolveLive(ref), (live) => this.openLive(ref, live))
|
|
3513
|
+
);
|
|
3514
|
+
}
|
|
3515
|
+
if (!this.expired(existing.installedAtMs)) return existing;
|
|
2725
3516
|
const snap = this.install(
|
|
2726
3517
|
key,
|
|
2727
|
-
|
|
3518
|
+
Snapshot.live(
|
|
3519
|
+
this.refreshedTarget(ref, existing, () => this.retrySoon(snap)),
|
|
3520
|
+
(live) => this.openLive(ref, live),
|
|
3521
|
+
existing.openedReader
|
|
3522
|
+
)
|
|
2728
3523
|
);
|
|
2729
|
-
return snap
|
|
3524
|
+
return snap;
|
|
3525
|
+
}
|
|
3526
|
+
/** The current resolved reader for a segment, refreshing on the TTL. */
|
|
3527
|
+
resolvedReader(ref) {
|
|
3528
|
+
return this.liveSnapshot(ref).reader;
|
|
2730
3529
|
}
|
|
2731
3530
|
/**
|
|
2732
3531
|
* A refresh failed transiently and the prior reader keeps serving: make this snapshot lapse after
|
|
@@ -2736,45 +3535,56 @@ var CrbmStorageChunkSource = class {
|
|
|
2736
3535
|
retrySoon(snap) {
|
|
2737
3536
|
snap.installedAtMs = this.now() - (this.currentGenTtlMs - Math.min(this.currentGenTtlMs, REFRESH_RETRY_MS));
|
|
2738
3537
|
}
|
|
2739
|
-
/**
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
3538
|
+
/**
|
|
3539
|
+
* TTL elapsed: cheaply re-resolve the pointer. The row read is always fresh, so what the new snapshot says of the
|
|
3540
|
+
* generation (its summary included) is what the row says now, never the prior snapshot's. The reader is reopened
|
|
3541
|
+
* only if the generation or the row changed, and only when a read asks for it.
|
|
3542
|
+
*/
|
|
3543
|
+
async refreshedTarget(ref, existing, onTransientFault) {
|
|
3544
|
+
const prior = await existing.target.catch(() => null);
|
|
2743
3545
|
try {
|
|
2744
|
-
|
|
3546
|
+
return await this.resolveLive(ref, prior);
|
|
2745
3547
|
} catch (err) {
|
|
2746
3548
|
if (!isTransientError(err)) throw err;
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
if (target === void 0) {
|
|
2750
|
-
if (current === null) return this.resolveLatest(ref);
|
|
3549
|
+
const alive = prior !== null;
|
|
3550
|
+
if (!alive) return this.resolveLive(ref);
|
|
2751
3551
|
onTransientFault();
|
|
2752
|
-
return
|
|
3552
|
+
return prior;
|
|
2753
3553
|
}
|
|
2754
|
-
if (target === null) return null;
|
|
2755
|
-
if (current !== null && target.generation === current.generation && target.lineage === current.lineage) {
|
|
2756
|
-
return current;
|
|
2757
|
-
}
|
|
2758
|
-
return this.openForTarget(ref, target);
|
|
2759
3554
|
}
|
|
2760
3555
|
/**
|
|
2761
|
-
* Memoize a
|
|
2762
|
-
*
|
|
2763
|
-
*
|
|
3556
|
+
* Memoize a snapshot as the segment's at `now()`, forgetting it later if it resolves to null / throws
|
|
3557
|
+
* (identity-guarded, so a stale cleanup never clobbers a fresher snapshot). Installing the (pending) snapshot
|
|
3558
|
+
* **synchronously** is what lets concurrent callers coalesce onto one in-flight resolve.
|
|
2764
3559
|
*/
|
|
2765
|
-
install(key,
|
|
2766
|
-
|
|
3560
|
+
install(key, snap) {
|
|
3561
|
+
snap.installedAtMs = this.now();
|
|
2767
3562
|
this.snapshots.set(key, snap);
|
|
2768
3563
|
const forgetIfStale = () => {
|
|
2769
3564
|
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
2770
3565
|
};
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
3566
|
+
snap.onReader = (reader) => {
|
|
3567
|
+
reader.then((r) => {
|
|
3568
|
+
if (r === null) {
|
|
3569
|
+
forgetIfStale();
|
|
3570
|
+
return;
|
|
3571
|
+
}
|
|
3572
|
+
if (this.snapshots.peek(key) === snap) this.snapshots.setWeight(key, r.retainedBytes);
|
|
3573
|
+
}, forgetIfStale);
|
|
3574
|
+
};
|
|
3575
|
+
if (snap.target !== void 0) {
|
|
3576
|
+
snap.target.then(async (live) => {
|
|
3577
|
+
if (live === null) {
|
|
3578
|
+
forgetIfStale();
|
|
3579
|
+
return;
|
|
3580
|
+
}
|
|
3581
|
+
if (snap.openedReader === void 0 && this.snapshots.peek(key) === snap) {
|
|
3582
|
+
this.snapshots.setWeight(key, live.bytes);
|
|
3583
|
+
}
|
|
3584
|
+
await snap.adopt(live);
|
|
3585
|
+
}, forgetIfStale);
|
|
3586
|
+
}
|
|
3587
|
+
if (snap.openedReader !== void 0) snap.onReader(snap.openedReader);
|
|
2778
3588
|
return snap;
|
|
2779
3589
|
}
|
|
2780
3590
|
now() {
|
|
@@ -2806,12 +3616,12 @@ var CrbmStorageChunkSource = class {
|
|
|
2806
3616
|
* concurrent sweep. One retry, then propagate — same contract, same eviction rule.
|
|
2807
3617
|
*/
|
|
2808
3618
|
async currentGeneration(ref) {
|
|
2809
|
-
const
|
|
3619
|
+
const snap = this.liveSnapshot(ref);
|
|
2810
3620
|
try {
|
|
2811
|
-
return (await
|
|
3621
|
+
return (await snap.reader)?.generation ?? null;
|
|
2812
3622
|
} catch (err) {
|
|
2813
3623
|
if (!isNotFoundError(err)) throw err;
|
|
2814
|
-
this.dropStale(segmentKey(ref),
|
|
3624
|
+
this.dropStale(segmentKey(ref), snap);
|
|
2815
3625
|
return (await this.resolvedReader(ref))?.generation ?? null;
|
|
2816
3626
|
}
|
|
2817
3627
|
}
|
|
@@ -2831,6 +3641,7 @@ var CrbmStorageChunkSource = class {
|
|
|
2831
3641
|
const key = segmentKey(ref);
|
|
2832
3642
|
this.snapshots.delete(key);
|
|
2833
3643
|
const pinnedPrefix = `${key}@`;
|
|
3644
|
+
this.distrusted.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2834
3645
|
this.snapshots.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2835
3646
|
this.replacedPins.deleteWhere((k) => k.startsWith(pinnedPrefix));
|
|
2836
3647
|
for (const k of this.checking.keys()) if (k.startsWith(pinnedPrefix)) this.checking.delete(k);
|
|
@@ -2851,13 +3662,13 @@ var CrbmStorageChunkSource = class {
|
|
|
2851
3662
|
* almost always served from the cached snapshot with no backend call at all. One retry, then propagate.
|
|
2852
3663
|
*/
|
|
2853
3664
|
async currentVersion(ref) {
|
|
2854
|
-
const
|
|
3665
|
+
const snap = this.liveSnapshot(ref);
|
|
2855
3666
|
let reader;
|
|
2856
3667
|
try {
|
|
2857
|
-
reader = await
|
|
3668
|
+
reader = await snap.reader;
|
|
2858
3669
|
} catch (err) {
|
|
2859
3670
|
if (!isNotFoundError(err)) throw err;
|
|
2860
|
-
this.dropStale(segmentKey(ref),
|
|
3671
|
+
this.dropStale(segmentKey(ref), snap);
|
|
2861
3672
|
reader = await this.resolvedReader(ref);
|
|
2862
3673
|
}
|
|
2863
3674
|
return reader === null ? null : versionOf(reader.generation, reader.lineage);
|
|
@@ -2880,34 +3691,38 @@ var CrbmStorageChunkSource = class {
|
|
|
2880
3691
|
}
|
|
2881
3692
|
}
|
|
2882
3693
|
async pinOnce(ref) {
|
|
2883
|
-
const
|
|
2884
|
-
if (
|
|
3694
|
+
const live = await this.resolveLive(ref);
|
|
3695
|
+
if (live === null) return null;
|
|
3696
|
+
const target = live.target;
|
|
2885
3697
|
const version = versionOf(target.generation, target.lineage);
|
|
2886
3698
|
let reader;
|
|
2887
3699
|
let opened = true;
|
|
2888
3700
|
if (target.lineage === void 0) {
|
|
2889
|
-
reader = await this.
|
|
2890
|
-
this.install(
|
|
3701
|
+
reader = await this.openLive(ref, live);
|
|
3702
|
+
this.install(
|
|
3703
|
+
this.pinnedKey(ref, version, reader.fingerprint),
|
|
3704
|
+
Snapshot.eager(Promise.resolve(reader))
|
|
3705
|
+
);
|
|
2891
3706
|
} else {
|
|
2892
3707
|
const epoch = this.invalidations;
|
|
2893
3708
|
const key = this.pinnedKey(ref, version);
|
|
2894
3709
|
let entry = this.snapshots.get(key);
|
|
2895
3710
|
opened = entry === void 0;
|
|
2896
|
-
entry ??= this.install(key, this.
|
|
3711
|
+
entry ??= this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2897
3712
|
reader = await entry.reader;
|
|
2898
3713
|
if (reader === null) return null;
|
|
2899
3714
|
const replaced = (r) => this.replacedPins.get(this.heldKey(ref, version, r.fingerprint)) !== void 0;
|
|
2900
3715
|
const known = !opened && replaced(reader);
|
|
2901
3716
|
if (!known && this.invalidations === epoch && this.snapshots.peek(key) === void 0) {
|
|
2902
|
-
this.install(key, Promise.resolve(reader));
|
|
3717
|
+
this.install(key, Snapshot.eager(Promise.resolve(reader)));
|
|
2903
3718
|
}
|
|
2904
3719
|
if (known) {
|
|
2905
3720
|
const current = this.snapshots.peek(key);
|
|
2906
|
-
let fresh = current !== void 0 && current !== entry ? current : this.install(key, this.
|
|
3721
|
+
let fresh = current !== void 0 && current !== entry ? current : this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2907
3722
|
reader = await fresh.reader;
|
|
2908
3723
|
if (reader === null) return null;
|
|
2909
3724
|
if (fresh === current && replaced(reader)) {
|
|
2910
|
-
fresh = this.install(key, this.
|
|
3725
|
+
fresh = this.install(key, Snapshot.eager(this.openLive(ref, live)));
|
|
2911
3726
|
reader = await fresh.reader;
|
|
2912
3727
|
if (reader === null) return null;
|
|
2913
3728
|
}
|
|
@@ -2973,7 +3788,7 @@ var CrbmStorageChunkSource = class {
|
|
|
2973
3788
|
if (entry === void 0) {
|
|
2974
3789
|
if (fingerprint !== void 0 && this.replacedPins.get(this.heldKey(ref, at, fingerprint)) !== void 0)
|
|
2975
3790
|
throw notThePinned(ref, generation);
|
|
2976
|
-
entry = this.install(key, this.openAt(ref, generation));
|
|
3791
|
+
entry = this.install(key, Snapshot.eager(this.openAt(ref, generation)));
|
|
2977
3792
|
}
|
|
2978
3793
|
let reader;
|
|
2979
3794
|
try {
|
|
@@ -3108,58 +3923,94 @@ var CrbmStorageChunkSource = class {
|
|
|
3108
3923
|
return false;
|
|
3109
3924
|
}
|
|
3110
3925
|
/**
|
|
3111
|
-
* Evict a snapshot we just failed to read from — matched by
|
|
3926
|
+
* Evict a snapshot we just failed to read from — matched by the snapshot itself, so a concurrent call's
|
|
3112
3927
|
* fresher snapshot is never clobbered. The identity guard is the whole point: `install` replaces the entry
|
|
3113
3928
|
* wholesale, so comparing anything else would drop a good snapshot on the floor.
|
|
3114
3929
|
*/
|
|
3115
|
-
dropStale(key,
|
|
3116
|
-
|
|
3117
|
-
if (cur !== void 0 && cur.reader === pending) this.snapshots.delete(key);
|
|
3118
|
-
}
|
|
3119
|
-
async resolveLatest(ref) {
|
|
3120
|
-
const target = await this.resolveTarget(ref);
|
|
3121
|
-
if (target === null) return null;
|
|
3122
|
-
return this.openForTarget(ref, target);
|
|
3930
|
+
dropStale(key, snap) {
|
|
3931
|
+
if (this.snapshots.get(key) === snap) this.snapshots.delete(key);
|
|
3123
3932
|
}
|
|
3124
3933
|
/** Open a {@link CrbmReader} for an already-resolved generation target (decrypting if the segment is encrypted). */
|
|
3125
3934
|
async openForTarget(ref, target) {
|
|
3935
|
+
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
3936
|
+
return this.openGeneration(ref, target, crypto);
|
|
3937
|
+
}
|
|
3938
|
+
openGeneration(ref, target, crypto) {
|
|
3126
3939
|
const genKey = {
|
|
3127
3940
|
namespace: ref.namespace,
|
|
3128
3941
|
segment: ref.segment,
|
|
3129
3942
|
generation: target.generation
|
|
3130
3943
|
};
|
|
3131
|
-
const crypto = await this.cryptoForRead(ref, target.generation, target.wrappedDeks);
|
|
3132
3944
|
return openChecked(this.driver, genKey, {
|
|
3133
3945
|
...this.readerOptions,
|
|
3134
3946
|
crypto,
|
|
3135
3947
|
lineage: target.lineage
|
|
3136
3948
|
});
|
|
3137
3949
|
}
|
|
3950
|
+
/**
|
|
3951
|
+
* Open the generation a resolution found, and hold the row's summary against the object while it is open anyway:
|
|
3952
|
+
* the same count and metadata, or the summary is not believed again in this process ({@link distrusted}). This
|
|
3953
|
+
* costs no request; a mismatch is not an error, since the object is the truth and this read has it.
|
|
3954
|
+
*/
|
|
3955
|
+
async openLive(ref, live) {
|
|
3956
|
+
const reader = await this.openGeneration(ref, live.target, await live.crypto());
|
|
3957
|
+
const summary = await live.summary();
|
|
3958
|
+
if (summary !== void 0 && !summaryAgrees(summary, describe(reader))) {
|
|
3959
|
+
this.distrusted.set(this.distrustKey(ref, live.target.generation, live.target.lineage), true);
|
|
3960
|
+
}
|
|
3961
|
+
return reader;
|
|
3962
|
+
}
|
|
3963
|
+
distrustKey(ref, generation, lineage) {
|
|
3964
|
+
return `${segmentKey(ref)}@${generation}:${String(lineage)}`;
|
|
3965
|
+
}
|
|
3966
|
+
/** What the row says of the generation, unless it is a summary this process has found to disagree with its object. */
|
|
3967
|
+
async trustedSummary(ref, live) {
|
|
3968
|
+
if (this.distrusted.peek(this.distrustKey(ref, live.target.generation, live.target.lineage)) !== void 0) {
|
|
3969
|
+
return void 0;
|
|
3970
|
+
}
|
|
3971
|
+
return live.summary();
|
|
3972
|
+
}
|
|
3138
3973
|
/**
|
|
3139
3974
|
* The current generation + its DEK wrappings: the registry's authoritative record (one strong read), or a
|
|
3140
3975
|
* `list` scan for the max generation when there's no registry (a registry-less source can only read cleartext
|
|
3141
|
-
* — there's nowhere a wrapped DEK could live).
|
|
3976
|
+
* — there's nowhere a wrapped DEK could live). The row's summary rides along, to be used or not by whoever
|
|
3977
|
+
* asks.
|
|
3142
3978
|
*/
|
|
3143
|
-
async
|
|
3979
|
+
async resolveLive(ref, prior) {
|
|
3144
3980
|
if (this.registry !== void 0) {
|
|
3145
3981
|
const record = await this.registry.get(ref);
|
|
3146
3982
|
if (record === null) return null;
|
|
3147
3983
|
if (record.status === "destroyed") return null;
|
|
3148
3984
|
if (record.currentGen === null) return null;
|
|
3985
|
+
const generation = record.currentGen;
|
|
3986
|
+
const keys = record.wrappedDeks;
|
|
3987
|
+
const unwrap = keys === void 0 || keys.length === 0 || this.keystore === void 0 ? void 0 : prior?.unwrap !== void 0 && sameKeys(prior.target.wrappedDeks, keys) ? prior.unwrap : once(() => this.keystore.openDek(keys));
|
|
3988
|
+
const crypto = once(() => this.cryptoForRead(ref, generation, keys, unwrap));
|
|
3149
3989
|
return {
|
|
3150
|
-
generation: record.
|
|
3151
|
-
|
|
3152
|
-
|
|
3990
|
+
target: { generation, lineage: record.token, wrappedDeks: record.wrappedDeks },
|
|
3991
|
+
crypto,
|
|
3992
|
+
unwrap,
|
|
3993
|
+
// The same checks as a read of the object makes (`requireEncryption`, a keystore for an encrypted row)
|
|
3994
|
+
// come first, then the row's summary, used only if it names this generation in the shape the keys call for.
|
|
3995
|
+
summary: once(async () => usableSummary(ref, record, (await crypto())?.aead)),
|
|
3996
|
+
bytes: SNAPSHOT_BASE_BYTES + (record.summary === void 0 ? 0 : JSON.stringify(record.summary).length)
|
|
3153
3997
|
};
|
|
3154
3998
|
}
|
|
3155
3999
|
let maxGen = -1;
|
|
3156
4000
|
for await (const key of this.driver.list(ref)) {
|
|
3157
4001
|
if (key.generation > maxGen) maxGen = key.generation;
|
|
3158
4002
|
}
|
|
3159
|
-
|
|
4003
|
+
if (maxGen < 0) return null;
|
|
4004
|
+
return {
|
|
4005
|
+
target: { generation: maxGen },
|
|
4006
|
+
crypto: () => Promise.resolve(void 0),
|
|
4007
|
+
unwrap: void 0,
|
|
4008
|
+
summary: () => Promise.resolve(void 0),
|
|
4009
|
+
bytes: SNAPSHOT_BASE_BYTES
|
|
4010
|
+
};
|
|
3160
4011
|
}
|
|
3161
4012
|
/** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
|
|
3162
|
-
async cryptoForRead(ref, generation, wrappedDeks) {
|
|
4013
|
+
async cryptoForRead(ref, generation, wrappedDeks, unwrap) {
|
|
3163
4014
|
if (wrappedDeks === void 0 || wrappedDeks.length === 0) {
|
|
3164
4015
|
if (this.requireEncryption) {
|
|
3165
4016
|
throw new KeyUnavailableError(
|
|
@@ -3168,12 +4019,13 @@ var CrbmStorageChunkSource = class {
|
|
|
3168
4019
|
}
|
|
3169
4020
|
return void 0;
|
|
3170
4021
|
}
|
|
3171
|
-
|
|
4022
|
+
const keystore = this.keystore;
|
|
4023
|
+
if (keystore === void 0) {
|
|
3172
4024
|
throw new KeyUnavailableError(
|
|
3173
4025
|
`segment "${ref.segment}" is encrypted but this CrbmStorageChunkSource has no keystore`
|
|
3174
4026
|
);
|
|
3175
4027
|
}
|
|
3176
|
-
const aead = await
|
|
4028
|
+
const aead = await (unwrap ?? (() => keystore.openDek(wrappedDeks)))();
|
|
3177
4029
|
return { aead, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
3178
4030
|
}
|
|
3179
4031
|
async getChunk(ref) {
|
|
@@ -3200,6 +4052,46 @@ var CrbmStorageChunkSource = class {
|
|
|
3200
4052
|
null
|
|
3201
4053
|
);
|
|
3202
4054
|
}
|
|
4055
|
+
/**
|
|
4056
|
+
* The current generation's number, id count and metadata. The answer comes from the row the snapshot resolved when
|
|
4057
|
+
* it can: the row's summary of the generation it names, on an active row, in the shape the row's keys call for
|
|
4058
|
+
* (a sealed one is opened with the segment's key), and one this process has not found to disagree with its
|
|
4059
|
+
* object. That is one registry read when the snapshot is cold, none when it is warm, and no read of the object.
|
|
4060
|
+
* With no summary it can use (an older row, one for another generation, one that does not open), it opens the
|
|
4061
|
+
* generation and answers from the object. A row that names an object that is gone still answers, while a read
|
|
4062
|
+
* of the object throws: the number is true of the generation the row names.
|
|
4063
|
+
*
|
|
4064
|
+
* What the row says is not confirmed here: a summary edited to agree with itself by whoever can write the row is
|
|
4065
|
+
* believed until the next time the object is opened, which holds it against the summary.
|
|
4066
|
+
*/
|
|
4067
|
+
async summary(ref) {
|
|
4068
|
+
validateUserRef(ref);
|
|
4069
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
4070
|
+
const snap = this.liveSnapshot(ref);
|
|
4071
|
+
try {
|
|
4072
|
+
const live = await snap.target;
|
|
4073
|
+
if (live === null) return null;
|
|
4074
|
+
const row = await this.trustedSummary(ref, live);
|
|
4075
|
+
if (row !== void 0) return summaryOfGeneration(live.target.generation, row);
|
|
4076
|
+
const reader = await snap.reader;
|
|
4077
|
+
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
4078
|
+
} catch (err) {
|
|
4079
|
+
if (attempt === 1) throw err;
|
|
4080
|
+
const checks = isNotFoundError(err) || isIntegrityError(err) || isValidationError(err);
|
|
4081
|
+
if (!checks || !isNotFoundError(err) && !await this.replacedUnder(ref, snap.reader, err)) {
|
|
4082
|
+
throw err;
|
|
4083
|
+
}
|
|
4084
|
+
this.dropStale(segmentKey(ref), snap);
|
|
4085
|
+
}
|
|
4086
|
+
}
|
|
4087
|
+
return null;
|
|
4088
|
+
}
|
|
4089
|
+
/** {@link summary} of a specific generation, from the object a pin holds. `held` as for {@link getChunkAt}. */
|
|
4090
|
+
async summaryAt(ref, generation, held) {
|
|
4091
|
+
validateUserRef(ref);
|
|
4092
|
+
const reader = await this.readerAt(ref, generation, held?.version, held?.fingerprint);
|
|
4093
|
+
return reader === null ? null : summaryOfGeneration(reader.generation, describe(reader));
|
|
4094
|
+
}
|
|
3203
4095
|
/**
|
|
3204
4096
|
* Run `read` against the pinned snapshot, healing the torn-read window generation GC can open: if the
|
|
3205
4097
|
* generation we resolved was superseded *and* swept (the grace window elapsed), the Storage driver throws
|
|
@@ -3228,40 +4120,143 @@ var CrbmStorageChunkSource = class {
|
|
|
3228
4120
|
*/
|
|
3229
4121
|
async withFreshSnapshot(ref, read, ifGone) {
|
|
3230
4122
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
3231
|
-
const
|
|
4123
|
+
const snap = this.liveSnapshot(ref);
|
|
4124
|
+
const pending = snap.reader;
|
|
3232
4125
|
try {
|
|
3233
4126
|
const reader = await pending;
|
|
3234
4127
|
if (reader === null) return ifGone;
|
|
3235
4128
|
return await read(reader);
|
|
3236
4129
|
} catch (err) {
|
|
3237
|
-
if (
|
|
3238
|
-
this.
|
|
4130
|
+
if (attempt === 1) throw err;
|
|
4131
|
+
if (!isNotFoundError(err) && !await this.replacedUnder(ref, pending, err)) throw err;
|
|
4132
|
+
this.dropStale(segmentKey(ref), snap);
|
|
3239
4133
|
}
|
|
3240
4134
|
}
|
|
3241
4135
|
return ifGone;
|
|
3242
4136
|
}
|
|
4137
|
+
/**
|
|
4138
|
+
* Whether `err`, raised by a read through the snapshot's reader, came from an object that is no longer the one the
|
|
4139
|
+
* reader opened. A generation number can be taken again once its object is deleted (an erasure removes the
|
|
4140
|
+
* generations above a rolled-back pointer that held the id, a collection pass can stop part-way, and a load then
|
|
4141
|
+
* numbers `currentGen + 1` again), so a reader that still holds the old object's index reads the new object's
|
|
4142
|
+
* bytes at the old offsets: a chunk checksum fails, or a range runs past the end. The object's footer says which it
|
|
4143
|
+
* was, as it does for a pin ({@link CrbmReader.sameObject}): replaced, or gone, re-resolves; the same object means
|
|
4144
|
+
* the error is real, and it stands. Only these two errors pay for the footer read, never a read that succeeded. A
|
|
4145
|
+
* transient fault in the check is rethrown; a footer that cannot be read says nothing, and the error stands.
|
|
4146
|
+
*/
|
|
4147
|
+
async replacedUnder(ref, pending, err) {
|
|
4148
|
+
if (!(isIntegrityError(err) || isValidationError(err))) return false;
|
|
4149
|
+
let reader;
|
|
4150
|
+
try {
|
|
4151
|
+
reader = await pending;
|
|
4152
|
+
} catch {
|
|
4153
|
+
return false;
|
|
4154
|
+
}
|
|
4155
|
+
if (reader === null) return false;
|
|
4156
|
+
const at = {
|
|
4157
|
+
namespace: ref.namespace,
|
|
4158
|
+
segment: ref.segment,
|
|
4159
|
+
generation: reader.generation
|
|
4160
|
+
};
|
|
4161
|
+
try {
|
|
4162
|
+
return !await CrbmReader.sameObject(storageBlobReader(this.driver, at), reader.fingerprint);
|
|
4163
|
+
} catch (checkErr) {
|
|
4164
|
+
if (isNotFoundError(checkErr)) return true;
|
|
4165
|
+
if (isTransientError(checkErr)) throw checkErr;
|
|
4166
|
+
return false;
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
3243
4169
|
};
|
|
3244
|
-
function
|
|
3245
|
-
const
|
|
4170
|
+
function* encodeEach(chunks) {
|
|
4171
|
+
for (const { chunkKey, bitmap } of chunks) {
|
|
4172
|
+
if (bitmap.isEmpty) continue;
|
|
4173
|
+
bitmap.optimize?.();
|
|
4174
|
+
yield { chunkKey, payload: bitmap.serialize(), cardinality: bitmap.size };
|
|
4175
|
+
}
|
|
4176
|
+
}
|
|
4177
|
+
async function writeEncodedChunks(driver, key, chunks, options) {
|
|
3246
4178
|
const tick = yieldEvery(options.clock);
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
4179
|
+
let chunkCount = 0;
|
|
4180
|
+
let cardinality = 0;
|
|
4181
|
+
let identity;
|
|
4182
|
+
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
4183
|
+
const writer = new CrbmWriter(sink, {
|
|
4184
|
+
generation: key.generation,
|
|
4185
|
+
crypto: options.crypto,
|
|
4186
|
+
metadata: options.metadata
|
|
4187
|
+
});
|
|
4188
|
+
for (const chunk of chunks) {
|
|
4189
|
+
await writer.addChunk(chunk.chunkKey, chunk.payload, chunk.cardinality);
|
|
4190
|
+
chunkCount++;
|
|
4191
|
+
cardinality += chunk.cardinality;
|
|
3253
4192
|
const pause = tick();
|
|
3254
4193
|
if (pause !== null) await pause;
|
|
3255
4194
|
}
|
|
3256
4195
|
await writer.finish();
|
|
4196
|
+
identity = writer.identity;
|
|
3257
4197
|
});
|
|
4198
|
+
if (identity === void 0) {
|
|
4199
|
+
throw new IntegrityError(
|
|
4200
|
+
`the driver committed ${key.segment}.${key.generation} without writing it`
|
|
4201
|
+
);
|
|
4202
|
+
}
|
|
4203
|
+
return {
|
|
4204
|
+
size,
|
|
4205
|
+
sha256,
|
|
4206
|
+
chunkCount,
|
|
4207
|
+
cardinality,
|
|
4208
|
+
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
4209
|
+
};
|
|
4210
|
+
}
|
|
4211
|
+
async function holdsObject(storage, key, fingerprint) {
|
|
4212
|
+
try {
|
|
4213
|
+
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
4214
|
+
} catch {
|
|
4215
|
+
return false;
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
function objectIsEncrypted(storage, key) {
|
|
4219
|
+
return footerSaysEncrypted(storageBlobReader(storage, key));
|
|
4220
|
+
}
|
|
4221
|
+
function rememberingTail(blob) {
|
|
4222
|
+
let first;
|
|
4223
|
+
return {
|
|
4224
|
+
getRange: (offset, length) => blob.getRange(offset, length),
|
|
4225
|
+
async getTail(maxBytes) {
|
|
4226
|
+
if (first !== void 0 && (maxBytes <= first.bytes.length || first.bytes.length === first.size)) {
|
|
4227
|
+
const take = Math.min(maxBytes, first.bytes.length);
|
|
4228
|
+
return { bytes: first.bytes.subarray(first.bytes.length - take), size: first.size };
|
|
4229
|
+
}
|
|
4230
|
+
const read = await blob.getTail(maxBytes);
|
|
4231
|
+
first ??= read;
|
|
4232
|
+
return read;
|
|
4233
|
+
}
|
|
4234
|
+
};
|
|
4235
|
+
}
|
|
4236
|
+
async function openRollbackTarget(storage, key, crypto) {
|
|
4237
|
+
const blob = rememberingTail(storageBlobReader(storage, key));
|
|
4238
|
+
await blob.getTail(DEFAULT_TAIL_BYTES);
|
|
4239
|
+
const encrypted = await footerSaysEncrypted(blob);
|
|
4240
|
+
if (encrypted !== (crypto !== void 0)) return { encrypted, reader: void 0 };
|
|
4241
|
+
const reader = await CrbmReader.open(blob, { crypto });
|
|
4242
|
+
if (reader.generation !== key.generation) {
|
|
4243
|
+
throw new IntegrityError(
|
|
4244
|
+
`segment "${key.segment}" generation ${key.generation}: its footer says generation ${reader.generation}`
|
|
4245
|
+
);
|
|
4246
|
+
}
|
|
4247
|
+
return { encrypted, reader };
|
|
3258
4248
|
}
|
|
3259
4249
|
async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
3260
4250
|
const chunkKeys = [];
|
|
3261
4251
|
let cardinality = 0;
|
|
3262
4252
|
const tick = yieldEvery(options.clock);
|
|
4253
|
+
let identity;
|
|
3263
4254
|
const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
|
|
3264
|
-
const writer = new CrbmWriter(sink, {
|
|
4255
|
+
const writer = new CrbmWriter(sink, {
|
|
4256
|
+
generation: key.generation,
|
|
4257
|
+
crypto: options.crypto,
|
|
4258
|
+
metadata: options.metadata
|
|
4259
|
+
});
|
|
3265
4260
|
for await (const { chunkKey, bitmap } of chunks) {
|
|
3266
4261
|
if (bitmap.isEmpty) continue;
|
|
3267
4262
|
bitmap.optimize?.();
|
|
@@ -3272,12 +4267,57 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
|
|
|
3272
4267
|
if (pause !== null) await pause;
|
|
3273
4268
|
}
|
|
3274
4269
|
await writer.finish();
|
|
4270
|
+
identity = writer.identity;
|
|
3275
4271
|
});
|
|
3276
|
-
|
|
4272
|
+
if (identity === void 0) {
|
|
4273
|
+
throw new IntegrityError(
|
|
4274
|
+
`the driver committed ${key.segment}.${key.generation} without writing it`
|
|
4275
|
+
);
|
|
4276
|
+
}
|
|
4277
|
+
return {
|
|
4278
|
+
size,
|
|
4279
|
+
sha256,
|
|
4280
|
+
chunkKeys,
|
|
4281
|
+
cardinality,
|
|
4282
|
+
fingerprint: fingerprintFor(identity.size, identity.footerCrc)
|
|
4283
|
+
};
|
|
4284
|
+
}
|
|
4285
|
+
function refuseCleartextOntoKey(key, record, cleartext) {
|
|
4286
|
+
if (cleartext !== true || record.wrappedDeks === void 0 || record.wrappedDeks.length === 0) {
|
|
4287
|
+
return;
|
|
4288
|
+
}
|
|
4289
|
+
throw new KeyUnavailableError(
|
|
4290
|
+
`segment "${key.segment}" is encrypted, and generation ${key.generation} was written in cleartext: another writer created the segment with a key while this one was writing. Nothing was published. Re-run the write with the keystore that holds the segment's key.`
|
|
4291
|
+
);
|
|
3277
4292
|
}
|
|
4293
|
+
var UNANSWERED_RESENDS = 3;
|
|
4294
|
+
var UNANSWERED_RESEND_BASE_MS = 500;
|
|
3278
4295
|
async function publishGeneration(registry, key, options = {}) {
|
|
4296
|
+
let fresh;
|
|
4297
|
+
let failedOn;
|
|
4298
|
+
let unanswered;
|
|
4299
|
+
let resends = 0;
|
|
4300
|
+
let sawUnanswered = false;
|
|
4301
|
+
let lastRecord;
|
|
3279
4302
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
3280
|
-
const
|
|
4303
|
+
const reused = attempt === 0 && options.row !== void 0;
|
|
4304
|
+
const record = fresh !== void 0 ? fresh : reused ? options.row ?? null : await registry.get(key);
|
|
4305
|
+
fresh = void 0;
|
|
4306
|
+
unanswered = void 0;
|
|
4307
|
+
lastRecord = record;
|
|
4308
|
+
if (failedOn !== void 0) {
|
|
4309
|
+
let landed;
|
|
4310
|
+
try {
|
|
4311
|
+
landed = await landedHere(record, failedOn, key, options.holdsOwnObject);
|
|
4312
|
+
} catch (proofErr) {
|
|
4313
|
+
throw outcomeUnknown(key, proofErr);
|
|
4314
|
+
}
|
|
4315
|
+
if (landed) return true;
|
|
4316
|
+
if (record !== null && record.status === "active" && record.currentGen === key.generation) {
|
|
4317
|
+
return false;
|
|
4318
|
+
}
|
|
4319
|
+
failedOn = void 0;
|
|
4320
|
+
}
|
|
3281
4321
|
try {
|
|
3282
4322
|
if (options.expectFrom !== void 0 && record?.currentGen !== options.expectFrom) {
|
|
3283
4323
|
return false;
|
|
@@ -3288,103 +4328,138 @@ async function publishGeneration(registry, key, options = {}) {
|
|
|
3288
4328
|
if (options.expectToken !== void 0 && record?.token !== options.expectToken) {
|
|
3289
4329
|
return false;
|
|
3290
4330
|
}
|
|
4331
|
+
if (record === null && options.summary !== void 0 && "sealed" in options.summary && options.wrappedDeks === void 0) {
|
|
4332
|
+
if (reused) continue;
|
|
4333
|
+
return false;
|
|
4334
|
+
}
|
|
3291
4335
|
if (record === null) {
|
|
3292
4336
|
await registry.create(key, {
|
|
3293
4337
|
currentGen: key.generation,
|
|
3294
|
-
wrappedDeks: options.wrappedDeks
|
|
4338
|
+
wrappedDeks: options.wrappedDeks,
|
|
4339
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
3295
4340
|
});
|
|
3296
4341
|
} else if (record.status === "destroyed") {
|
|
3297
4342
|
throw new ValidationError(
|
|
3298
4343
|
`segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
|
|
3299
4344
|
);
|
|
3300
4345
|
} else if (record.currentGen === null) {
|
|
4346
|
+
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
3301
4347
|
await registry.compareAndSwap(key, record.token, {
|
|
3302
4348
|
currentGen: key.generation,
|
|
3303
|
-
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks }
|
|
4349
|
+
...options.wrappedDeks === void 0 ? {} : { wrappedDeks: options.wrappedDeks },
|
|
4350
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
3304
4351
|
});
|
|
3305
4352
|
} else if (record.currentGen > key.generation) {
|
|
3306
4353
|
return false;
|
|
3307
4354
|
} else if (record.currentGen === key.generation) {
|
|
4355
|
+
if (reused) continue;
|
|
3308
4356
|
return true;
|
|
3309
4357
|
} else {
|
|
3310
4358
|
if (options.wrappedDeks !== void 0 && options.wrappedDeks.length > 0) {
|
|
3311
4359
|
throw new ValidationError(
|
|
3312
|
-
`publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen}
|
|
4360
|
+
`publishGeneration: refusing to publish generation ${key.generation} of "${key.segment}" with new key material onto a segment that already has generation ${record.currentGen}: its generations are not encrypted under this key, and nothing was published. If another writer published to the segment while this one was writing, re-run the write, which then uses the segment's own key; if the segment's generations are cleartext, encrypt a new segment and load into that instead.`
|
|
3313
4361
|
);
|
|
3314
4362
|
}
|
|
3315
|
-
|
|
4363
|
+
refuseCleartextOntoKey(key, record, options.cleartext);
|
|
4364
|
+
await registry.compareAndSwap(key, record.token, {
|
|
4365
|
+
currentGen: key.generation,
|
|
4366
|
+
...options.summary === void 0 ? {} : { summary: options.summary }
|
|
4367
|
+
});
|
|
3316
4368
|
}
|
|
3317
4369
|
return true;
|
|
3318
4370
|
} catch (err) {
|
|
3319
|
-
if (isWriteConflictError(err))
|
|
3320
|
-
|
|
4371
|
+
if (isWriteConflictError(err)) {
|
|
4372
|
+
failedOn = record;
|
|
4373
|
+
continue;
|
|
4374
|
+
}
|
|
4375
|
+
if (!isTransientError(err)) throw err;
|
|
4376
|
+
options.onUnanswered?.();
|
|
4377
|
+
sawUnanswered = true;
|
|
4378
|
+
let now;
|
|
4379
|
+
try {
|
|
4380
|
+
now = await registry.get(key);
|
|
4381
|
+
} catch (readErr) {
|
|
4382
|
+
if (isIntegrityError(readErr)) throw readErr;
|
|
4383
|
+
throw outcomeUnknown(key, err);
|
|
4384
|
+
}
|
|
4385
|
+
if (sameRow(now, record)) {
|
|
4386
|
+
const sleep = options.clock?.sleep;
|
|
4387
|
+
if (sleep === void 0 || resends >= UNANSWERED_RESENDS) throw outcomeUnknown(key, err);
|
|
4388
|
+
const bound = UNANSWERED_RESEND_BASE_MS * 2 ** resends;
|
|
4389
|
+
resends += 1;
|
|
4390
|
+
await sleep.call(options.clock, Math.floor((options.rng?.next() ?? 1) * bound));
|
|
4391
|
+
unanswered = outcomeUnknown(key, err);
|
|
4392
|
+
fresh = now;
|
|
4393
|
+
continue;
|
|
4394
|
+
}
|
|
4395
|
+
failedOn = record;
|
|
4396
|
+
fresh = now;
|
|
3321
4397
|
}
|
|
3322
4398
|
}
|
|
3323
4399
|
const final = await registry.get(key);
|
|
3324
|
-
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation)
|
|
3325
|
-
|
|
4400
|
+
if (final !== null && final.currentGen !== null && final.currentGen >= key.generation) {
|
|
4401
|
+
if (final.currentGen !== key.generation) return false;
|
|
4402
|
+
if (!sawUnanswered) return true;
|
|
4403
|
+
try {
|
|
4404
|
+
return await landedHere(final, lastRecord ?? null, key, options.holdsOwnObject);
|
|
4405
|
+
} catch (proofErr) {
|
|
4406
|
+
throw outcomeUnknown(key, proofErr);
|
|
4407
|
+
}
|
|
4408
|
+
}
|
|
4409
|
+
if (unanswered !== void 0) throw unanswered;
|
|
3326
4410
|
throw new WriteConflictError(
|
|
3327
4411
|
`publishGeneration: contention setting currentGen for ${key.segment}`
|
|
3328
4412
|
);
|
|
3329
4413
|
}
|
|
4414
|
+
async function landedHere(now, before, key, holdsOwnObject) {
|
|
4415
|
+
if (now === null || now.status === "destroyed" || now.currentGen !== key.generation) return false;
|
|
4416
|
+
if (before !== null && !sameIncarnation(before, now)) return false;
|
|
4417
|
+
if (holdsOwnObject === void 0) {
|
|
4418
|
+
throw new TransientError(
|
|
4419
|
+
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled, since the caller gave no way to tell its own object from another's. Its object is kept; re-run the write, which numbers past it.`
|
|
4420
|
+
);
|
|
4421
|
+
}
|
|
4422
|
+
return holdsOwnObject();
|
|
4423
|
+
}
|
|
4424
|
+
function sameRow(a, b) {
|
|
4425
|
+
return a === null ? b === null : b !== null && a.token === b.token;
|
|
4426
|
+
}
|
|
4427
|
+
function outcomeUnknown(key, cause) {
|
|
4428
|
+
if (isTransientError(cause)) return cause;
|
|
4429
|
+
return new TransientError(
|
|
4430
|
+
`publish of "${key.segment}" generation ${key.generation}: whether its registry write landed could not be settled (${cause?.name ?? "error"} while checking its object). Its object is kept, since the write may still land; re-run the write, which numbers past it.`,
|
|
4431
|
+
{ cause }
|
|
4432
|
+
);
|
|
4433
|
+
}
|
|
4434
|
+
async function provesOwnObject(storage, key, fingerprint) {
|
|
4435
|
+
try {
|
|
4436
|
+
return await CrbmReader.sameObject(storageBlobReader(storage, key), fingerprint);
|
|
4437
|
+
} catch (err) {
|
|
4438
|
+
if (isNotFoundError(err)) return false;
|
|
4439
|
+
throw err;
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
3330
4442
|
async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
3331
4443
|
if (options.keystore === void 0 && options.requireEncryption === true) {
|
|
3332
4444
|
throw new ValidationError("requireEncryption: a load needs a keystore to write encrypted");
|
|
3333
4445
|
}
|
|
3334
4446
|
const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
for (const [chunkKey, rems] of pendingByChunk) {
|
|
3342
|
-
if (rems.length === 0) continue;
|
|
3343
|
-
const existingBitmap = byChunk.get(chunkKey);
|
|
3344
|
-
if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
|
|
3345
|
-
else existingBitmap.addMany(rems);
|
|
3346
|
-
rems.length = 0;
|
|
3347
|
-
const pause = tickChunk();
|
|
3348
|
-
if (pause !== null) await pause;
|
|
3349
|
-
}
|
|
3350
|
-
pending = 0;
|
|
3351
|
-
};
|
|
3352
|
-
const ingest = (id) => {
|
|
3353
|
-
const { chunkKey, remainder } = splitId(id);
|
|
3354
|
-
let bucket = pendingByChunk.get(chunkKey);
|
|
3355
|
-
if (bucket === void 0) {
|
|
3356
|
-
bucket = [];
|
|
3357
|
-
pendingByChunk.set(chunkKey, bucket);
|
|
3358
|
-
}
|
|
3359
|
-
bucket.push(remainder);
|
|
3360
|
-
return ++pending >= BULK_FLUSH_IDS;
|
|
3361
|
-
};
|
|
3362
|
-
if (Symbol.asyncIterator in ids) {
|
|
3363
|
-
for await (const id of ids) {
|
|
3364
|
-
if (ingest(id)) await flushPending();
|
|
3365
|
-
const pause = tickId();
|
|
3366
|
-
if (pause !== null) await pause;
|
|
3367
|
-
}
|
|
4447
|
+
let chunks;
|
|
4448
|
+
if (ids instanceof DecodedLoadInput && ids.bitmap.encodeChunks !== void 0) {
|
|
4449
|
+
const pause = yieldEvery(options.clock, 1);
|
|
4450
|
+
await pause();
|
|
4451
|
+
chunks = ids.bitmap.encodeChunks();
|
|
4452
|
+
await pause();
|
|
3368
4453
|
} else {
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
if (pause !== null) await pause;
|
|
3373
|
-
}
|
|
3374
|
-
}
|
|
3375
|
-
await flushPending();
|
|
3376
|
-
let cardinality = 0;
|
|
3377
|
-
const chunks = [];
|
|
3378
|
-
for (const [chunkKey, bitmap] of byChunk) {
|
|
3379
|
-
cardinality += bitmap.size;
|
|
3380
|
-
chunks.push({ chunkKey, bitmap });
|
|
3381
|
-
const pause = tickChunk();
|
|
3382
|
-
if (pause !== null) await pause;
|
|
4454
|
+
chunks = encodeEach(
|
|
4455
|
+
await bucketIds(ids instanceof DecodedLoadInput ? ids.bitmap : ids, codec, options.clock)
|
|
4456
|
+
);
|
|
3383
4457
|
}
|
|
3384
4458
|
if (options.keystore !== void 0 && options.registry === void 0) {
|
|
3385
4459
|
throw new ValidationError("an encrypted load requires a registry to store the wrapped DEK");
|
|
3386
4460
|
}
|
|
3387
|
-
const
|
|
4461
|
+
const passed = options.row;
|
|
4462
|
+
const existing = passed !== void 0 && passed !== null && (passed.wrappedDeks === void 0 || passed.wrappedDeks.length === 0) ? passed : options.registry !== void 0 ? await options.registry.get(key) : null;
|
|
3388
4463
|
if (existing?.status === "destroyed") {
|
|
3389
4464
|
throw new ValidationError(
|
|
3390
4465
|
`segment "${key.segment}" is destroyed (crypto-shredded) \u2014 refusing to write; use a new segment`
|
|
@@ -3413,13 +4488,25 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
3413
4488
|
crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
|
|
3414
4489
|
}
|
|
3415
4490
|
}
|
|
3416
|
-
const { size, sha256 } = await
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
4491
|
+
const { size, sha256, chunkCount, cardinality, fingerprint } = await writeEncodedChunks(
|
|
4492
|
+
driver,
|
|
4493
|
+
key,
|
|
4494
|
+
chunks,
|
|
4495
|
+
{ crypto, clock: options.clock, metadata: options.metadata }
|
|
4496
|
+
);
|
|
4497
|
+
const summary = summaryOf(
|
|
4498
|
+
key,
|
|
4499
|
+
key.generation,
|
|
4500
|
+
{ cardinality, metadata: options.metadata },
|
|
4501
|
+
crypto?.aead
|
|
4502
|
+
);
|
|
3420
4503
|
if (options.registry !== void 0 && options.publish !== false) {
|
|
3421
4504
|
const becameCurrent = await publishGeneration(options.registry, key, {
|
|
3422
|
-
wrappedDeks: newWrapped
|
|
4505
|
+
wrappedDeks: newWrapped,
|
|
4506
|
+
summary,
|
|
4507
|
+
cleartext: crypto === void 0,
|
|
4508
|
+
holdsOwnObject: () => provesOwnObject(driver, key, fingerprint),
|
|
4509
|
+
clock: options.clock
|
|
3423
4510
|
});
|
|
3424
4511
|
if (becameCurrent) {
|
|
3425
4512
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -3429,16 +4516,72 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
|
|
|
3429
4516
|
generation: key.generation
|
|
3430
4517
|
});
|
|
3431
4518
|
}
|
|
3432
|
-
return {
|
|
3433
|
-
size,
|
|
3434
|
-
sha256,
|
|
3435
|
-
chunkCount
|
|
3436
|
-
cardinality,
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
4519
|
+
return {
|
|
4520
|
+
size,
|
|
4521
|
+
sha256,
|
|
4522
|
+
chunkCount,
|
|
4523
|
+
cardinality,
|
|
4524
|
+
summary,
|
|
4525
|
+
becameCurrent,
|
|
4526
|
+
wrappedDeks: newWrapped,
|
|
4527
|
+
encrypted: crypto !== void 0,
|
|
4528
|
+
fingerprint
|
|
4529
|
+
};
|
|
4530
|
+
}
|
|
4531
|
+
return {
|
|
4532
|
+
size,
|
|
4533
|
+
sha256,
|
|
4534
|
+
chunkCount,
|
|
4535
|
+
cardinality,
|
|
4536
|
+
summary,
|
|
4537
|
+
wrappedDeks: newWrapped,
|
|
4538
|
+
encrypted: crypto !== void 0,
|
|
4539
|
+
fingerprint
|
|
4540
|
+
};
|
|
4541
|
+
}
|
|
4542
|
+
async function bucketIds(ids, codec, clock) {
|
|
4543
|
+
const byChunk = /* @__PURE__ */ new Map();
|
|
4544
|
+
const pendingByChunk = /* @__PURE__ */ new Map();
|
|
4545
|
+
let pending = 0;
|
|
4546
|
+
const tickChunk = yieldEvery(clock);
|
|
4547
|
+
const tickId = yieldEvery(clock, YIELD_EVERY_IDS);
|
|
4548
|
+
const flushPending = async () => {
|
|
4549
|
+
for (const [chunkKey, rems] of pendingByChunk) {
|
|
4550
|
+
if (rems.length === 0) continue;
|
|
4551
|
+
const existingBitmap = byChunk.get(chunkKey);
|
|
4552
|
+
if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
|
|
4553
|
+
else existingBitmap.addMany(rems);
|
|
4554
|
+
rems.length = 0;
|
|
4555
|
+
const pause = tickChunk();
|
|
4556
|
+
if (pause !== null) await pause;
|
|
4557
|
+
}
|
|
4558
|
+
pending = 0;
|
|
4559
|
+
};
|
|
4560
|
+
const ingest = (id) => {
|
|
4561
|
+
const { chunkKey, remainder } = splitId(id);
|
|
4562
|
+
let bucket = pendingByChunk.get(chunkKey);
|
|
4563
|
+
if (bucket === void 0) {
|
|
4564
|
+
bucket = [];
|
|
4565
|
+
pendingByChunk.set(chunkKey, bucket);
|
|
4566
|
+
}
|
|
4567
|
+
bucket.push(remainder);
|
|
4568
|
+
return ++pending >= BULK_FLUSH_IDS;
|
|
4569
|
+
};
|
|
4570
|
+
if (Symbol.asyncIterator in ids) {
|
|
4571
|
+
for await (const id of ids) {
|
|
4572
|
+
if (ingest(id)) await flushPending();
|
|
4573
|
+
const pause = tickId();
|
|
4574
|
+
if (pause !== null) await pause;
|
|
4575
|
+
}
|
|
4576
|
+
} else {
|
|
4577
|
+
for (const id of ids) {
|
|
4578
|
+
if (ingest(id)) await flushPending();
|
|
4579
|
+
const pause = tickId();
|
|
4580
|
+
if (pause !== null) await pause;
|
|
4581
|
+
}
|
|
3440
4582
|
}
|
|
3441
|
-
|
|
4583
|
+
await flushPending();
|
|
4584
|
+
return [...byChunk].map(([chunkKey, bitmap]) => ({ chunkKey, bitmap })).sort((a, b) => a.chunkKey - b.chunkKey);
|
|
3442
4585
|
}
|
|
3443
4586
|
function openGenerationReader(storage, key, crypto, options = {}) {
|
|
3444
4587
|
return openChecked(storage, key, { ...options, crypto });
|
|
@@ -3467,6 +4610,11 @@ async function verifyGeneration(storage, key, expected, crypto) {
|
|
|
3467
4610
|
`verify failed for ${key.segment}.${key.generation}: cardinality ${reader.count()} != ${expected.cardinality}`
|
|
3468
4611
|
);
|
|
3469
4612
|
}
|
|
4613
|
+
if (!summaryAgrees(expected, { cardinality: reader.count(), metadata: reader.metadata })) {
|
|
4614
|
+
throw new IntegrityError(
|
|
4615
|
+
`verify failed for ${key.segment}.${key.generation}: the object's metadata is not the metadata it was written with`
|
|
4616
|
+
);
|
|
4617
|
+
}
|
|
3470
4618
|
}
|
|
3471
4619
|
|
|
3472
4620
|
// src/core/pinned-storage-source.ts
|
|
@@ -3515,6 +4663,11 @@ var PinnedStorageChunkSource = class {
|
|
|
3515
4663
|
if (pin === void 0) return this.inner.cardinalities(ref);
|
|
3516
4664
|
return pin.generation === null ? Promise.resolve(null) : this.inner.cardinalitiesAt(ref, pin.generation, heldBy(pin));
|
|
3517
4665
|
}
|
|
4666
|
+
summary(ref) {
|
|
4667
|
+
const pin = this.pinFor(ref);
|
|
4668
|
+
if (pin === void 0) return this.inner.summary(ref);
|
|
4669
|
+
return pin.generation === null ? Promise.resolve(null) : this.inner.summaryAt(ref, pin.generation, heldBy(pin));
|
|
4670
|
+
}
|
|
3518
4671
|
currentGeneration(ref) {
|
|
3519
4672
|
const pin = this.pinFor(ref);
|
|
3520
4673
|
return pin === void 0 ? this.inner.currentGeneration(ref) : Promise.resolve(pin.generation);
|
|
@@ -3551,67 +4704,58 @@ var PinnedStorageChunkSource = class {
|
|
|
3551
4704
|
}
|
|
3552
4705
|
};
|
|
3553
4706
|
|
|
3554
|
-
// src/core/
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
4707
|
+
// src/core/retry.ts
|
|
4708
|
+
var DEFAULT_RETRY_POLICY = {
|
|
4709
|
+
maxAttempts: 4,
|
|
4710
|
+
baseDelayMs: 50,
|
|
4711
|
+
maxDelayMs: 2e3,
|
|
4712
|
+
backoffFactor: 2,
|
|
4713
|
+
jitter: "full"
|
|
4714
|
+
};
|
|
4715
|
+
function backoffDelayMs(policy, attempt) {
|
|
4716
|
+
const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
|
|
4717
|
+
return Math.min(policy.maxDelayMs, raw);
|
|
3562
4718
|
}
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
const
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
if (after === null)
|
|
3576
|
-
throw new WriteConflictError(`registry row for segment ${ref.segment} was purged mid-pass`);
|
|
3577
|
-
if (record.status === "destroyed" && after.token !== record.token) {
|
|
3578
|
-
throw new WriteConflictError(
|
|
3579
|
-
`segment ${ref.segment} changed incarnation while its generations were being listed`
|
|
4719
|
+
function applyJitter(policy, delayMs, rng) {
|
|
4720
|
+
if (policy.jitter === "none") return delayMs;
|
|
4721
|
+
return rng.next() * delayMs;
|
|
4722
|
+
}
|
|
4723
|
+
function retryRead(op, retry) {
|
|
4724
|
+
return retry === void 0 ? op() : withRetry(op, retry.policy ?? DEFAULT_RETRY_POLICY, retry);
|
|
4725
|
+
}
|
|
4726
|
+
async function withRetry(op, policy, deps) {
|
|
4727
|
+
const retryable = deps.isRetryable ?? isTransientError;
|
|
4728
|
+
if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
|
|
4729
|
+
throw new ValidationError(
|
|
4730
|
+
`retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
|
|
3580
4731
|
);
|
|
3581
4732
|
}
|
|
3582
|
-
const
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
}
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
4733
|
+
const attempts = Math.floor(policy.maxAttempts);
|
|
4734
|
+
let lastErr;
|
|
4735
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
4736
|
+
try {
|
|
4737
|
+
return await op();
|
|
4738
|
+
} catch (err) {
|
|
4739
|
+
lastErr = err;
|
|
4740
|
+
if (attempt >= attempts || !retryable(err)) throw err;
|
|
4741
|
+
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
4742
|
+
try {
|
|
4743
|
+
deps.onRetry?.({ attempt, delayMs, err });
|
|
4744
|
+
} catch {
|
|
4745
|
+
}
|
|
4746
|
+
await deps.clock.sleep(delayMs);
|
|
3593
4747
|
}
|
|
3594
|
-
};
|
|
3595
|
-
const toDelete = record.status === "destroyed" ? gens.sort((a, b) => a - b) : cutoff === null ? (
|
|
3596
|
-
// No Storage pointer yet, so "below current" selects nothing and there is nothing safe to infer: an object
|
|
3597
|
-
// here is either a load about to publish or an orphan we cannot tell apart from it. Deleting would race
|
|
3598
|
-
// that publish into a dangling pointer. It is collected once a pointer exists.
|
|
3599
|
-
[]
|
|
3600
|
-
) : (
|
|
3601
|
-
// Delete generations below the cutoff, except the newest `keep` of them (the grace window).
|
|
3602
|
-
gens.filter((g) => g < cutoff).sort((a, b) => b - a).slice(keep)
|
|
3603
|
-
);
|
|
3604
|
-
for (const generation of toDelete) {
|
|
3605
|
-
await stillCollectable();
|
|
3606
|
-
await deps.storage.delete({ namespace: ref.namespace, segment: ref.segment, generation });
|
|
3607
4748
|
}
|
|
3608
|
-
|
|
4749
|
+
throw lastErr;
|
|
3609
4750
|
}
|
|
3610
4751
|
|
|
3611
4752
|
// src/core/load.ts
|
|
3612
4753
|
async function currentCardinality(ref, deps, record) {
|
|
3613
|
-
if (record === null || record.currentGen === null || record.status === "destroyed")
|
|
4754
|
+
if (record === null || record.currentGen === null || record.status === "destroyed") {
|
|
4755
|
+
return { cardinality: null, fromSummary: false };
|
|
4756
|
+
}
|
|
3614
4757
|
const generation = record.currentGen;
|
|
4758
|
+
let aead;
|
|
3615
4759
|
let crypto;
|
|
3616
4760
|
const wrapped = record.wrappedDeks;
|
|
3617
4761
|
if (wrapped !== void 0 && wrapped.length > 0) {
|
|
@@ -3620,23 +4764,29 @@ async function currentCardinality(ref, deps, record) {
|
|
|
3620
4764
|
`segment "${ref.segment}" is encrypted but load was given no keystore`
|
|
3621
4765
|
);
|
|
3622
4766
|
}
|
|
3623
|
-
|
|
3624
|
-
|
|
4767
|
+
aead = await deps.keystore.openDek(wrapped);
|
|
4768
|
+
const opened = aead;
|
|
4769
|
+
crypto = { aead: opened, aadFor: (scope) => aadFor(ref, generation, scope) };
|
|
3625
4770
|
}
|
|
4771
|
+
const described = usableSummary(ref, record, aead);
|
|
4772
|
+
if (described !== void 0) return { cardinality: described.cardinality, fromSummary: true };
|
|
3626
4773
|
let reader;
|
|
3627
4774
|
try {
|
|
3628
|
-
reader = await
|
|
4775
|
+
reader = await retryRead(
|
|
4776
|
+
() => openGenerationReader(deps.storage, { ...ref, generation }, crypto),
|
|
4777
|
+
deps.readRetry
|
|
4778
|
+
);
|
|
3629
4779
|
} catch (err) {
|
|
3630
4780
|
if (!isNotFoundError(err)) throw err;
|
|
3631
|
-
return null;
|
|
4781
|
+
return { cardinality: null, fromSummary: false };
|
|
3632
4782
|
}
|
|
3633
4783
|
let total = 0;
|
|
3634
4784
|
for (const n of reader.cardinalities().values()) total += n;
|
|
3635
|
-
return total;
|
|
4785
|
+
return { cardinality: total, fromSummary: false };
|
|
3636
4786
|
}
|
|
3637
|
-
async function loadSegment(ref,
|
|
4787
|
+
async function loadSegment(ref, input, deps, options = {}) {
|
|
3638
4788
|
validateUserRef(ref);
|
|
3639
|
-
requireCodec(deps.codec, "loadSegment");
|
|
4789
|
+
const codec = requireCodec(deps.codec, "loadSegment");
|
|
3640
4790
|
const keep = options.keep ?? 1;
|
|
3641
4791
|
if (!Number.isInteger(keep) || keep < 0) {
|
|
3642
4792
|
throw new ValidationError(`keep must be a non-negative integer; got ${String(keep)}`);
|
|
@@ -3656,13 +4806,19 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3656
4806
|
throw new ValidationError(`guard.minRetained must be a fraction in 0..1; got ${String(s)}`);
|
|
3657
4807
|
}
|
|
3658
4808
|
}
|
|
4809
|
+
const metadata = copiedMetadata(options.metadata, (message) => {
|
|
4810
|
+
throw new ValidationError(message);
|
|
4811
|
+
});
|
|
3659
4812
|
const audit = safeAudit(options.audit ?? NOOP_AUDIT);
|
|
4813
|
+
const ids = prepareLoadInput(input, codec);
|
|
4814
|
+
assertRegistryCanWrite(deps.registry, "load");
|
|
3660
4815
|
const row = await deps.registry.get(ref);
|
|
3661
4816
|
const fromToken = row?.token;
|
|
3662
4817
|
const fromGeneration = row?.currentGen ?? void 0;
|
|
3663
4818
|
const needsBefore = guard?.minRetained !== void 0 || options.allowEmpty !== true;
|
|
3664
|
-
const
|
|
3665
|
-
const
|
|
4819
|
+
const current = needsBefore ? await currentCardinality(ref, deps, row) : { cardinality: null, fromSummary: false };
|
|
4820
|
+
const before = current.cardinality;
|
|
4821
|
+
const { generation, checked } = await nextLoadGeneration(ref, deps, row);
|
|
3666
4822
|
const key = { namespace: ref.namespace, segment: ref.segment, generation };
|
|
3667
4823
|
let written;
|
|
3668
4824
|
try {
|
|
@@ -3672,7 +4828,9 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3672
4828
|
keystore: deps.keystore,
|
|
3673
4829
|
requireEncryption: deps.requireEncryption,
|
|
3674
4830
|
codec: deps.codec,
|
|
3675
|
-
clock: deps.clock
|
|
4831
|
+
clock: deps.clock,
|
|
4832
|
+
metadata,
|
|
4833
|
+
row
|
|
3676
4834
|
});
|
|
3677
4835
|
} catch (err) {
|
|
3678
4836
|
if (!isWriteConflictError(err)) throw err;
|
|
@@ -3696,16 +4854,28 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3696
4854
|
collected: []
|
|
3697
4855
|
};
|
|
3698
4856
|
}
|
|
3699
|
-
const
|
|
4857
|
+
const reclaim = async () => {
|
|
3700
4858
|
const now = await deps.registry.get(ref);
|
|
3701
|
-
if (now
|
|
4859
|
+
if (now !== null && (now.token === fromToken || now.status === "destroyed")) {
|
|
4860
|
+
await deps.storage.delete(key);
|
|
4861
|
+
return;
|
|
4862
|
+
}
|
|
4863
|
+
const keyed = now !== null && now.wrappedDeks !== void 0 && now.wrappedDeks.length > 0;
|
|
4864
|
+
if ((now === null || keyed && !written.encrypted) && await holdsObject(deps.storage, key, written.fingerprint)) {
|
|
4865
|
+
await deps.storage.delete(key);
|
|
4866
|
+
}
|
|
4867
|
+
};
|
|
4868
|
+
let unanswered = false;
|
|
4869
|
+
const refuse = async (reason) => {
|
|
4870
|
+
await reclaim();
|
|
3702
4871
|
audit.onEvent({
|
|
3703
4872
|
kind: "segment.load-refused",
|
|
3704
4873
|
segment: ref.segment,
|
|
3705
4874
|
namespace: ref.namespace,
|
|
3706
4875
|
generation,
|
|
3707
4876
|
reason,
|
|
3708
|
-
cardinality: written.cardinality
|
|
4877
|
+
cardinality: written.cardinality,
|
|
4878
|
+
...unanswered ? { unanswered: true } : {}
|
|
3709
4879
|
});
|
|
3710
4880
|
return {
|
|
3711
4881
|
generation,
|
|
@@ -3728,19 +4898,37 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3728
4898
|
if (guard?.minRetained !== void 0 && before !== null) {
|
|
3729
4899
|
if (written.cardinality < before * guard.minRetained) return refuse("min-retained");
|
|
3730
4900
|
}
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
4901
|
+
let published;
|
|
4902
|
+
try {
|
|
4903
|
+
published = await publishGeneration(deps.registry, key, {
|
|
4904
|
+
row,
|
|
4905
|
+
wrappedDeks: written.wrappedDeks,
|
|
4906
|
+
summary: written.summary,
|
|
4907
|
+
cleartext: !written.encrypted,
|
|
4908
|
+
...fromToken === void 0 ? {} : { expectToken: fromToken },
|
|
4909
|
+
...needsBefore && fromGeneration !== void 0 ? { expectFrom: fromGeneration } : {},
|
|
4910
|
+
// The third case, and the one the two fences above structurally cannot cover: the guard judged a segment
|
|
4911
|
+
// that had NO ROW. Both `expectFrom` and `expectToken` compare against a value read from a row, so with no
|
|
4912
|
+
// row both are omitted and the publish becomes a bare forward-only advance — which lands over anything
|
|
4913
|
+
// that appeared in between. `before` was `null`, so the empty and `minRetained` bounds had nothing to
|
|
4914
|
+
// judge and passed vacuously. Verified: an empty generation published over a thousand ids that a
|
|
4915
|
+
// concurrent writer had created meanwhile, reporting success. A guarded write therefore has to fence on
|
|
4916
|
+
// the ABSENCE it relied on, exactly as it fences on the pointer it relied on.
|
|
4917
|
+
...needsBefore && row === null ? { expectAbsent: true } : {},
|
|
4918
|
+
// A registry write that fails without an answer is reconciled by reading the row: the pointer at this number
|
|
4919
|
+
// is this load's publish only over the object this load wrote, which one footer read proves.
|
|
4920
|
+
holdsOwnObject: () => provesOwnObject(deps.storage, key, written.fingerprint),
|
|
4921
|
+
// And the wait before a fresh write, when the first left the row as it was.
|
|
4922
|
+
clock: deps.clock,
|
|
4923
|
+
rng: deps.rng ?? deps.readRetry?.rng,
|
|
4924
|
+
onUnanswered: () => {
|
|
4925
|
+
unanswered = true;
|
|
4926
|
+
}
|
|
4927
|
+
});
|
|
4928
|
+
} catch (err) {
|
|
4929
|
+
if (isValidationError(err) || err instanceof KeyUnavailableError) await reclaim();
|
|
4930
|
+
throw err;
|
|
4931
|
+
}
|
|
3744
4932
|
if (!published) return refuse("superseded");
|
|
3745
4933
|
audit.onEvent({
|
|
3746
4934
|
kind: "segment.publish",
|
|
@@ -3748,7 +4936,13 @@ async function loadSegment(ref, ids, deps, options = {}) {
|
|
|
3748
4936
|
namespace: ref.namespace,
|
|
3749
4937
|
generation
|
|
3750
4938
|
});
|
|
3751
|
-
const
|
|
4939
|
+
const currentObjectGone = needsBefore && before === null;
|
|
4940
|
+
const collected = await collectAfterLoad(ref, deps, {
|
|
4941
|
+
generation,
|
|
4942
|
+
keep,
|
|
4943
|
+
byName: checked && !currentObjectGone && deps.collectByListing !== true,
|
|
4944
|
+
...current.fromSummary && fromGeneration !== void 0 ? { proveCurrent: { ...ref, generation: fromGeneration } } : {}
|
|
4945
|
+
});
|
|
3752
4946
|
return {
|
|
3753
4947
|
generation,
|
|
3754
4948
|
published: true,
|
|
@@ -3768,7 +4962,25 @@ async function listGenerations(ref, deps) {
|
|
|
3768
4962
|
const current = record?.currentGen ?? null;
|
|
3769
4963
|
const seen = /* @__PURE__ */ new Set();
|
|
3770
4964
|
for await (const key of deps.storage.list(ref)) seen.add(key.generation);
|
|
3771
|
-
|
|
4965
|
+
const described = record === null ? void 0 : await currentDescription(ref, record, deps);
|
|
4966
|
+
return [...seen].sort((a, b) => a - b).map(
|
|
4967
|
+
(generation) => generation === current ? { generation, current: true, ...described } : { generation, current: false }
|
|
4968
|
+
);
|
|
4969
|
+
}
|
|
4970
|
+
async function currentDescription(ref, record, deps) {
|
|
4971
|
+
if (record.summary === void 0) return void 0;
|
|
4972
|
+
let aead;
|
|
4973
|
+
if ("sealed" in record.summary && record.wrappedDeks !== void 0 && deps.keystore !== void 0) {
|
|
4974
|
+
try {
|
|
4975
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
4976
|
+
} catch {
|
|
4977
|
+
return void 0;
|
|
4978
|
+
}
|
|
4979
|
+
}
|
|
4980
|
+
const usable = usableSummary(ref, record, aead);
|
|
4981
|
+
if (usable === void 0) return void 0;
|
|
4982
|
+
const metadata = usable.metadata;
|
|
4983
|
+
return metadata === void 0 || Object.keys(metadata).length === 0 ? { cardinality: usable.cardinality } : { cardinality: usable.cardinality, metadata: Object.freeze({ ...metadata }) };
|
|
3772
4984
|
}
|
|
3773
4985
|
async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
3774
4986
|
validateUserRef(ref);
|
|
@@ -3802,8 +5014,34 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3802
5014
|
`rollback: generation ${toGeneration} of "${ref.segment}" is above the current pointer (${record.currentGen}) \u2014 it may never have been published. Pass { allowForward: true } if you are undoing an earlier rollback.`
|
|
3803
5015
|
);
|
|
3804
5016
|
}
|
|
5017
|
+
const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
5018
|
+
let aead;
|
|
5019
|
+
if (keyed && deps.keystore !== void 0) {
|
|
5020
|
+
try {
|
|
5021
|
+
aead = await deps.keystore.openDek(record.wrappedDeks);
|
|
5022
|
+
} catch {
|
|
5023
|
+
}
|
|
5024
|
+
}
|
|
5025
|
+
const crypto = aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, toGeneration, scope) };
|
|
5026
|
+
const { encrypted, reader } = await openRollbackTarget(
|
|
5027
|
+
deps.storage,
|
|
5028
|
+
{ ...ref, generation: toGeneration },
|
|
5029
|
+
crypto
|
|
5030
|
+
);
|
|
5031
|
+
if (encrypted !== keyed) {
|
|
5032
|
+
throw new IntegrityError(
|
|
5033
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" is ${encrypted ? "encrypted" : "cleartext"}, but the segment is ${keyed ? "encrypted" : "cleartext"}, so no read of the segment could use it; it was never one of its generations. Roll back to another generation.`
|
|
5034
|
+
);
|
|
5035
|
+
}
|
|
5036
|
+
const summary = reader === void 0 ? void 0 : summaryOf(
|
|
5037
|
+
ref,
|
|
5038
|
+
toGeneration,
|
|
5039
|
+
{ cardinality: reader.count(), metadata: reader.metadata },
|
|
5040
|
+
aead
|
|
5041
|
+
);
|
|
3805
5042
|
const { token } = await deps.registry.compareAndSwap(ref, record.token, {
|
|
3806
|
-
currentGen: toGeneration
|
|
5043
|
+
currentGen: toGeneration,
|
|
5044
|
+
summary
|
|
3807
5045
|
});
|
|
3808
5046
|
let stillThere = false;
|
|
3809
5047
|
for await (const key of deps.storage.list(ref)) {
|
|
@@ -3812,15 +5050,31 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3812
5050
|
break;
|
|
3813
5051
|
}
|
|
3814
5052
|
}
|
|
3815
|
-
|
|
5053
|
+
let replaced = false;
|
|
5054
|
+
if (stillThere && reader !== void 0 && toGeneration > (record.currentGen ?? -1)) {
|
|
5055
|
+
try {
|
|
5056
|
+
replaced = !await provesOwnObject(
|
|
5057
|
+
deps.storage,
|
|
5058
|
+
{ ...ref, generation: toGeneration },
|
|
5059
|
+
reader.fingerprint
|
|
5060
|
+
);
|
|
5061
|
+
} catch (err) {
|
|
5062
|
+
if (!isIntegrityError(err)) throw err;
|
|
5063
|
+
replaced = true;
|
|
5064
|
+
}
|
|
5065
|
+
}
|
|
5066
|
+
if (!stillThere || replaced) {
|
|
3816
5067
|
let undone = false;
|
|
3817
5068
|
try {
|
|
3818
|
-
await deps.registry.compareAndSwap(ref, token, {
|
|
5069
|
+
await deps.registry.compareAndSwap(ref, token, {
|
|
5070
|
+
currentGen: record.currentGen,
|
|
5071
|
+
summary: describingSummary(record)
|
|
5072
|
+
});
|
|
3819
5073
|
undone = true;
|
|
3820
5074
|
} catch {
|
|
3821
5075
|
}
|
|
3822
5076
|
throw new NotFoundError(
|
|
3823
|
-
`rollback: generation ${toGeneration} of "${ref.segment}" was collected while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
5077
|
+
`rollback: generation ${toGeneration} of "${ref.segment}" was ${replaced ? "replaced" : "collected"} while the pointer was moving` + (undone ? " \u2014 the pointer was put back" : `, and the move back failed, so the pointer may still name ${toGeneration} (a write that landed and lost its response reads as a failure here). Check which generation is current, then re-run a rollback to a generation that exists, or restore the object.`)
|
|
3824
5078
|
);
|
|
3825
5079
|
}
|
|
3826
5080
|
safeAudit(options.audit ?? NOOP_AUDIT).onEvent({
|
|
@@ -3832,6 +5086,12 @@ async function rollbackSegment(ref, toGeneration, deps, options = {}) {
|
|
|
3832
5086
|
});
|
|
3833
5087
|
return { fromGeneration: record.currentGen, generation: toGeneration };
|
|
3834
5088
|
}
|
|
5089
|
+
function describingSummary(record) {
|
|
5090
|
+
const { summary } = record;
|
|
5091
|
+
if (summary === void 0 || summary.generation !== record.currentGen) return void 0;
|
|
5092
|
+
const keyed = record.wrappedDeks !== void 0 && record.wrappedDeks.length > 0;
|
|
5093
|
+
return "sealed" in summary === keyed ? summary : void 0;
|
|
5094
|
+
}
|
|
3835
5095
|
|
|
3836
5096
|
// src/core/erase-id.ts
|
|
3837
5097
|
var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
|
|
@@ -3841,6 +5101,7 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3841
5101
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
3842
5102
|
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
|
|
3843
5103
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
5104
|
+
const read = (op) => retryRead(op, deps.readRetry);
|
|
3844
5105
|
const record = await deps.registry.get(ref);
|
|
3845
5106
|
if (record === null) return { ...base, erased: false, reason: "absent", collected: [] };
|
|
3846
5107
|
if (record.status === "destroyed")
|
|
@@ -3877,13 +5138,18 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3877
5138
|
return row.currentGen === from && row.token === fromToken ? null : "superseded";
|
|
3878
5139
|
};
|
|
3879
5140
|
const holds = async (generation2) => {
|
|
5141
|
+
const key2 = { ...base, generation: generation2 };
|
|
5142
|
+
const chunkIn = (crypto) => read(async () => (await openGenerationReader(deps.storage, key2, crypto)).getChunk(chunkKey));
|
|
3880
5143
|
try {
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
5144
|
+
let bytes;
|
|
5145
|
+
try {
|
|
5146
|
+
bytes = await chunkIn(cryptoAt(generation2));
|
|
5147
|
+
} catch (err) {
|
|
5148
|
+
if (!isIntegrityError(err) || cryptoAt(generation2) === void 0 || await read(() => objectIsEncrypted(deps.storage, key2))) {
|
|
5149
|
+
throw err;
|
|
5150
|
+
}
|
|
5151
|
+
bytes = await chunkIn(void 0);
|
|
5152
|
+
}
|
|
3887
5153
|
return bytes !== null && codec.safeDeserialize(bytes, maxBytes).has(remainder);
|
|
3888
5154
|
} catch (err) {
|
|
3889
5155
|
if (isNotFoundError(err)) return null;
|
|
@@ -3953,27 +5219,40 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3953
5219
|
const stage = async () => {
|
|
3954
5220
|
let written;
|
|
3955
5221
|
try {
|
|
3956
|
-
const reader = await openGenerationReader(deps.storage, fromKey, cryptoAt(from));
|
|
3957
|
-
const bytes = await reader.getChunk(chunkKey);
|
|
5222
|
+
const reader = await read(() => openGenerationReader(deps.storage, fromKey, cryptoAt(from)));
|
|
5223
|
+
const bytes = await read(() => reader.getChunk(chunkKey));
|
|
3958
5224
|
if (bytes === null) return await notInCurrent();
|
|
3959
5225
|
const target = codec.safeDeserialize(bytes, maxBytes);
|
|
3960
5226
|
assertRemaindersInRange(target, chunkKey);
|
|
3961
5227
|
if (!target.has(remainder)) return await notInCurrent();
|
|
3962
5228
|
target.remove(remainder);
|
|
5229
|
+
assertRegistryCanWrite(deps.registry, "eraseIdFromSegment");
|
|
3963
5230
|
const generation2 = await nextGeneration(ref, deps);
|
|
3964
5231
|
const key2 = { ...base, generation: generation2 };
|
|
5232
|
+
const metadata = metadataToCarry(
|
|
5233
|
+
reader.metadata,
|
|
5234
|
+
aead === void 0 ? void 0 : usableSummary(ref, record, aead)
|
|
5235
|
+
);
|
|
3965
5236
|
const tally = await writeCrbmGenerationStream(
|
|
3966
5237
|
deps.storage,
|
|
3967
5238
|
key2,
|
|
3968
|
-
rewrite(reader, chunkKey, target, codec, maxBytes),
|
|
3969
|
-
{ crypto: cryptoAt(generation2), clock: deps.clock }
|
|
5239
|
+
rewrite(reader, chunkKey, target, codec, maxBytes, read),
|
|
5240
|
+
{ crypto: cryptoAt(generation2), clock: deps.clock, metadata }
|
|
3970
5241
|
);
|
|
3971
5242
|
written = generation2;
|
|
3972
5243
|
const beforeVerify = await deps.registry.get(ref);
|
|
3973
5244
|
const early = rowVerdict(beforeVerify);
|
|
3974
5245
|
if (early !== null) return refused(early, written);
|
|
3975
|
-
await
|
|
3976
|
-
|
|
5246
|
+
await read(
|
|
5247
|
+
() => verifyGeneration(deps.storage, key2, { ...tally, metadata }, cryptoAt(generation2))
|
|
5248
|
+
);
|
|
5249
|
+
const summary2 = summaryOf(
|
|
5250
|
+
ref,
|
|
5251
|
+
generation2,
|
|
5252
|
+
{ cardinality: tally.cardinality, metadata },
|
|
5253
|
+
aead
|
|
5254
|
+
);
|
|
5255
|
+
return { generation: generation2, key: key2, fingerprint: tally.fingerprint, summary: summary2 };
|
|
3977
5256
|
} catch (err) {
|
|
3978
5257
|
if (!isNotFoundError(err)) throw err;
|
|
3979
5258
|
let now;
|
|
@@ -3998,10 +5277,16 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
3998
5277
|
if (!staged.erased && staged.generation !== void 0) await discardRefused(staged.generation);
|
|
3999
5278
|
return staged;
|
|
4000
5279
|
}
|
|
4001
|
-
const { generation, key } = staged;
|
|
5280
|
+
const { generation, key, fingerprint, summary } = staged;
|
|
4002
5281
|
const published = await publishGeneration(deps.registry, key, {
|
|
4003
5282
|
expectFrom: from,
|
|
4004
|
-
expectToken: fromToken
|
|
5283
|
+
expectToken: fromToken,
|
|
5284
|
+
summary,
|
|
5285
|
+
// A write that ends without an answer is settled by reading the row, and a pointer at this number is this
|
|
5286
|
+
// rewrite's only over the object it wrote: the footer proves it, so another incarnation's cannot pass for it.
|
|
5287
|
+
holdsOwnObject: () => provesOwnObject(deps.storage, key, fingerprint),
|
|
5288
|
+
clock: deps.clock,
|
|
5289
|
+
rng: deps.rng ?? deps.readRetry?.rng
|
|
4005
5290
|
});
|
|
4006
5291
|
if (!published) {
|
|
4007
5292
|
await discardRefused(generation);
|
|
@@ -4026,14 +5311,14 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
4026
5311
|
if (left !== void 0) throw cannotRemove(left);
|
|
4027
5312
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
4028
5313
|
}
|
|
4029
|
-
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes) {
|
|
5314
|
+
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
4030
5315
|
const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
4031
5316
|
for (const k of keys) {
|
|
4032
5317
|
if (k === chunkKey) {
|
|
4033
5318
|
yield { chunkKey: k, bitmap: replacement };
|
|
4034
5319
|
continue;
|
|
4035
5320
|
}
|
|
4036
|
-
const bytes = await reader.getChunk(k);
|
|
5321
|
+
const bytes = await read(() => reader.getChunk(k));
|
|
4037
5322
|
if (bytes === null) continue;
|
|
4038
5323
|
const bitmap = codec.safeDeserialize(bytes, maxBytes);
|
|
4039
5324
|
assertRemaindersInRange(bitmap, k);
|
|
@@ -4052,8 +5337,8 @@ function assertRemaindersInRange(bitmap, chunkKey) {
|
|
|
4052
5337
|
// src/drivers/crypto.ts
|
|
4053
5338
|
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
4054
5339
|
var KEY_BYTES = 32;
|
|
4055
|
-
var
|
|
4056
|
-
var
|
|
5340
|
+
var NONCE_BYTES2 = 12;
|
|
5341
|
+
var TAG_BYTES2 = 16;
|
|
4057
5342
|
var NodeAead = class {
|
|
4058
5343
|
key;
|
|
4059
5344
|
constructor(key) {
|
|
@@ -4062,14 +5347,14 @@ var NodeAead = class {
|
|
|
4062
5347
|
this.key = Buffer.from(key);
|
|
4063
5348
|
}
|
|
4064
5349
|
seal(plaintext, aad) {
|
|
4065
|
-
const nonce = randomBytes(
|
|
5350
|
+
const nonce = randomBytes(NONCE_BYTES2);
|
|
4066
5351
|
const cipher = createCipheriv("aes-256-gcm", this.key, nonce);
|
|
4067
5352
|
cipher.setAAD(aad);
|
|
4068
5353
|
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
|
4069
5354
|
return { nonce, ciphertext, tag: cipher.getAuthTag() };
|
|
4070
5355
|
}
|
|
4071
5356
|
open(sealed, aad) {
|
|
4072
|
-
if (sealed.nonce.length !==
|
|
5357
|
+
if (sealed.nonce.length !== NONCE_BYTES2 || sealed.tag.length !== TAG_BYTES2)
|
|
4073
5358
|
throw new IntegrityError("AEAD nonce/tag have the wrong length");
|
|
4074
5359
|
const decipher = createDecipheriv("aes-256-gcm", this.key, sealed.nonce);
|
|
4075
5360
|
decipher.setAAD(aad);
|
|
@@ -4089,11 +5374,11 @@ function wrapKey(dek, kek, keyId) {
|
|
|
4089
5374
|
}
|
|
4090
5375
|
function unwrapKey(wrapped, kek, keyId) {
|
|
4091
5376
|
const blob = Buffer.from(wrapped, "base64");
|
|
4092
|
-
if (blob.length <
|
|
5377
|
+
if (blob.length < NONCE_BYTES2 + TAG_BYTES2)
|
|
4093
5378
|
throw new IntegrityError("wrapped DEK is too short to be valid");
|
|
4094
|
-
const nonce = blob.subarray(0,
|
|
4095
|
-
const tag = blob.subarray(blob.length -
|
|
4096
|
-
const ciphertext = blob.subarray(
|
|
5379
|
+
const nonce = blob.subarray(0, NONCE_BYTES2);
|
|
5380
|
+
const tag = blob.subarray(blob.length - TAG_BYTES2);
|
|
5381
|
+
const ciphertext = blob.subarray(NONCE_BYTES2, blob.length - TAG_BYTES2);
|
|
4097
5382
|
return new NodeAead(kek).open({ nonce, ciphertext, tag }, new TextEncoder().encode(keyId));
|
|
4098
5383
|
}
|
|
4099
5384
|
var InProcessKeystore = class {
|
|
@@ -4334,7 +5619,10 @@ async function shredSegment(ref, deps, allowCleartext, op = "destroySegment") {
|
|
|
4334
5619
|
status: "destroyed",
|
|
4335
5620
|
wrappedDeks: void 0,
|
|
4336
5621
|
// ← the crypto-shred: the only copy of the DEK wrappings is gone
|
|
4337
|
-
keyId: void 0
|
|
5622
|
+
keyId: void 0,
|
|
5623
|
+
// The current generation's cached count and metadata go with it: sealed, they cannot be opened without
|
|
5624
|
+
// the wrappings; clear, they would outlive the segment on its tombstone.
|
|
5625
|
+
summary: void 0
|
|
4338
5626
|
});
|
|
4339
5627
|
return { ...base, destroyed: true, cryptoShredded: encrypted };
|
|
4340
5628
|
} catch (err) {
|
|
@@ -4473,39 +5761,108 @@ function assertNotDestroyed(record, ref) {
|
|
|
4473
5761
|
|
|
4474
5762
|
// src/core/retention-sweep.ts
|
|
4475
5763
|
var DEFAULT_RETIRE_LIMIT = 100;
|
|
5764
|
+
var MAX_CONSECUTIVE_PURGE_FAULTS = 3;
|
|
4476
5765
|
var DEFAULT_TOMBSTONE_GRACE_MS = 864e5;
|
|
4477
5766
|
var DEFAULT_LOOKBACK_BUCKETS = 7;
|
|
4478
5767
|
async function rowsFromDueIndex(registry, options) {
|
|
4479
5768
|
const rows = [];
|
|
4480
|
-
const
|
|
5769
|
+
const pointers = /* @__PURE__ */ new Map();
|
|
5770
|
+
const litter = [];
|
|
5771
|
+
const gone = /* @__PURE__ */ new Set();
|
|
4481
5772
|
for (const bucket of dueBucketsAt(options.now, options.lookbackBuckets)) {
|
|
4482
5773
|
for await (const pointer of registry.list(dueNamespace(bucket))) {
|
|
4483
5774
|
const ref = decodeDueName(pointer.segment);
|
|
4484
5775
|
if (ref === null) continue;
|
|
4485
5776
|
if (isReservedNamespace(ref.namespace)) continue;
|
|
4486
5777
|
if (options.namespace !== void 0 && ref.namespace !== options.namespace) continue;
|
|
4487
|
-
const
|
|
4488
|
-
|
|
4489
|
-
|
|
5778
|
+
const found = {
|
|
5779
|
+
ref: { namespace: pointer.namespace, segment: pointer.segment },
|
|
5780
|
+
target: ref,
|
|
5781
|
+
token: pointer.token
|
|
5782
|
+
};
|
|
5783
|
+
const key = segmentKey(ref);
|
|
5784
|
+
const known = pointers.get(key);
|
|
5785
|
+
if (known !== void 0) {
|
|
5786
|
+
known.push(found);
|
|
5787
|
+
continue;
|
|
5788
|
+
}
|
|
5789
|
+
if (gone.has(key)) {
|
|
5790
|
+
litter.push(found);
|
|
5791
|
+
continue;
|
|
5792
|
+
}
|
|
4490
5793
|
if (rows.length >= options.maxScanSegments) {
|
|
4491
5794
|
throw new BudgetExceededError(
|
|
4492
5795
|
`retireExpired: the due index yielded more than ${options.maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Raise \`maxScanSegments\`, narrow with \`namespace\`, or reduce \`lookbackBuckets\`.`
|
|
4493
5796
|
);
|
|
4494
5797
|
}
|
|
4495
5798
|
const live = await registry.get(ref);
|
|
4496
|
-
if (live === null)
|
|
5799
|
+
if (live === null) {
|
|
5800
|
+
gone.add(key);
|
|
5801
|
+
litter.push(found);
|
|
5802
|
+
continue;
|
|
5803
|
+
}
|
|
4497
5804
|
rows.push(live);
|
|
5805
|
+
pointers.set(key, [found]);
|
|
4498
5806
|
}
|
|
4499
5807
|
}
|
|
4500
|
-
return rows;
|
|
5808
|
+
return { rows, pointers, litter };
|
|
5809
|
+
}
|
|
5810
|
+
async function rowsFromFleet(registry, options) {
|
|
5811
|
+
const held = [];
|
|
5812
|
+
const rows = await drainRegistry(registry, {
|
|
5813
|
+
namespace: options.namespace,
|
|
5814
|
+
maxScanSegments: options.maxScanSegments,
|
|
5815
|
+
op: "retireExpired",
|
|
5816
|
+
onReserved: (pointer) => {
|
|
5817
|
+
if (held.length >= options.maxScanSegments) return;
|
|
5818
|
+
const target = decodeDueName(pointer.segment);
|
|
5819
|
+
if (target === null || isReservedNamespace(target.namespace)) return;
|
|
5820
|
+
held.push({
|
|
5821
|
+
ref: { namespace: pointer.namespace, segment: pointer.segment },
|
|
5822
|
+
target,
|
|
5823
|
+
token: pointer.token
|
|
5824
|
+
});
|
|
5825
|
+
}
|
|
5826
|
+
});
|
|
5827
|
+
const pointers = /* @__PURE__ */ new Map();
|
|
5828
|
+
const litter = [];
|
|
5829
|
+
if (held.length === 0) return { rows, pointers, litter };
|
|
5830
|
+
const present = new Set(rows.map((rec) => segmentKey(rec)));
|
|
5831
|
+
for (const found of held) {
|
|
5832
|
+
const key = segmentKey(found.target);
|
|
5833
|
+
if (!present.has(key)) litter.push(found);
|
|
5834
|
+
else if (pointers.has(key)) pointers.get(key).push(found);
|
|
5835
|
+
else pointers.set(key, [found]);
|
|
5836
|
+
}
|
|
5837
|
+
return { rows, pointers, litter };
|
|
4501
5838
|
}
|
|
4502
|
-
async function forgetDuePointer(registry, ref, expiresAt) {
|
|
5839
|
+
async function forgetDuePointer(registry, ref, expiresAt, onFault) {
|
|
4503
5840
|
if (!canIndex(ref)) return;
|
|
4504
5841
|
try {
|
|
4505
5842
|
await registry.delete(dueIndexRef(dueBucket(expiresAt), ref));
|
|
5843
|
+
} catch (err) {
|
|
5844
|
+
onFault(err);
|
|
5845
|
+
}
|
|
5846
|
+
}
|
|
5847
|
+
async function filePurgePointer(registry, ref, purgeAt) {
|
|
5848
|
+
if (!canIndex(ref)) return;
|
|
5849
|
+
try {
|
|
5850
|
+
await registry.create(dueIndexRef(dueBucket(purgeAt), ref), { currentGen: null });
|
|
4506
5851
|
} catch {
|
|
4507
5852
|
}
|
|
4508
5853
|
}
|
|
5854
|
+
async function forgetPurgePointers(registry, ref, purgeAt, found, onFault) {
|
|
5855
|
+
if (!canIndex(ref)) return;
|
|
5856
|
+
const filed = dueIndexRef(dueBucket(purgeAt), ref);
|
|
5857
|
+
const all = found.some((p) => p.ref.namespace === filed.namespace) ? found : [...found, { ref: filed, target: ref }];
|
|
5858
|
+
for (const pointer of all) {
|
|
5859
|
+
try {
|
|
5860
|
+
await registry.delete(pointer.ref, pointer.token);
|
|
5861
|
+
} catch (err) {
|
|
5862
|
+
onFault(err);
|
|
5863
|
+
}
|
|
5864
|
+
}
|
|
5865
|
+
}
|
|
4509
5866
|
async function retireExpired(deps, options) {
|
|
4510
5867
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
4511
5868
|
const now = options.now;
|
|
@@ -4538,32 +5895,40 @@ async function retireExpired(deps, options) {
|
|
|
4538
5895
|
}
|
|
4539
5896
|
const dryRun = options.dryRun === true;
|
|
4540
5897
|
const scan = options.scan ?? "fleet";
|
|
4541
|
-
const
|
|
5898
|
+
const scanned = scan === "index" ? await rowsFromDueIndex(deps.registry, {
|
|
4542
5899
|
now,
|
|
4543
5900
|
lookbackBuckets: options.lookbackBuckets ?? DEFAULT_LOOKBACK_BUCKETS,
|
|
4544
5901
|
namespace: options.namespace,
|
|
4545
5902
|
maxScanSegments
|
|
4546
|
-
}) : await
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
op: "retireExpired"
|
|
4550
|
-
});
|
|
5903
|
+
}) : await rowsFromFleet(deps.registry, { namespace: options.namespace, maxScanSegments });
|
|
5904
|
+
const rows = scanned.rows;
|
|
5905
|
+
const removesRows = deps.registry.capabilities().conditionalDelete === true;
|
|
4551
5906
|
const shards = options.shards;
|
|
4552
5907
|
const totalShards = options.totalShards ?? 0;
|
|
4553
|
-
const
|
|
5908
|
+
const owned = (ref) => shards === void 0 || totalShards <= 1 || shards.includes(shardOf(segmentKey(ref), totalShards));
|
|
5909
|
+
const mine = rows.filter(owned);
|
|
4554
5910
|
const entries = [];
|
|
4555
5911
|
let eligible = 0;
|
|
4556
5912
|
let retired = 0;
|
|
4557
5913
|
let wouldRetire = 0;
|
|
4558
5914
|
let tombstonesPurged = 0;
|
|
4559
5915
|
let limited = false;
|
|
5916
|
+
let purgeFaults = 0;
|
|
5917
|
+
let firstPurgeFault;
|
|
5918
|
+
const noteFault = (err) => {
|
|
5919
|
+
if (isWriteConflictError(err)) return;
|
|
5920
|
+
purgeFaults += 1;
|
|
5921
|
+
firstPurgeFault ??= failureReason(err);
|
|
5922
|
+
};
|
|
5923
|
+
let purgeRun = 0;
|
|
5924
|
+
let purging = true;
|
|
4560
5925
|
let attempted = 0;
|
|
4561
5926
|
for (const rec of mine) {
|
|
4562
5927
|
const ref = { namespace: rec.namespace, segment: rec.segment };
|
|
4563
5928
|
const base = { segment: rec.segment, namespace: rec.namespace };
|
|
4564
5929
|
const policy = readRetentionPolicy(rec.retention);
|
|
4565
5930
|
if (rec.status === "destroyed") {
|
|
4566
|
-
if (!purgeTombstones) continue;
|
|
5931
|
+
if (!purgeTombstones || !purging) continue;
|
|
4567
5932
|
const retiredAt = retirementStamp(rec.retention);
|
|
4568
5933
|
if (retiredAt === null) continue;
|
|
4569
5934
|
if (now - retiredAt < grace) continue;
|
|
@@ -4580,13 +5945,31 @@ async function retireExpired(deps, options) {
|
|
|
4580
5945
|
continue;
|
|
4581
5946
|
}
|
|
4582
5947
|
}
|
|
4583
|
-
if (!dryRun)
|
|
4584
|
-
|
|
5948
|
+
if (!dryRun) {
|
|
5949
|
+
await deps.registry.delete(ref, rec.token);
|
|
5950
|
+
tombstonesPurged += 1;
|
|
5951
|
+
purgeRun = 0;
|
|
5952
|
+
if (removesRows) {
|
|
5953
|
+
await forgetPurgePointers(
|
|
5954
|
+
deps.registry,
|
|
5955
|
+
ref,
|
|
5956
|
+
retiredAt + grace,
|
|
5957
|
+
scanned.pointers.get(segmentKey(ref)) ?? [],
|
|
5958
|
+
noteFault
|
|
5959
|
+
);
|
|
5960
|
+
}
|
|
5961
|
+
}
|
|
4585
5962
|
entries.push({
|
|
4586
5963
|
...base,
|
|
4587
5964
|
action: dryRun ? "would-purge-tombstone" : "purged-tombstone"
|
|
4588
5965
|
});
|
|
4589
5966
|
} catch (err) {
|
|
5967
|
+
if (!isWriteConflictError(err)) {
|
|
5968
|
+
attempted -= 1;
|
|
5969
|
+
purgeRun += 1;
|
|
5970
|
+
if (purgeRun >= MAX_CONSECUTIVE_PURGE_FAULTS) purging = false;
|
|
5971
|
+
noteFault(err);
|
|
5972
|
+
}
|
|
4590
5973
|
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4591
5974
|
}
|
|
4592
5975
|
continue;
|
|
@@ -4630,22 +6013,27 @@ async function retireExpired(deps, options) {
|
|
|
4630
6013
|
}
|
|
4631
6014
|
retired += 1;
|
|
4632
6015
|
entries.push({ ...base, action: "retired", expiresAt: livePolicy.expiresAt, result });
|
|
4633
|
-
await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt);
|
|
6016
|
+
await forgetDuePointer(deps.registry, ref, livePolicy.expiresAt, noteFault);
|
|
4634
6017
|
if (purgeTombstones && result.generationsDeleted.length === 0 && result.generationsRemaining.length === 0) {
|
|
4635
6018
|
const tombstone = await deps.registry.get(ref).catch(() => null);
|
|
4636
6019
|
if (tombstone?.status === "destroyed") {
|
|
4637
6020
|
const deleted = await deps.registry.delete(ref, tombstone.token).then(
|
|
4638
6021
|
() => true,
|
|
4639
|
-
() =>
|
|
6022
|
+
(err) => {
|
|
6023
|
+
noteFault(err);
|
|
6024
|
+
return false;
|
|
6025
|
+
}
|
|
4640
6026
|
);
|
|
4641
6027
|
if (deleted) continue;
|
|
4642
6028
|
}
|
|
4643
6029
|
}
|
|
4644
|
-
await stampRetirement(deps.registry, ref, now).catch(() =>
|
|
6030
|
+
const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
|
|
6031
|
+
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
4645
6032
|
} catch (err) {
|
|
4646
6033
|
const after = await deps.registry.get(ref).catch(() => null);
|
|
4647
6034
|
if (after?.status === "destroyed") {
|
|
4648
|
-
await stampRetirement(deps.registry, ref, now).catch(() =>
|
|
6035
|
+
const stamped = await stampRetirement(deps.registry, ref, now).catch(() => false);
|
|
6036
|
+
if (stamped && removesRows) await filePurgePointer(deps.registry, ref, now + grace);
|
|
4649
6037
|
retired += 1;
|
|
4650
6038
|
entries.push({
|
|
4651
6039
|
...base,
|
|
@@ -4666,6 +6054,20 @@ async function retireExpired(deps, options) {
|
|
|
4666
6054
|
entries.push({ ...base, action: "skipped", reason: failureReason(err) });
|
|
4667
6055
|
}
|
|
4668
6056
|
}
|
|
6057
|
+
if (!dryRun && removesRows) {
|
|
6058
|
+
let attemptedLitter = 0;
|
|
6059
|
+
for (const pointer of scanned.litter) {
|
|
6060
|
+
if (!owned(pointer.target)) continue;
|
|
6061
|
+
if (attemptedLitter >= limit) break;
|
|
6062
|
+
attemptedLitter += 1;
|
|
6063
|
+
try {
|
|
6064
|
+
if (await deps.registry.get(pointer.target) !== null) continue;
|
|
6065
|
+
await deps.registry.delete(pointer.ref, pointer.token);
|
|
6066
|
+
} catch (err) {
|
|
6067
|
+
noteFault(err);
|
|
6068
|
+
}
|
|
6069
|
+
}
|
|
6070
|
+
}
|
|
4669
6071
|
return {
|
|
4670
6072
|
scanned: mine.length,
|
|
4671
6073
|
eligible,
|
|
@@ -4673,6 +6075,8 @@ async function retireExpired(deps, options) {
|
|
|
4673
6075
|
wouldRetire,
|
|
4674
6076
|
tombstonesPurged,
|
|
4675
6077
|
limited,
|
|
6078
|
+
purgeFaults,
|
|
6079
|
+
...firstPurgeFault === void 0 ? {} : { firstPurgeFault },
|
|
4676
6080
|
dryRun,
|
|
4677
6081
|
entries
|
|
4678
6082
|
};
|
|
@@ -4688,16 +6092,17 @@ function retirementStamp(meta) {
|
|
|
4688
6092
|
async function stampRetirement(registry, ref, now) {
|
|
4689
6093
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4690
6094
|
const rec = await registry.get(ref);
|
|
4691
|
-
if (rec === null || rec.status !== "destroyed") return;
|
|
6095
|
+
if (rec === null || rec.status !== "destroyed") return false;
|
|
4692
6096
|
try {
|
|
4693
6097
|
await registry.compareAndSwap(ref, rec.token, {
|
|
4694
6098
|
retention: { ...rec.retention, [RETIRED_AT]: now }
|
|
4695
6099
|
});
|
|
4696
|
-
return;
|
|
6100
|
+
return true;
|
|
4697
6101
|
} catch (err) {
|
|
4698
6102
|
if (!isWriteConflictError(err)) throw err;
|
|
4699
6103
|
}
|
|
4700
6104
|
}
|
|
6105
|
+
return false;
|
|
4701
6106
|
}
|
|
4702
6107
|
async function isFullyReclaimed(deps, ref) {
|
|
4703
6108
|
for await (const key of deps.storage.list(ref)) {
|
|
@@ -4718,8 +6123,45 @@ async function generationsPresent(storage, ref) {
|
|
|
4718
6123
|
for await (const key of storage.list(ref)) present.add(key.generation);
|
|
4719
6124
|
return present;
|
|
4720
6125
|
}
|
|
6126
|
+
async function checkSummary(ref, live, deps) {
|
|
6127
|
+
const generation = live.currentGen;
|
|
6128
|
+
const keyed = live.wrappedDeks !== void 0 && live.wrappedDeks.length > 0;
|
|
6129
|
+
let aead;
|
|
6130
|
+
if (keyed) {
|
|
6131
|
+
if (deps.keystore === void 0) return { kind: "ok", unchecked: true };
|
|
6132
|
+
try {
|
|
6133
|
+
aead = await deps.keystore.openDek(live.wrappedDeks);
|
|
6134
|
+
} catch (err) {
|
|
6135
|
+
if (isIntegrityError(err) || isCloudRoaringError(err) && err.name === "KeyUnavailableError")
|
|
6136
|
+
return { kind: "ok", unchecked: true };
|
|
6137
|
+
throw err;
|
|
6138
|
+
}
|
|
6139
|
+
}
|
|
6140
|
+
const described = usableSummary(ref, live, aead);
|
|
6141
|
+
if (described === void 0) return { kind: "ok", unchecked: keyed };
|
|
6142
|
+
const reader = await openGenerationReader(
|
|
6143
|
+
deps.storage,
|
|
6144
|
+
{ namespace: ref.namespace, segment: ref.segment, generation },
|
|
6145
|
+
aead === void 0 ? void 0 : { aead, aadFor: (scope) => aadFor(ref, generation, scope) }
|
|
6146
|
+
);
|
|
6147
|
+
let cardinality = 0;
|
|
6148
|
+
for (const n of reader.cardinalities().values()) cardinality += n;
|
|
6149
|
+
if (summaryAgrees(described, { cardinality, metadata: reader.metadata })) return { kind: "ok" };
|
|
6150
|
+
return {
|
|
6151
|
+
kind: "issue",
|
|
6152
|
+
issue: {
|
|
6153
|
+
segment: ref.segment,
|
|
6154
|
+
namespace: ref.namespace,
|
|
6155
|
+
currentGen: generation,
|
|
6156
|
+
issue: "summary-mismatch"
|
|
6157
|
+
}
|
|
6158
|
+
};
|
|
6159
|
+
}
|
|
4721
6160
|
async function runConsistencyCheck(deps, options = {}) {
|
|
4722
6161
|
if (options.namespace !== void 0) validateUserNamespace(options.namespace);
|
|
6162
|
+
if (options.summaries !== void 0 && typeof options.summaries !== "boolean") {
|
|
6163
|
+
throw new ValidationError(`summaries must be a boolean; got ${String(options.summaries)}`);
|
|
6164
|
+
}
|
|
4723
6165
|
const concurrency = options.concurrency ?? DEFAULT_CHECK_CONCURRENCY;
|
|
4724
6166
|
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
4725
6167
|
throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
@@ -4741,7 +6183,10 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4741
6183
|
if (!live || live.status === "destroyed") return { kind: "ok" };
|
|
4742
6184
|
if (live.currentGen === null) return { kind: "ok" };
|
|
4743
6185
|
const present = await generationsPresent(deps.storage, ref);
|
|
4744
|
-
if (present.has(live.currentGen))
|
|
6186
|
+
if (present.has(live.currentGen)) {
|
|
6187
|
+
if (options.summaries !== true || live.summary === void 0) return { kind: "ok" };
|
|
6188
|
+
return await checkSummary(ref, live, deps);
|
|
6189
|
+
}
|
|
4745
6190
|
return {
|
|
4746
6191
|
kind: "issue",
|
|
4747
6192
|
issue: {
|
|
@@ -4764,53 +6209,13 @@ async function runConsistencyCheck(deps, options = {}) {
|
|
|
4764
6209
|
});
|
|
4765
6210
|
const inconsistent = [];
|
|
4766
6211
|
const errored = [];
|
|
6212
|
+
let summariesUnchecked = 0;
|
|
4767
6213
|
for (const r of results) {
|
|
4768
6214
|
if (r.kind === "issue") inconsistent.push(r.issue);
|
|
4769
6215
|
else if (r.kind === "error") errored.push(r.error);
|
|
6216
|
+
else if (r.unchecked === true) summariesUnchecked += 1;
|
|
4770
6217
|
}
|
|
4771
|
-
return { checked: recs.length, inconsistent, errored };
|
|
4772
|
-
}
|
|
4773
|
-
|
|
4774
|
-
// src/core/retry.ts
|
|
4775
|
-
var DEFAULT_RETRY_POLICY = {
|
|
4776
|
-
maxAttempts: 4,
|
|
4777
|
-
baseDelayMs: 50,
|
|
4778
|
-
maxDelayMs: 2e3,
|
|
4779
|
-
backoffFactor: 2,
|
|
4780
|
-
jitter: "full"
|
|
4781
|
-
};
|
|
4782
|
-
function backoffDelayMs(policy, attempt) {
|
|
4783
|
-
const raw = policy.baseDelayMs * policy.backoffFactor ** (attempt - 1);
|
|
4784
|
-
return Math.min(policy.maxDelayMs, raw);
|
|
4785
|
-
}
|
|
4786
|
-
function applyJitter(policy, delayMs, rng) {
|
|
4787
|
-
if (policy.jitter === "none") return delayMs;
|
|
4788
|
-
return rng.next() * delayMs;
|
|
4789
|
-
}
|
|
4790
|
-
async function withRetry(op, policy, deps) {
|
|
4791
|
-
const retryable = deps.isRetryable ?? isTransientError;
|
|
4792
|
-
if (!Number.isFinite(policy.maxAttempts) || policy.maxAttempts < 1) {
|
|
4793
|
-
throw new ValidationError(
|
|
4794
|
-
`retry.maxAttempts must be a finite number >= 1 (got ${String(policy.maxAttempts)})`
|
|
4795
|
-
);
|
|
4796
|
-
}
|
|
4797
|
-
const attempts = Math.floor(policy.maxAttempts);
|
|
4798
|
-
let lastErr;
|
|
4799
|
-
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
4800
|
-
try {
|
|
4801
|
-
return await op();
|
|
4802
|
-
} catch (err) {
|
|
4803
|
-
lastErr = err;
|
|
4804
|
-
if (attempt >= attempts || !retryable(err)) throw err;
|
|
4805
|
-
const delayMs = applyJitter(policy, backoffDelayMs(policy, attempt), deps.rng);
|
|
4806
|
-
try {
|
|
4807
|
-
deps.onRetry?.({ attempt, delayMs, err });
|
|
4808
|
-
} catch {
|
|
4809
|
-
}
|
|
4810
|
-
await deps.clock.sleep(delayMs);
|
|
4811
|
-
}
|
|
4812
|
-
}
|
|
4813
|
-
throw lastErr;
|
|
6218
|
+
return options.summaries === true ? { checked: recs.length, inconsistent, errored, summariesUnchecked } : { checked: recs.length, inconsistent, errored };
|
|
4814
6219
|
}
|
|
4815
6220
|
|
|
4816
6221
|
// src/drivers/retry/retrying-chunk-source.ts
|
|
@@ -4832,6 +6237,7 @@ var RetryingStorageChunkSource = class {
|
|
|
4832
6237
|
/** Present only when the inner source supports it — so capability detection stays honest. */
|
|
4833
6238
|
sizeOf;
|
|
4834
6239
|
cardinalities;
|
|
6240
|
+
summary;
|
|
4835
6241
|
currentGeneration;
|
|
4836
6242
|
invalidate;
|
|
4837
6243
|
exists;
|
|
@@ -4856,6 +6262,10 @@ var RetryingStorageChunkSource = class {
|
|
|
4856
6262
|
if (innerCardinalities) {
|
|
4857
6263
|
this.cardinalities = (ref) => withRetry(() => innerCardinalities.call(inner, ref), this.policy, this.deps);
|
|
4858
6264
|
}
|
|
6265
|
+
const innerSummary = inner.summary;
|
|
6266
|
+
if (innerSummary) {
|
|
6267
|
+
this.summary = (ref) => withRetry(() => innerSummary.call(inner, ref), this.policy, this.deps);
|
|
6268
|
+
}
|
|
4859
6269
|
const innerCurrentGeneration = inner.currentGeneration;
|
|
4860
6270
|
if (innerCurrentGeneration) {
|
|
4861
6271
|
this.currentGeneration = (ref) => withRetry(() => innerCurrentGeneration.call(inner, ref), this.policy, this.deps);
|