@cloudbitmaps/roaring 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +23 -17
- package/README.md +13 -4
- package/dist/bin/export-segments.js +471 -173
- package/dist/bin/export-segments.js.map +3 -3
- package/dist/index.d.ts +261 -60
- package/dist/index.js +462 -169
- package/dist/index.js.map +3 -3
- package/dist/roaring-codec.d.ts +23 -22
- package/package.json +4 -3
|
@@ -21,17 +21,20 @@ import {
|
|
|
21
21
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
22
22
|
SegmentEngine,
|
|
23
23
|
LEASE_SKEW_MS,
|
|
24
|
+
BudgetExceededError,
|
|
24
25
|
LeaseExpiredError,
|
|
25
26
|
MAX_LEASE_MS,
|
|
26
27
|
NotFoundError,
|
|
27
28
|
UnsupportedError as UnsupportedError2,
|
|
28
|
-
ValidationError as
|
|
29
|
+
ValidationError as ValidationError5,
|
|
30
|
+
errorKind,
|
|
29
31
|
WriteConflictError,
|
|
30
32
|
checkBudget,
|
|
31
33
|
collectWithinBudget,
|
|
32
34
|
excludingReservedRows,
|
|
33
35
|
dropSegment,
|
|
34
36
|
eraseIdFromSegment,
|
|
37
|
+
findIdUnderTombstone,
|
|
35
38
|
judgeLoad,
|
|
36
39
|
loadSegment,
|
|
37
40
|
loadSegmentChunks,
|
|
@@ -52,7 +55,12 @@ import {
|
|
|
52
55
|
splitId,
|
|
53
56
|
takeLease
|
|
54
57
|
} from "@cloudbitmaps/core";
|
|
55
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
pointerIdOf,
|
|
60
|
+
requireBool,
|
|
61
|
+
requirePositiveInt,
|
|
62
|
+
validateSegmentRef
|
|
63
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
56
64
|
|
|
57
65
|
// src/open-charging-storage.ts
|
|
58
66
|
var OpenChargingStorage = class {
|
|
@@ -68,6 +76,7 @@ var OpenChargingStorage = class {
|
|
|
68
76
|
}
|
|
69
77
|
inner;
|
|
70
78
|
charge;
|
|
79
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
71
80
|
free = /* @__PURE__ */ new Set();
|
|
72
81
|
capabilities() {
|
|
73
82
|
return this.inner.capabilities();
|
|
@@ -80,9 +89,15 @@ var OpenChargingStorage = class {
|
|
|
80
89
|
getRange(key, offset, length) {
|
|
81
90
|
return this.inner.getRange(key, offset, length);
|
|
82
91
|
}
|
|
83
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
94
|
+
* generation is charged at its first open only.
|
|
95
|
+
*/
|
|
84
96
|
async getTail(key, maxBytes) {
|
|
85
|
-
if (!this.free.has(key.generation))
|
|
97
|
+
if (!this.free.has(key.generation)) {
|
|
98
|
+
this.charge();
|
|
99
|
+
this.free.add(key.generation);
|
|
100
|
+
}
|
|
86
101
|
return this.inner.getTail(key, maxBytes);
|
|
87
102
|
}
|
|
88
103
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -96,7 +111,12 @@ var OpenChargingStorage = class {
|
|
|
96
111
|
|
|
97
112
|
// src/index.ts
|
|
98
113
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
99
|
-
import {
|
|
114
|
+
import {
|
|
115
|
+
destroySegment as coreDestroySegment,
|
|
116
|
+
eraseNamespace as coreEraseNamespace
|
|
117
|
+
} from "@cloudbitmaps/core";
|
|
118
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
119
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
100
120
|
|
|
101
121
|
// src/reserved-namespace.ts
|
|
102
122
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -111,7 +131,7 @@ function refuseReservedNamespace(namespace) {
|
|
|
111
131
|
|
|
112
132
|
// src/roaring-codec.ts
|
|
113
133
|
import roaring from "roaring";
|
|
114
|
-
import { IntegrityError as IntegrityError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
134
|
+
import { IntegrityError as IntegrityError2, ValidationError as ValidationError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
115
135
|
|
|
116
136
|
// src/portable/layout.ts
|
|
117
137
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -359,10 +379,6 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
359
379
|
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
360
380
|
}
|
|
361
381
|
}
|
|
362
|
-
/** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
|
|
363
|
-
unwrap() {
|
|
364
|
-
return this.bitmap;
|
|
365
|
-
}
|
|
366
382
|
serialize() {
|
|
367
383
|
return this.bitmap.serialize(SerializationFormat.portable);
|
|
368
384
|
}
|
|
@@ -467,7 +483,7 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
467
483
|
}
|
|
468
484
|
};
|
|
469
485
|
function bitmapAsLoadInput(input) {
|
|
470
|
-
return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
|
|
486
|
+
return input instanceof RoaringBitmap32 || input instanceof ReadonlyRoaringView ? { bitmap: input } : input;
|
|
471
487
|
}
|
|
472
488
|
var roaringCodec = {
|
|
473
489
|
empty: () => SafeBitmap.empty(),
|
|
@@ -475,6 +491,31 @@ var roaringCodec = {
|
|
|
475
491
|
safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
476
492
|
owns: (bitmap) => bitmap instanceof SafeBitmap
|
|
477
493
|
};
|
|
494
|
+
var ReadonlyRoaringView = class {
|
|
495
|
+
#bitmap;
|
|
496
|
+
constructor(bitmap) {
|
|
497
|
+
this.#bitmap = bitmap;
|
|
498
|
+
}
|
|
499
|
+
get size() {
|
|
500
|
+
return this.#bitmap.size;
|
|
501
|
+
}
|
|
502
|
+
has(id) {
|
|
503
|
+
return this.#bitmap.has(id);
|
|
504
|
+
}
|
|
505
|
+
[Symbol.iterator]() {
|
|
506
|
+
return this.#bitmap[Symbol.iterator]();
|
|
507
|
+
}
|
|
508
|
+
/** The portable bytes, for exactly the argument `'portable'`; any other call is a `ValidationError`. */
|
|
509
|
+
serialize(...args) {
|
|
510
|
+
if (args.length !== 1 || args[0] !== "portable") {
|
|
511
|
+
const got = args.length === 0 ? "no argument" : args.map((a) => typeof a === "string" ? `'${a}'` : typeof a).join(", ");
|
|
512
|
+
throw new ValidationError2(
|
|
513
|
+
`a decoded bitmap serializes only as serialize('portable'); got ${got}`
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
return this.#bitmap.serialize();
|
|
517
|
+
}
|
|
518
|
+
};
|
|
478
519
|
|
|
479
520
|
// src/system-clock.ts
|
|
480
521
|
var SystemClock = class {
|
|
@@ -614,17 +655,18 @@ import {
|
|
|
614
655
|
DEFAULT_RETRY_POLICY,
|
|
615
656
|
NOOP_METRICS,
|
|
616
657
|
RetryingStorageChunkSource,
|
|
617
|
-
ValidationError as
|
|
658
|
+
ValidationError as ValidationError3,
|
|
618
659
|
isStorageBackend
|
|
619
660
|
} from "@cloudbitmaps/core";
|
|
620
661
|
function resolveRetryPolicy(partial) {
|
|
621
|
-
const
|
|
662
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
663
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
622
664
|
return {
|
|
623
|
-
maxAttempts:
|
|
665
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
624
666
|
baseDelayMs,
|
|
625
|
-
maxDelayMs: partial.maxDelayMs
|
|
626
|
-
backoffFactor:
|
|
627
|
-
jitter:
|
|
667
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
668
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
669
|
+
jitter: orDefault("jitter")
|
|
628
670
|
};
|
|
629
671
|
}
|
|
630
672
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -637,7 +679,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
637
679
|
if (options.retry !== false) {
|
|
638
680
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
639
681
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
640
|
-
throw new
|
|
682
|
+
throw new ValidationError3("retry.onRetry must be a function");
|
|
641
683
|
}
|
|
642
684
|
retryOptions = {
|
|
643
685
|
clock,
|
|
@@ -661,13 +703,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
661
703
|
source: resolved.source,
|
|
662
704
|
driver: resolved.driver,
|
|
663
705
|
registry: resolved.registry,
|
|
706
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
664
707
|
retryOptions
|
|
665
708
|
};
|
|
666
709
|
}
|
|
667
710
|
function resolveStorageSource(options, clock) {
|
|
668
711
|
const storage = options.storage;
|
|
669
712
|
if (storage === null || typeof storage !== "object") {
|
|
670
|
-
throw new
|
|
713
|
+
throw new ValidationError3(
|
|
671
714
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
672
715
|
);
|
|
673
716
|
}
|
|
@@ -677,14 +720,14 @@ function resolveStorageSource(options, clock) {
|
|
|
677
720
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
678
721
|
const isBackend = isStorageBackend(storage);
|
|
679
722
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
680
|
-
throw new
|
|
723
|
+
throw new ValidationError3(
|
|
681
724
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
682
725
|
);
|
|
683
726
|
}
|
|
684
727
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
685
728
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
686
729
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
687
|
-
throw new
|
|
730
|
+
throw new ValidationError3(
|
|
688
731
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
689
732
|
);
|
|
690
733
|
}
|
|
@@ -693,11 +736,11 @@ function resolveStorageSource(options, clock) {
|
|
|
693
736
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
694
737
|
if (!storageOk || !registryOk) {
|
|
695
738
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
696
|
-
throw new
|
|
739
|
+
throw new ValidationError3(
|
|
697
740
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
698
741
|
);
|
|
699
742
|
}
|
|
700
|
-
throw new
|
|
743
|
+
throw new ValidationError3(
|
|
701
744
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
702
745
|
);
|
|
703
746
|
}
|
|
@@ -718,18 +761,18 @@ function resolveStorageSource(options, clock) {
|
|
|
718
761
|
};
|
|
719
762
|
}
|
|
720
763
|
if (hasGetChunk && hasPutImmutable) {
|
|
721
|
-
throw new
|
|
764
|
+
throw new ValidationError3(
|
|
722
765
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
723
766
|
);
|
|
724
767
|
}
|
|
725
768
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
726
|
-
throw new
|
|
769
|
+
throw new ValidationError3(
|
|
727
770
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
728
771
|
);
|
|
729
772
|
}
|
|
730
773
|
if (hasGetChunk) {
|
|
731
774
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
732
|
-
throw new
|
|
775
|
+
throw new ValidationError3(
|
|
733
776
|
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
734
777
|
);
|
|
735
778
|
}
|
|
@@ -751,32 +794,32 @@ function resolveStorageSource(options, clock) {
|
|
|
751
794
|
}
|
|
752
795
|
|
|
753
796
|
// src/options.ts
|
|
754
|
-
import { ValidationError as
|
|
797
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core";
|
|
755
798
|
var keysOf = () => (keys) => keys;
|
|
756
799
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
757
800
|
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
758
801
|
if (options === void 0 || options === null) return void 0;
|
|
759
802
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
760
|
-
return new
|
|
803
|
+
return new ValidationError4(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
761
804
|
}
|
|
762
805
|
const bag = options;
|
|
763
806
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
764
807
|
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
765
808
|
if (unknown.length > 0) {
|
|
766
|
-
return new
|
|
809
|
+
return new ValidationError4(
|
|
767
810
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
768
811
|
);
|
|
769
812
|
}
|
|
770
813
|
const audit = bag.audit;
|
|
771
814
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
772
|
-
return new
|
|
815
|
+
return new ValidationError4(
|
|
773
816
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
774
817
|
);
|
|
775
818
|
}
|
|
776
819
|
for (const name of booleans) {
|
|
777
820
|
const v = bag[name];
|
|
778
821
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
779
|
-
return new
|
|
822
|
+
return new ValidationError4(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
780
823
|
}
|
|
781
824
|
}
|
|
782
825
|
return void 0;
|
|
@@ -796,16 +839,12 @@ function checkOptions(options, op, known, booleans, misplaced) {
|
|
|
796
839
|
}
|
|
797
840
|
|
|
798
841
|
// src/index.ts
|
|
842
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
799
843
|
import {
|
|
800
|
-
MemoryStorage,
|
|
801
|
-
LocalFsStorage,
|
|
802
|
-
destroySegment,
|
|
803
|
-
eraseNamespace,
|
|
804
844
|
excludingReservedRows as excludingReservedRows2,
|
|
805
845
|
readRetentionPolicy,
|
|
806
|
-
InProcessKeystore,
|
|
807
846
|
CloudRoaringError,
|
|
808
|
-
ValidationError as
|
|
847
|
+
ValidationError as ValidationError6,
|
|
809
848
|
WriteConflictError as WriteConflictError2,
|
|
810
849
|
IntegrityError as IntegrityError3,
|
|
811
850
|
NotFoundError as NotFoundError2,
|
|
@@ -813,11 +852,11 @@ import {
|
|
|
813
852
|
CapabilityError,
|
|
814
853
|
TransientError,
|
|
815
854
|
KeyUnavailableError,
|
|
816
|
-
BudgetExceededError,
|
|
855
|
+
BudgetExceededError as BudgetExceededError2,
|
|
817
856
|
LeaseExpiredError as LeaseExpiredError2,
|
|
818
857
|
LeaseLimitError,
|
|
819
858
|
StaleOperandError,
|
|
820
|
-
errorKind,
|
|
859
|
+
errorKind as errorKind2,
|
|
821
860
|
CountingMetricsSink,
|
|
822
861
|
RecordingAuditSink
|
|
823
862
|
} from "@cloudbitmaps/core";
|
|
@@ -827,13 +866,15 @@ var SystemRng = class {
|
|
|
827
866
|
return Math.random();
|
|
828
867
|
}
|
|
829
868
|
};
|
|
830
|
-
function validateRef(ref) {
|
|
869
|
+
function validateRef(ref, verb) {
|
|
831
870
|
if (typeof ref !== "object" || ref === null) {
|
|
832
|
-
throw new
|
|
871
|
+
throw new ValidationError5(
|
|
833
872
|
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
834
873
|
);
|
|
835
874
|
}
|
|
875
|
+
refuseUnknownRefKeys(ref, verb);
|
|
836
876
|
validateSegmentRef(ref);
|
|
877
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
837
878
|
}
|
|
838
879
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
839
880
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
@@ -848,15 +889,20 @@ function validateConcurrency(concurrency) {
|
|
|
848
889
|
function requireScope(options, op) {
|
|
849
890
|
if (options.namespace !== void 0) {
|
|
850
891
|
if (typeof options.namespace !== "string") {
|
|
851
|
-
throw new
|
|
892
|
+
throw new ValidationError5(
|
|
852
893
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
853
894
|
);
|
|
854
895
|
}
|
|
855
896
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
856
897
|
}
|
|
857
898
|
refuseReservedNamespace(options.namespace);
|
|
899
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
900
|
+
throw new ValidationError5(
|
|
901
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
858
904
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
859
|
-
throw new
|
|
905
|
+
throw new ValidationError5(
|
|
860
906
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
861
907
|
);
|
|
862
908
|
}
|
|
@@ -903,7 +949,7 @@ var CALL_KEYS = {
|
|
|
903
949
|
"metadata",
|
|
904
950
|
"keep"
|
|
905
951
|
]),
|
|
906
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
952
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
907
953
|
segments: keysOf()(["namespace"]),
|
|
908
954
|
subjectReport: keysOf()([
|
|
909
955
|
"namespace",
|
|
@@ -964,16 +1010,16 @@ var originOf;
|
|
|
964
1010
|
function assertOneStore(verb, role, h, origin) {
|
|
965
1011
|
if (h instanceof Segment) {
|
|
966
1012
|
if (originOf(h) === origin) return;
|
|
967
|
-
throw new
|
|
1013
|
+
throw new ValidationError5(
|
|
968
1014
|
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
969
1015
|
);
|
|
970
1016
|
}
|
|
971
1017
|
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
972
|
-
throw new
|
|
1018
|
+
throw new ValidationError5(
|
|
973
1019
|
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
974
1020
|
);
|
|
975
1021
|
}
|
|
976
|
-
throw new
|
|
1022
|
+
throw new ValidationError5(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
977
1023
|
}
|
|
978
1024
|
var MATERIALIZE_MANY_KEYS = [
|
|
979
1025
|
"operands",
|
|
@@ -1042,6 +1088,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1042
1088
|
// no underlying driver to write through).
|
|
1043
1089
|
storageDriver;
|
|
1044
1090
|
registry;
|
|
1091
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
1092
|
+
backend;
|
|
1045
1093
|
keystore;
|
|
1046
1094
|
requireEncryption;
|
|
1047
1095
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1061,7 +1109,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1061
1109
|
*/
|
|
1062
1110
|
static rejectUnknownOptions(options) {
|
|
1063
1111
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1064
|
-
throw new
|
|
1112
|
+
throw new ValidationError5(
|
|
1065
1113
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1066
1114
|
);
|
|
1067
1115
|
}
|
|
@@ -1079,7 +1127,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1079
1127
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1080
1128
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1081
1129
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1082
|
-
throw new
|
|
1130
|
+
throw new ValidationError5(
|
|
1083
1131
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1084
1132
|
);
|
|
1085
1133
|
}
|
|
@@ -1089,7 +1137,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1089
1137
|
if (unknown.length === 0) return;
|
|
1090
1138
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1091
1139
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1092
|
-
throw new
|
|
1140
|
+
throw new ValidationError5(
|
|
1093
1141
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1094
1142
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1095
1143
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1107,23 +1155,51 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1107
1155
|
];
|
|
1108
1156
|
const entropy = options.seams?.entropy;
|
|
1109
1157
|
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1110
|
-
throw new
|
|
1158
|
+
throw new ValidationError5(
|
|
1111
1159
|
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1112
1160
|
);
|
|
1113
1161
|
}
|
|
1114
1162
|
for (const [seam, name, needs] of seams) {
|
|
1115
|
-
if (seam === void 0
|
|
1116
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1117
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1118
|
-
throw new
|
|
1163
|
+
if (seam === void 0) continue;
|
|
1164
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
1165
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
1166
|
+
throw new ValidationError5(
|
|
1119
1167
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1120
1168
|
);
|
|
1121
1169
|
}
|
|
1122
1170
|
}
|
|
1123
1171
|
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
1174
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
1175
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
1176
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
1177
|
+
*/
|
|
1178
|
+
static rejectBadValues(options) {
|
|
1179
|
+
const cache = options.cache;
|
|
1180
|
+
if (cache !== void 0) {
|
|
1181
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
1182
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
1183
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
1184
|
+
const ttl = cache.ttlMs;
|
|
1185
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
1186
|
+
throw new ValidationError5(
|
|
1187
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
1192
|
+
}
|
|
1193
|
+
/**
|
|
1194
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
1195
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
1196
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
1197
|
+
* calls, throws `ValidationError`.
|
|
1198
|
+
*/
|
|
1124
1199
|
constructor(options) {
|
|
1125
1200
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1126
1201
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1202
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1127
1203
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1128
1204
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1129
1205
|
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
@@ -1154,6 +1230,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1154
1230
|
this.metrics = metrics;
|
|
1155
1231
|
this.storageDriver = wired.driver;
|
|
1156
1232
|
this.registry = wired.registry;
|
|
1233
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
1234
|
+
this.backend = wired.backend;
|
|
1157
1235
|
this.keystore = options.encryption?.keystore;
|
|
1158
1236
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1159
1237
|
}
|
|
@@ -1197,17 +1275,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1197
1275
|
*/
|
|
1198
1276
|
segment(name, options) {
|
|
1199
1277
|
if (typeof name === "object" && name !== null) {
|
|
1200
|
-
throw new
|
|
1278
|
+
throw new ValidationError5(
|
|
1201
1279
|
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1202
1280
|
);
|
|
1203
1281
|
}
|
|
1204
1282
|
if (options !== void 0 && options !== null) {
|
|
1205
1283
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1206
|
-
throw new
|
|
1284
|
+
throw new ValidationError5("segment: options must be an object such as { namespace }");
|
|
1207
1285
|
}
|
|
1208
1286
|
const proto = Object.getPrototypeOf(options);
|
|
1209
1287
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1210
|
-
throw new
|
|
1288
|
+
throw new ValidationError5(
|
|
1211
1289
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1212
1290
|
);
|
|
1213
1291
|
}
|
|
@@ -1215,7 +1293,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1215
1293
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1216
1294
|
);
|
|
1217
1295
|
if (unknown.length > 0) {
|
|
1218
|
-
throw new
|
|
1296
|
+
throw new ValidationError5(
|
|
1219
1297
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1220
1298
|
);
|
|
1221
1299
|
}
|
|
@@ -1297,7 +1375,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1297
1375
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1298
1376
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1299
1377
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1300
|
-
keep: options?.keep
|
|
1378
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1301
1379
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1302
1380
|
};
|
|
1303
1381
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1325,20 +1403,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1325
1403
|
/** The one body of the forms above. */
|
|
1326
1404
|
async materializeMany(options) {
|
|
1327
1405
|
if (typeof options !== "object" || options === null) {
|
|
1328
|
-
throw new
|
|
1406
|
+
throw new ValidationError5(
|
|
1329
1407
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1330
1408
|
);
|
|
1331
1409
|
}
|
|
1332
1410
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1333
1411
|
const unknown = unknownManyKeys(options);
|
|
1334
1412
|
if (unknown.length > 0) {
|
|
1335
|
-
throw new
|
|
1413
|
+
throw new ValidationError5(
|
|
1336
1414
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1337
1415
|
);
|
|
1338
1416
|
}
|
|
1339
1417
|
const dryRunIn = options.dryRun;
|
|
1340
1418
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1341
|
-
throw new
|
|
1419
|
+
throw new ValidationError5("materializeMany: dryRun must be a boolean");
|
|
1342
1420
|
}
|
|
1343
1421
|
const dryRun = dryRunIn === true;
|
|
1344
1422
|
const operandsIn = options.operands;
|
|
@@ -1350,7 +1428,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1350
1428
|
const feedIn = options.feed;
|
|
1351
1429
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1352
1430
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1353
|
-
throw new
|
|
1431
|
+
throw new ValidationError5(
|
|
1354
1432
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1355
1433
|
);
|
|
1356
1434
|
}
|
|
@@ -1360,15 +1438,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1360
1438
|
const budgetOption = options.budget;
|
|
1361
1439
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1362
1440
|
if (typeof pinOption !== "boolean")
|
|
1363
|
-
throw new
|
|
1441
|
+
throw new ValidationError5("materializeMany: pin must be a boolean");
|
|
1364
1442
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1365
|
-
throw new
|
|
1443
|
+
throw new ValidationError5("materializeMany: allowAbsentOperands must be a boolean");
|
|
1366
1444
|
}
|
|
1367
1445
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1368
|
-
throw new
|
|
1446
|
+
throw new ValidationError5("materializeMany: operands must be an object of name to segment");
|
|
1369
1447
|
}
|
|
1370
1448
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1371
|
-
throw new
|
|
1449
|
+
throw new ValidationError5("materializeMany: outputs must be a non-empty array");
|
|
1372
1450
|
}
|
|
1373
1451
|
const names = Object.keys(operandsIn);
|
|
1374
1452
|
const handles = /* @__PURE__ */ new Map();
|
|
@@ -1379,14 +1457,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1379
1457
|
}
|
|
1380
1458
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1381
1459
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1382
|
-
throw new
|
|
1460
|
+
throw new ValidationError5(
|
|
1383
1461
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1384
1462
|
);
|
|
1385
1463
|
}
|
|
1386
1464
|
if (mayBeEmptyIn.length > 0) {
|
|
1387
1465
|
const first = mayBeEmptyIn[0];
|
|
1388
1466
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1389
|
-
throw new
|
|
1467
|
+
throw new ValidationError5(
|
|
1390
1468
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1391
1469
|
);
|
|
1392
1470
|
}
|
|
@@ -1396,13 +1474,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1396
1474
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1397
1475
|
outputsIn.forEach((raw, i) => {
|
|
1398
1476
|
if (typeof raw !== "object" || raw === null) {
|
|
1399
|
-
throw new
|
|
1477
|
+
throw new ValidationError5(`outputs[${i}] must be an object`);
|
|
1400
1478
|
}
|
|
1401
1479
|
const bad = Object.keys(raw).filter(
|
|
1402
1480
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1403
1481
|
);
|
|
1404
1482
|
if (bad.length > 0) {
|
|
1405
|
-
throw new
|
|
1483
|
+
throw new ValidationError5(
|
|
1406
1484
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1407
1485
|
);
|
|
1408
1486
|
}
|
|
@@ -1410,12 +1488,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1410
1488
|
const dest = spec.dest;
|
|
1411
1489
|
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1412
1490
|
if (operandKeys.has(dest.key())) {
|
|
1413
|
-
throw new
|
|
1491
|
+
throw new ValidationError5(
|
|
1414
1492
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1415
1493
|
);
|
|
1416
1494
|
}
|
|
1417
1495
|
if (destKeys.has(dest.key())) {
|
|
1418
|
-
throw new
|
|
1496
|
+
throw new ValidationError5(
|
|
1419
1497
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1420
1498
|
);
|
|
1421
1499
|
}
|
|
@@ -1488,13 +1566,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1488
1566
|
held.set(key, { at, handle });
|
|
1489
1567
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1490
1568
|
if (!samePin(prior.at, at)) {
|
|
1491
|
-
throw new
|
|
1569
|
+
throw new ValidationError5(
|
|
1492
1570
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1493
1571
|
);
|
|
1494
1572
|
}
|
|
1495
1573
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1496
1574
|
if (!pinOption) {
|
|
1497
|
-
throw new
|
|
1575
|
+
throw new ValidationError5(
|
|
1498
1576
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1499
1577
|
);
|
|
1500
1578
|
}
|
|
@@ -1582,7 +1660,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1582
1660
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1583
1661
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1584
1662
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1585
|
-
throw new
|
|
1663
|
+
throw new ValidationError5(
|
|
1586
1664
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1587
1665
|
);
|
|
1588
1666
|
}
|
|
@@ -1632,13 +1710,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1632
1710
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1633
1711
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1634
1712
|
*
|
|
1713
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1714
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1715
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1716
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1717
|
+
*
|
|
1635
1718
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1636
1719
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1637
1720
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1638
1721
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1639
1722
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
1723
|
+
*
|
|
1724
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
1725
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
1726
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
1727
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
1728
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
1729
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
1730
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
1731
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
1732
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
1733
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
1734
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
1735
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
1736
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1640
1737
|
*/
|
|
1641
|
-
async subjectReport(id, options
|
|
1738
|
+
async subjectReport(id, options) {
|
|
1642
1739
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1643
1740
|
options ??= {};
|
|
1644
1741
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1653,26 +1750,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1653
1750
|
budget,
|
|
1654
1751
|
"subjectReport"
|
|
1655
1752
|
);
|
|
1753
|
+
let opens = 0;
|
|
1754
|
+
const charge = () => {
|
|
1755
|
+
opens += 1;
|
|
1756
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
1757
|
+
throw new BudgetExceededError(
|
|
1758
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
1759
|
+
{
|
|
1760
|
+
option: "budget.maxRequests",
|
|
1761
|
+
limit: budget.maxRequests,
|
|
1762
|
+
spent: budget.maxRequests + 1
|
|
1763
|
+
}
|
|
1764
|
+
);
|
|
1765
|
+
}
|
|
1766
|
+
};
|
|
1656
1767
|
const membership = await mapWithConcurrency(
|
|
1657
1768
|
recs,
|
|
1658
1769
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1659
1770
|
async (rec) => {
|
|
1660
|
-
if (rec.status === "destroyed") return null;
|
|
1661
1771
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1772
|
+
if (rec.status === "destroyed") {
|
|
1773
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
1774
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
1775
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
1776
|
+
return held === null ? null : { tombstone: true, ref };
|
|
1777
|
+
}
|
|
1662
1778
|
if (this.crbmSource !== void 0) {
|
|
1663
1779
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1664
|
-
const
|
|
1665
|
-
if (rowVersionOfLive(held) !==
|
|
1780
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1781
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1666
1782
|
}
|
|
1667
|
-
return await this.engine.has(ref, id) ? {
|
|
1783
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1668
1784
|
}
|
|
1669
1785
|
);
|
|
1670
|
-
const
|
|
1671
|
-
|
|
1786
|
+
const listed = membership.filter((m) => m !== null);
|
|
1787
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
1788
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
1789
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1672
1790
|
}
|
|
1673
1791
|
/**
|
|
1674
1792
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1675
|
-
* from the bucket on return
|
|
1793
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
1794
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
1795
|
+
*
|
|
1796
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1797
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1798
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1799
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1676
1800
|
*
|
|
1677
1801
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1678
1802
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1737,15 +1861,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1737
1861
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1738
1862
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1739
1863
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1740
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1741
|
-
*
|
|
1864
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
1865
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
1866
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
1867
|
+
* that exhausts the budget is
|
|
1742
1868
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1743
1869
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1744
1870
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1745
1871
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1746
1872
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1747
1873
|
*/
|
|
1748
|
-
async eraseSubject(id, options
|
|
1874
|
+
async eraseSubject(id, options) {
|
|
1749
1875
|
this.erasureEpoch++;
|
|
1750
1876
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1751
1877
|
options ??= {};
|
|
@@ -1801,7 +1927,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1801
1927
|
segment: rec.segment,
|
|
1802
1928
|
namespace: rec.namespace,
|
|
1803
1929
|
erased: false,
|
|
1804
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
1930
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
1931
|
+
// the class lets a note be matched to the guide's table
|
|
1805
1932
|
};
|
|
1806
1933
|
} finally {
|
|
1807
1934
|
try {
|
|
@@ -1822,11 +1949,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1822
1949
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1823
1950
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1824
1951
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1952
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1825
1953
|
*
|
|
1826
1954
|
* Needs a backend.
|
|
1827
1955
|
*/
|
|
1828
1956
|
async generations(ref) {
|
|
1829
|
-
validateRef(ref);
|
|
1957
|
+
ref = validateRef(ref, "generations");
|
|
1830
1958
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1831
1959
|
}
|
|
1832
1960
|
/**
|
|
@@ -1856,7 +1984,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1856
1984
|
* ```
|
|
1857
1985
|
*/
|
|
1858
1986
|
async exists(ref) {
|
|
1859
|
-
validateRef(ref);
|
|
1987
|
+
ref = validateRef(ref, "exists");
|
|
1860
1988
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1861
1989
|
}
|
|
1862
1990
|
/**
|
|
@@ -1913,8 +2041,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1913
2041
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1914
2042
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1915
2043
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1916
|
-
* crypto-shredded
|
|
1917
|
-
*
|
|
2044
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
2045
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
2046
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1918
2047
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1919
2048
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1920
2049
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1925,8 +2054,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1925
2054
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1926
2055
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1927
2056
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1928
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1929
|
-
*
|
|
2057
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
2058
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
2059
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
2060
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1930
2061
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1931
2062
|
*
|
|
1932
2063
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1956,7 +2087,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1956
2087
|
* Needs a backend.
|
|
1957
2088
|
*/
|
|
1958
2089
|
async rollback(ref, toGeneration, options = {}) {
|
|
1959
|
-
validateRef(ref);
|
|
2090
|
+
ref = validateRef(ref, "rollback");
|
|
1960
2091
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1961
2092
|
options ??= {};
|
|
1962
2093
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1973,8 +2104,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1973
2104
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1974
2105
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1975
2106
|
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. A bare
|
|
1976
|
-
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A
|
|
1977
|
-
* `ValidationError`, since each byte would load as an id:
|
|
2107
|
+
* `RoaringBitmap32`, or a bitmap `deserializePortable` returned, passed as ids loads as `{ bitmap }`. A
|
|
2108
|
+
* `Uint8Array` or `Buffer` passed as ids is refused with `ValidationError`, since each byte would load as an id:
|
|
2109
|
+
* pass bytes as `{ serialized }`.
|
|
2110
|
+
*
|
|
2111
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
2112
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
2113
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
2114
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1978
2115
|
*
|
|
1979
2116
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1980
2117
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -2033,7 +2170,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2033
2170
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
2034
2171
|
* what the pass deleted by name, and one name may have been gone already.
|
|
2035
2172
|
*
|
|
2036
|
-
*
|
|
2173
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
2174
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
2175
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
2176
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
2177
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
2178
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
2179
|
+
*
|
|
2180
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
2037
2181
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
2038
2182
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
2039
2183
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2056,8 +2200,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2056
2200
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2057
2201
|
*/
|
|
2058
2202
|
async load(ref, input, options = {}) {
|
|
2059
|
-
validateRef(ref);
|
|
2060
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
2203
|
+
ref = validateRef(ref, "load");
|
|
2204
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
2205
|
+
if (typeof input === "object" && input !== null) {
|
|
2206
|
+
const reads = STREAM_SOURCES.get(input);
|
|
2207
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
2208
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
2209
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
2210
|
+
throw new ValidationError5(
|
|
2211
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
2212
|
+
);
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2061
2215
|
options ??= {};
|
|
2062
2216
|
const deps = this.lifecycleDeps("load");
|
|
2063
2217
|
try {
|
|
@@ -2067,11 +2221,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2067
2221
|
}
|
|
2068
2222
|
}
|
|
2069
2223
|
/**
|
|
2070
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
2224
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
2225
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2071
2226
|
*
|
|
2072
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2073
|
-
*
|
|
2074
|
-
*
|
|
2227
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
2228
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
2229
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
2230
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
2231
|
+
* encrypted segment it does both.
|
|
2075
2232
|
*
|
|
2076
2233
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2077
2234
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2112,9 +2269,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2112
2269
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2113
2270
|
*/
|
|
2114
2271
|
async dropSegment(ref, options) {
|
|
2115
|
-
validateRef(ref);
|
|
2272
|
+
ref = validateRef(ref, "dropSegment");
|
|
2116
2273
|
if (options === void 0 || options === null) {
|
|
2117
|
-
throw new
|
|
2274
|
+
throw new ValidationError5("dropSegment: options { confirmSegment } are required");
|
|
2118
2275
|
}
|
|
2119
2276
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2120
2277
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2129,8 +2286,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2129
2286
|
}
|
|
2130
2287
|
}
|
|
2131
2288
|
/**
|
|
2132
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2133
|
-
*
|
|
2289
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
2290
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
2291
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2134
2292
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2135
2293
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2136
2294
|
*
|
|
@@ -2152,10 +2310,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2152
2310
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2153
2311
|
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2154
2312
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2155
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
2313
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2156
2314
|
*/
|
|
2157
2315
|
async setRetention(ref, policy) {
|
|
2158
|
-
validateRef(ref);
|
|
2316
|
+
ref = validateRef(ref, "setRetention");
|
|
2159
2317
|
return setSegmentRetention(
|
|
2160
2318
|
ref,
|
|
2161
2319
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2169,16 +2327,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2169
2327
|
* "never expires" on a segment someone believes is expiring.
|
|
2170
2328
|
*/
|
|
2171
2329
|
async getRetention(ref) {
|
|
2172
|
-
validateRef(ref);
|
|
2330
|
+
ref = validateRef(ref, "getRetention");
|
|
2173
2331
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2174
2332
|
}
|
|
2175
2333
|
/**
|
|
2176
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2177
|
-
* when there was none
|
|
2334
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
2335
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
2336
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
2337
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2178
2338
|
* to the setter is how a typo becomes a deletion.
|
|
2179
2339
|
*/
|
|
2180
2340
|
async clearRetention(ref) {
|
|
2181
|
-
validateRef(ref);
|
|
2341
|
+
ref = validateRef(ref, "clearRetention");
|
|
2182
2342
|
return clearSegmentRetention(ref, {
|
|
2183
2343
|
registry: this.requireRegistry("clearRetention"),
|
|
2184
2344
|
clock: this.clock,
|
|
@@ -2203,20 +2363,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2203
2363
|
*
|
|
2204
2364
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2205
2365
|
* for (const e of swept.entries) {
|
|
2206
|
-
*
|
|
2366
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
2367
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2207
2368
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2208
2369
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2209
2370
|
* }
|
|
2210
2371
|
* }
|
|
2211
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
2372
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2212
2373
|
* ```
|
|
2213
2374
|
*
|
|
2214
2375
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2215
2376
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2216
2377
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2217
|
-
* `limit` (default 100) cut the pass short and more
|
|
2218
|
-
*
|
|
2219
|
-
*
|
|
2378
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
2379
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
2380
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
2381
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2220
2382
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2221
2383
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2222
2384
|
*
|
|
@@ -2226,9 +2388,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2226
2388
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2227
2389
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2228
2390
|
*
|
|
2229
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
2391
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2230
2392
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2231
|
-
* first, since nothing else
|
|
2393
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
2394
|
+
* only
|
|
2395
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2232
2396
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2233
2397
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2234
2398
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2249,7 +2413,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2249
2413
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2250
2414
|
const result = await retireExpired(
|
|
2251
2415
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2252
|
-
{ ...opts, now: opts.now
|
|
2416
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2253
2417
|
);
|
|
2254
2418
|
if (opts.dryRun !== true) {
|
|
2255
2419
|
for (const entry of result.entries) {
|
|
@@ -2296,7 +2460,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2296
2460
|
* ```
|
|
2297
2461
|
*/
|
|
2298
2462
|
invalidate(ref) {
|
|
2299
|
-
validateRef(ref);
|
|
2463
|
+
ref = validateRef(ref, "invalidate");
|
|
2300
2464
|
refuseReservedNamespace(ref.namespace);
|
|
2301
2465
|
this.engine.invalidate(ref);
|
|
2302
2466
|
}
|
|
@@ -2356,11 +2520,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2356
2520
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2357
2521
|
throw err;
|
|
2358
2522
|
}
|
|
2359
|
-
const pinnedAt = {
|
|
2360
|
-
generation: at?.generation ?? null,
|
|
2361
|
-
version: at?.version ?? null,
|
|
2362
|
-
fingerprint: at?.fingerprint ?? null
|
|
2363
|
-
};
|
|
2523
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2364
2524
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2365
2525
|
return makeSegment({
|
|
2366
2526
|
engine: this.engineWithPins(pins),
|
|
@@ -2430,8 +2590,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2430
2590
|
if (held.has(key)) {
|
|
2431
2591
|
const before = held.get(key);
|
|
2432
2592
|
if (!samePin(before, at)) {
|
|
2433
|
-
throw new
|
|
2434
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
2593
|
+
throw new ValidationError5(
|
|
2594
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2435
2595
|
);
|
|
2436
2596
|
}
|
|
2437
2597
|
} else {
|
|
@@ -2462,8 +2622,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2462
2622
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2463
2623
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2464
2624
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2465
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2466
|
-
*
|
|
2625
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
2626
|
+
* dropped) are skipped. Pair
|
|
2627
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2467
2628
|
*
|
|
2468
2629
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2469
2630
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2496,6 +2657,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2496
2657
|
options
|
|
2497
2658
|
);
|
|
2498
2659
|
}
|
|
2660
|
+
/**
|
|
2661
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
2662
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
2663
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
2664
|
+
*
|
|
2665
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
2666
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
2667
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
2668
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
2669
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
2670
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
2671
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
2672
|
+
*
|
|
2673
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
2674
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
2675
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
2676
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
2677
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
2678
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
2679
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
2680
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
2681
|
+
* timing.
|
|
2682
|
+
*
|
|
2683
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
2684
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
2685
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
2686
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
2687
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
2688
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
2689
|
+
*
|
|
2690
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
2691
|
+
*/
|
|
2692
|
+
async preflight() {
|
|
2693
|
+
const deps = this.lifecycleDeps("preflight");
|
|
2694
|
+
const backend = this.backend;
|
|
2695
|
+
let checkLifecycle;
|
|
2696
|
+
try {
|
|
2697
|
+
const method = backend?.checkLifecycle;
|
|
2698
|
+
if (typeof method === "function")
|
|
2699
|
+
checkLifecycle = method.bind(backend);
|
|
2700
|
+
} catch {
|
|
2701
|
+
}
|
|
2702
|
+
return runPreflight({
|
|
2703
|
+
storage: deps.storage,
|
|
2704
|
+
registry: deps.registry,
|
|
2705
|
+
clock: this.clock,
|
|
2706
|
+
entropy: this.entropy,
|
|
2707
|
+
checkLifecycle
|
|
2708
|
+
});
|
|
2709
|
+
}
|
|
2499
2710
|
/**
|
|
2500
2711
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2501
2712
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2504,8 +2715,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2504
2715
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2505
2716
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2506
2717
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2507
|
-
*
|
|
2508
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
2718
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
2719
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
2720
|
+
* check `failed`. The `export-segments`
|
|
2509
2721
|
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2510
2722
|
*/
|
|
2511
2723
|
async exportSegments(sink, options = {}) {
|
|
@@ -2540,12 +2752,23 @@ var readOptions = (options) => ({
|
|
|
2540
2752
|
});
|
|
2541
2753
|
var NO_SEGMENTS = [];
|
|
2542
2754
|
function segmentList(list, what) {
|
|
2543
|
-
if (!Array.isArray(list)) throw new
|
|
2755
|
+
if (!Array.isArray(list)) throw new ValidationError5(`${what} must be an array of segments`);
|
|
2544
2756
|
return list;
|
|
2545
2757
|
}
|
|
2758
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
2759
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
2760
|
+
function readsFrom(stream, origin, keys) {
|
|
2761
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
2762
|
+
return stream;
|
|
2763
|
+
}
|
|
2546
2764
|
function operandsLabel(verb) {
|
|
2547
2765
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2548
2766
|
}
|
|
2767
|
+
function refuseNoOperands(verb) {
|
|
2768
|
+
throw new ValidationError5(
|
|
2769
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
2770
|
+
);
|
|
2771
|
+
}
|
|
2549
2772
|
var AS_IDS = {
|
|
2550
2773
|
failing,
|
|
2551
2774
|
guard: guardIds,
|
|
@@ -2562,9 +2785,9 @@ var CombineChunks = class {
|
|
|
2562
2785
|
var AS_CHUNKS = {
|
|
2563
2786
|
guard: guardItems,
|
|
2564
2787
|
failing: (err) => refusing(err)(),
|
|
2565
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2566
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2567
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
2788
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
2789
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
2790
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2568
2791
|
};
|
|
2569
2792
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2570
2793
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2623,7 +2846,7 @@ var Segment = class _Segment {
|
|
|
2623
2846
|
*/
|
|
2624
2847
|
constructor(parts) {
|
|
2625
2848
|
if (!minting) {
|
|
2626
|
-
throw new
|
|
2849
|
+
throw new ValidationError5(
|
|
2627
2850
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2628
2851
|
);
|
|
2629
2852
|
}
|
|
@@ -2662,7 +2885,7 @@ var Segment = class _Segment {
|
|
|
2662
2885
|
*
|
|
2663
2886
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2664
2887
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2665
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
2888
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2666
2889
|
*
|
|
2667
2890
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2668
2891
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2737,11 +2960,11 @@ var Segment = class _Segment {
|
|
|
2737
2960
|
leaseUntilOf(op, options) {
|
|
2738
2961
|
if (options === void 0 || options === null) return void 0;
|
|
2739
2962
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2740
|
-
throw new
|
|
2963
|
+
throw new ValidationError5(`${op}: options must be an object such as { leaseUntil }`);
|
|
2741
2964
|
}
|
|
2742
2965
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2743
2966
|
if (unknown.length > 0) {
|
|
2744
|
-
throw new
|
|
2967
|
+
throw new ValidationError5(
|
|
2745
2968
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2746
2969
|
);
|
|
2747
2970
|
}
|
|
@@ -2749,17 +2972,17 @@ var Segment = class _Segment {
|
|
|
2749
2972
|
if (until === void 0) return void 0;
|
|
2750
2973
|
const now = this.clock.now();
|
|
2751
2974
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2752
|
-
throw new
|
|
2975
|
+
throw new ValidationError5(
|
|
2753
2976
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2754
2977
|
);
|
|
2755
2978
|
}
|
|
2756
2979
|
if (until <= now) {
|
|
2757
|
-
throw new
|
|
2980
|
+
throw new ValidationError5(
|
|
2758
2981
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2759
2982
|
);
|
|
2760
2983
|
}
|
|
2761
2984
|
if (until > now + MAX_LEASE_MS) {
|
|
2762
|
-
throw new
|
|
2985
|
+
throw new ValidationError5(
|
|
2763
2986
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2764
2987
|
);
|
|
2765
2988
|
}
|
|
@@ -2837,10 +3060,10 @@ var Segment = class _Segment {
|
|
|
2837
3060
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2838
3061
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2839
3062
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2840
|
-
* than `generation` and `
|
|
3063
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2841
3064
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2842
3065
|
*
|
|
2843
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
3066
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2844
3067
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2845
3068
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2846
3069
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2853,18 +3076,32 @@ var Segment = class _Segment {
|
|
|
2853
3076
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2854
3077
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2855
3078
|
* is in `options`, the second.
|
|
3079
|
+
*
|
|
3080
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
3081
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
3082
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2856
3083
|
*/
|
|
2857
3084
|
async pinAt(at, options) {
|
|
2858
3085
|
this.assertLease();
|
|
2859
|
-
const { generation, fingerprint } = at ?? {};
|
|
2860
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
3086
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
3087
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2861
3088
|
if (unknown.length > 0) {
|
|
2862
|
-
throw new
|
|
2863
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
3089
|
+
throw new ValidationError5(
|
|
3090
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
3091
|
+
);
|
|
3092
|
+
}
|
|
3093
|
+
if (generation === null) {
|
|
3094
|
+
throw new ValidationError5(
|
|
3095
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
3096
|
+
);
|
|
3097
|
+
}
|
|
3098
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
3099
|
+
throw new ValidationError5(
|
|
3100
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2864
3101
|
);
|
|
2865
3102
|
}
|
|
2866
3103
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2867
|
-
throw new
|
|
3104
|
+
throw new ValidationError5(
|
|
2868
3105
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2869
3106
|
);
|
|
2870
3107
|
}
|
|
@@ -2881,9 +3118,11 @@ var Segment = class _Segment {
|
|
|
2881
3118
|
return segmentKey(this.ref);
|
|
2882
3119
|
}
|
|
2883
3120
|
/**
|
|
2884
|
-
* Refuse an `*Into`
|
|
2885
|
-
* segment of another store
|
|
2886
|
-
*
|
|
3121
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
3122
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
3123
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
3124
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
3125
|
+
* first. Called by the three `*Into` verbs only.
|
|
2887
3126
|
*
|
|
2888
3127
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2889
3128
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
@@ -2893,8 +3132,28 @@ var Segment = class _Segment {
|
|
|
2893
3132
|
this.assertSegments(verb, "the destination", [dest]);
|
|
2894
3133
|
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
2895
3134
|
this.assertSegments(verb, "an exclude", excludes);
|
|
3135
|
+
const base = verb.slice(0, -"Into".length);
|
|
3136
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
3137
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
2896
3138
|
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
2897
3139
|
}
|
|
3140
|
+
/**
|
|
3141
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
3142
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
3143
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
3144
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
3145
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
3146
|
+
*/
|
|
3147
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
3148
|
+
const key = dest.key();
|
|
3149
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
3150
|
+
if (role === void 0) return;
|
|
3151
|
+
const { namespace, segment } = dest.ref;
|
|
3152
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
3153
|
+
throw new ValidationError5(
|
|
3154
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
3155
|
+
);
|
|
3156
|
+
}
|
|
2898
3157
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2899
3158
|
guarded(out, handles, result) {
|
|
2900
3159
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
@@ -3006,7 +3265,11 @@ var Segment = class _Segment {
|
|
|
3006
3265
|
engine.iterate(ref, bounds),
|
|
3007
3266
|
() => engine.iterateBatches(ref, bounds)
|
|
3008
3267
|
);
|
|
3009
|
-
return
|
|
3268
|
+
return readsFrom(
|
|
3269
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
3270
|
+
this.origin,
|
|
3271
|
+
[this.key()]
|
|
3272
|
+
);
|
|
3010
3273
|
}
|
|
3011
3274
|
/**
|
|
3012
3275
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -3055,7 +3318,11 @@ var Segment = class _Segment {
|
|
|
3055
3318
|
n,
|
|
3056
3319
|
options == null ? void 0 : rangeOf(options)
|
|
3057
3320
|
);
|
|
3058
|
-
return
|
|
3321
|
+
return readsFrom(
|
|
3322
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
3323
|
+
this.origin,
|
|
3324
|
+
[this.key()]
|
|
3325
|
+
);
|
|
3059
3326
|
}
|
|
3060
3327
|
/**
|
|
3061
3328
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3094,7 +3361,8 @@ var Segment = class _Segment {
|
|
|
3094
3361
|
*
|
|
3095
3362
|
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3096
3363
|
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3097
|
-
* first read, before any request.
|
|
3364
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
3365
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3098
3366
|
*/
|
|
3099
3367
|
intersect(others, options) {
|
|
3100
3368
|
const bad = optionsError(
|
|
@@ -3126,6 +3394,15 @@ var Segment = class _Segment {
|
|
|
3126
3394
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3127
3395
|
*
|
|
3128
3396
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
3397
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
3398
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
3399
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
3400
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
3401
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
3402
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
3403
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
3404
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
3405
|
+
* once, at one instant.
|
|
3129
3406
|
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3130
3407
|
*/
|
|
3131
3408
|
async intersectInto(dest, others, options) {
|
|
@@ -3140,7 +3417,8 @@ var Segment = class _Segment {
|
|
|
3140
3417
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3141
3418
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3142
3419
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3143
|
-
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes
|
|
3420
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
3421
|
+
* empty `others` reads `this` itself.
|
|
3144
3422
|
*/
|
|
3145
3423
|
union(others, options) {
|
|
3146
3424
|
const bad = optionsError(
|
|
@@ -3208,6 +3486,7 @@ var Segment = class _Segment {
|
|
|
3208
3486
|
try {
|
|
3209
3487
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3210
3488
|
const others = segmentList(operands, operandsLabel(verb));
|
|
3489
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3211
3490
|
handles = [this, ...others, ...exclude];
|
|
3212
3491
|
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3213
3492
|
this.assertSegments(verb, "an exclude", exclude);
|
|
@@ -3221,17 +3500,21 @@ var Segment = class _Segment {
|
|
|
3221
3500
|
} catch (err) {
|
|
3222
3501
|
return out.failing(err);
|
|
3223
3502
|
}
|
|
3503
|
+
const keys = handles.map((h) => h.key());
|
|
3504
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3224
3505
|
if (verb === "andNot") {
|
|
3225
3506
|
const refs2 = operands.map((o) => o.ref);
|
|
3226
3507
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3227
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3508
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3228
3509
|
}
|
|
3229
3510
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3230
3511
|
const opts = this.refsIn(options, exclude);
|
|
3231
|
-
return
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3512
|
+
return known(
|
|
3513
|
+
this.guarded(
|
|
3514
|
+
out,
|
|
3515
|
+
handles,
|
|
3516
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3517
|
+
)
|
|
3235
3518
|
);
|
|
3236
3519
|
}
|
|
3237
3520
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
@@ -3285,22 +3568,37 @@ ${USAGE}`,
|
|
|
3285
3568
|
code: 2
|
|
3286
3569
|
};
|
|
3287
3570
|
}
|
|
3571
|
+
var EXPORT_VARIABLES = [
|
|
3572
|
+
"CR_EXPORT_ROOT",
|
|
3573
|
+
"CR_EXPORT_OUT",
|
|
3574
|
+
"CR_EXPORT_FORMAT",
|
|
3575
|
+
"CR_EXPORT_NAMESPACE"
|
|
3576
|
+
];
|
|
3288
3577
|
function parseConfig(env) {
|
|
3578
|
+
const unknown = Object.keys(env).filter(
|
|
3579
|
+
(name) => name.startsWith("CR_EXPORT_") && !EXPORT_VARIABLES.includes(name)
|
|
3580
|
+
);
|
|
3581
|
+
if (unknown.length > 0) {
|
|
3582
|
+
const which = unknown.length > 1 ? `${unknown.join(", ")} are not settings export-segments takes.` : `${unknown[0]} is not a setting export-segments takes.`;
|
|
3583
|
+
throw new ValidationError6(
|
|
3584
|
+
`${which} It takes ${EXPORT_VARIABLES.join(", ")}; run export-segments --help`
|
|
3585
|
+
);
|
|
3586
|
+
}
|
|
3289
3587
|
const root = env.CR_EXPORT_ROOT;
|
|
3290
3588
|
if (root === void 0 || root === "") {
|
|
3291
|
-
throw new
|
|
3589
|
+
throw new ValidationError6(
|
|
3292
3590
|
"CR_EXPORT_ROOT is not set: the directory that holds the store's storage/ and registry/. export-segments also needs CR_EXPORT_OUT; run export-segments --help"
|
|
3293
3591
|
);
|
|
3294
3592
|
}
|
|
3295
3593
|
const out = env.CR_EXPORT_OUT;
|
|
3296
3594
|
if (out === void 0 || out === "") {
|
|
3297
|
-
throw new
|
|
3595
|
+
throw new ValidationError6(
|
|
3298
3596
|
"CR_EXPORT_OUT is not set: the directory to write the dump to. export-segments also needs CR_EXPORT_ROOT; run export-segments --help"
|
|
3299
3597
|
);
|
|
3300
3598
|
}
|
|
3301
3599
|
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3302
3600
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3303
|
-
throw new
|
|
3601
|
+
throw new ValidationError6(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3304
3602
|
}
|
|
3305
3603
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
3306
3604
|
return { root, out, format, namespace };
|
|
@@ -3315,7 +3613,7 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3315
3613
|
}
|
|
3316
3614
|
const found = await lstat(dir);
|
|
3317
3615
|
const refuse = (why) => {
|
|
3318
|
-
throw new
|
|
3616
|
+
throw new ValidationError6(`the export directory is not one this tool made: ${why}`);
|
|
3319
3617
|
};
|
|
3320
3618
|
if (found.isSymbolicLink()) refuse("a namespace directory is a symbolic link");
|
|
3321
3619
|
if (!found.isDirectory()) refuse("a namespace path is not a directory");
|
|
@@ -3330,7 +3628,7 @@ function fsSink(out) {
|
|
|
3330
3628
|
async open(ref, ext) {
|
|
3331
3629
|
const part = namespacePathPart(ref.namespace);
|
|
3332
3630
|
if (part.toLowerCase() === MANIFEST_NAME) {
|
|
3333
|
-
throw new
|
|
3631
|
+
throw new ValidationError6(
|
|
3334
3632
|
`a namespace named ${MANIFEST_NAME} (in any letter case) cannot be exported to a directory: its folder would be the manifest's path`
|
|
3335
3633
|
);
|
|
3336
3634
|
}
|
|
@@ -3375,10 +3673,10 @@ async function requireStoreRoot(root) {
|
|
|
3375
3673
|
try {
|
|
3376
3674
|
entries = await readdir(root);
|
|
3377
3675
|
} catch {
|
|
3378
|
-
throw new
|
|
3676
|
+
throw new ValidationError6("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3379
3677
|
}
|
|
3380
3678
|
if (entries.length === 0) {
|
|
3381
|
-
throw new
|
|
3679
|
+
throw new ValidationError6(
|
|
3382
3680
|
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3383
3681
|
);
|
|
3384
3682
|
}
|
|
@@ -3394,7 +3692,7 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3394
3692
|
const store = new CloudRoaring({ storage });
|
|
3395
3693
|
const priorManifest = await lstat(join(config.out, MANIFEST_NAME)).catch(() => null);
|
|
3396
3694
|
if (priorManifest?.isDirectory() === true) {
|
|
3397
|
-
throw new
|
|
3695
|
+
throw new ValidationError6(
|
|
3398
3696
|
`${join(config.out, MANIFEST_NAME)} is a directory, so the manifest cannot be written there: remove or rename it, or export to a fresh directory`
|
|
3399
3697
|
);
|
|
3400
3698
|
}
|