@cloudbitmaps/roaring 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +23 -17
- package/README.md +13 -4
- package/dist/bin/export-segments.js +471 -173
- package/dist/bin/export-segments.js.map +3 -3
- package/dist/index.d.ts +261 -60
- package/dist/index.js +462 -169
- package/dist/index.js.map +3 -3
- package/dist/roaring-codec.d.ts +23 -22
- package/package.json +4 -3
package/dist/index.js
CHANGED
|
@@ -11,17 +11,20 @@ import {
|
|
|
11
11
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
12
12
|
SegmentEngine,
|
|
13
13
|
LEASE_SKEW_MS,
|
|
14
|
+
BudgetExceededError,
|
|
14
15
|
LeaseExpiredError,
|
|
15
16
|
MAX_LEASE_MS,
|
|
16
17
|
NotFoundError,
|
|
17
18
|
UnsupportedError as UnsupportedError2,
|
|
18
|
-
ValidationError as
|
|
19
|
+
ValidationError as ValidationError5,
|
|
20
|
+
errorKind,
|
|
19
21
|
WriteConflictError,
|
|
20
22
|
checkBudget,
|
|
21
23
|
collectWithinBudget,
|
|
22
24
|
excludingReservedRows,
|
|
23
25
|
dropSegment,
|
|
24
26
|
eraseIdFromSegment,
|
|
27
|
+
findIdUnderTombstone,
|
|
25
28
|
judgeLoad,
|
|
26
29
|
loadSegment,
|
|
27
30
|
loadSegmentChunks,
|
|
@@ -42,7 +45,12 @@ import {
|
|
|
42
45
|
splitId,
|
|
43
46
|
takeLease
|
|
44
47
|
} from "@cloudbitmaps/core";
|
|
45
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
pointerIdOf,
|
|
50
|
+
requireBool,
|
|
51
|
+
requirePositiveInt,
|
|
52
|
+
validateSegmentRef
|
|
53
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
46
54
|
|
|
47
55
|
// src/open-charging-storage.ts
|
|
48
56
|
var OpenChargingStorage = class {
|
|
@@ -58,6 +66,7 @@ var OpenChargingStorage = class {
|
|
|
58
66
|
}
|
|
59
67
|
inner;
|
|
60
68
|
charge;
|
|
69
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
61
70
|
free = /* @__PURE__ */ new Set();
|
|
62
71
|
capabilities() {
|
|
63
72
|
return this.inner.capabilities();
|
|
@@ -70,9 +79,15 @@ var OpenChargingStorage = class {
|
|
|
70
79
|
getRange(key, offset, length) {
|
|
71
80
|
return this.inner.getRange(key, offset, length);
|
|
72
81
|
}
|
|
73
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
84
|
+
* generation is charged at its first open only.
|
|
85
|
+
*/
|
|
74
86
|
async getTail(key, maxBytes) {
|
|
75
|
-
if (!this.free.has(key.generation))
|
|
87
|
+
if (!this.free.has(key.generation)) {
|
|
88
|
+
this.charge();
|
|
89
|
+
this.free.add(key.generation);
|
|
90
|
+
}
|
|
76
91
|
return this.inner.getTail(key, maxBytes);
|
|
77
92
|
}
|
|
78
93
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -86,7 +101,12 @@ var OpenChargingStorage = class {
|
|
|
86
101
|
|
|
87
102
|
// src/index.ts
|
|
88
103
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
89
|
-
import {
|
|
104
|
+
import {
|
|
105
|
+
destroySegment as coreDestroySegment,
|
|
106
|
+
eraseNamespace as coreEraseNamespace
|
|
107
|
+
} from "@cloudbitmaps/core";
|
|
108
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
109
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
90
110
|
|
|
91
111
|
// src/reserved-namespace.ts
|
|
92
112
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -101,7 +121,7 @@ function refuseReservedNamespace(namespace) {
|
|
|
101
121
|
|
|
102
122
|
// src/roaring-codec.ts
|
|
103
123
|
import roaring from "roaring";
|
|
104
|
-
import { IntegrityError as IntegrityError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
124
|
+
import { IntegrityError as IntegrityError2, ValidationError as ValidationError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
105
125
|
|
|
106
126
|
// src/portable/layout.ts
|
|
107
127
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -349,10 +369,6 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
349
369
|
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
350
370
|
}
|
|
351
371
|
}
|
|
352
|
-
/** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
|
|
353
|
-
unwrap() {
|
|
354
|
-
return this.bitmap;
|
|
355
|
-
}
|
|
356
372
|
serialize() {
|
|
357
373
|
return this.bitmap.serialize(SerializationFormat.portable);
|
|
358
374
|
}
|
|
@@ -457,7 +473,7 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
457
473
|
}
|
|
458
474
|
};
|
|
459
475
|
function bitmapAsLoadInput(input) {
|
|
460
|
-
return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
|
|
476
|
+
return input instanceof RoaringBitmap32 || input instanceof ReadonlyRoaringView ? { bitmap: input } : input;
|
|
461
477
|
}
|
|
462
478
|
var roaringCodec = {
|
|
463
479
|
empty: () => SafeBitmap.empty(),
|
|
@@ -465,8 +481,33 @@ var roaringCodec = {
|
|
|
465
481
|
safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
466
482
|
owns: (bitmap) => bitmap instanceof SafeBitmap
|
|
467
483
|
};
|
|
484
|
+
var ReadonlyRoaringView = class {
|
|
485
|
+
#bitmap;
|
|
486
|
+
constructor(bitmap) {
|
|
487
|
+
this.#bitmap = bitmap;
|
|
488
|
+
}
|
|
489
|
+
get size() {
|
|
490
|
+
return this.#bitmap.size;
|
|
491
|
+
}
|
|
492
|
+
has(id) {
|
|
493
|
+
return this.#bitmap.has(id);
|
|
494
|
+
}
|
|
495
|
+
[Symbol.iterator]() {
|
|
496
|
+
return this.#bitmap[Symbol.iterator]();
|
|
497
|
+
}
|
|
498
|
+
/** The portable bytes, for exactly the argument `'portable'`; any other call is a `ValidationError`. */
|
|
499
|
+
serialize(...args) {
|
|
500
|
+
if (args.length !== 1 || args[0] !== "portable") {
|
|
501
|
+
const got = args.length === 0 ? "no argument" : args.map((a) => typeof a === "string" ? `'${a}'` : typeof a).join(", ");
|
|
502
|
+
throw new ValidationError2(
|
|
503
|
+
`a decoded bitmap serializes only as serialize('portable'); got ${got}`
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
return this.#bitmap.serialize();
|
|
507
|
+
}
|
|
508
|
+
};
|
|
468
509
|
function deserializePortable(bytes) {
|
|
469
|
-
return decodeSerialized(bytes, roaringCodec)
|
|
510
|
+
return new ReadonlyRoaringView(decodeSerialized(bytes, roaringCodec));
|
|
470
511
|
}
|
|
471
512
|
|
|
472
513
|
// src/system-clock.ts
|
|
@@ -607,17 +648,18 @@ import {
|
|
|
607
648
|
DEFAULT_RETRY_POLICY,
|
|
608
649
|
NOOP_METRICS,
|
|
609
650
|
RetryingStorageChunkSource,
|
|
610
|
-
ValidationError as
|
|
651
|
+
ValidationError as ValidationError3,
|
|
611
652
|
isStorageBackend
|
|
612
653
|
} from "@cloudbitmaps/core";
|
|
613
654
|
function resolveRetryPolicy(partial) {
|
|
614
|
-
const
|
|
655
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
656
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
615
657
|
return {
|
|
616
|
-
maxAttempts:
|
|
658
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
617
659
|
baseDelayMs,
|
|
618
|
-
maxDelayMs: partial.maxDelayMs
|
|
619
|
-
backoffFactor:
|
|
620
|
-
jitter:
|
|
660
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
661
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
662
|
+
jitter: orDefault("jitter")
|
|
621
663
|
};
|
|
622
664
|
}
|
|
623
665
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -630,7 +672,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
630
672
|
if (options.retry !== false) {
|
|
631
673
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
632
674
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
633
|
-
throw new
|
|
675
|
+
throw new ValidationError3("retry.onRetry must be a function");
|
|
634
676
|
}
|
|
635
677
|
retryOptions = {
|
|
636
678
|
clock,
|
|
@@ -654,13 +696,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
654
696
|
source: resolved.source,
|
|
655
697
|
driver: resolved.driver,
|
|
656
698
|
registry: resolved.registry,
|
|
699
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
657
700
|
retryOptions
|
|
658
701
|
};
|
|
659
702
|
}
|
|
660
703
|
function resolveStorageSource(options, clock) {
|
|
661
704
|
const storage = options.storage;
|
|
662
705
|
if (storage === null || typeof storage !== "object") {
|
|
663
|
-
throw new
|
|
706
|
+
throw new ValidationError3(
|
|
664
707
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
665
708
|
);
|
|
666
709
|
}
|
|
@@ -670,14 +713,14 @@ function resolveStorageSource(options, clock) {
|
|
|
670
713
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
671
714
|
const isBackend = isStorageBackend(storage);
|
|
672
715
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
673
|
-
throw new
|
|
716
|
+
throw new ValidationError3(
|
|
674
717
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
675
718
|
);
|
|
676
719
|
}
|
|
677
720
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
678
721
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
679
722
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
680
|
-
throw new
|
|
723
|
+
throw new ValidationError3(
|
|
681
724
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
682
725
|
);
|
|
683
726
|
}
|
|
@@ -686,11 +729,11 @@ function resolveStorageSource(options, clock) {
|
|
|
686
729
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
687
730
|
if (!storageOk || !registryOk) {
|
|
688
731
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
689
|
-
throw new
|
|
732
|
+
throw new ValidationError3(
|
|
690
733
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
691
734
|
);
|
|
692
735
|
}
|
|
693
|
-
throw new
|
|
736
|
+
throw new ValidationError3(
|
|
694
737
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
695
738
|
);
|
|
696
739
|
}
|
|
@@ -711,18 +754,18 @@ function resolveStorageSource(options, clock) {
|
|
|
711
754
|
};
|
|
712
755
|
}
|
|
713
756
|
if (hasGetChunk && hasPutImmutable) {
|
|
714
|
-
throw new
|
|
757
|
+
throw new ValidationError3(
|
|
715
758
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
716
759
|
);
|
|
717
760
|
}
|
|
718
761
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
719
|
-
throw new
|
|
762
|
+
throw new ValidationError3(
|
|
720
763
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
721
764
|
);
|
|
722
765
|
}
|
|
723
766
|
if (hasGetChunk) {
|
|
724
767
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
725
|
-
throw new
|
|
768
|
+
throw new ValidationError3(
|
|
726
769
|
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
727
770
|
);
|
|
728
771
|
}
|
|
@@ -744,32 +787,32 @@ function resolveStorageSource(options, clock) {
|
|
|
744
787
|
}
|
|
745
788
|
|
|
746
789
|
// src/options.ts
|
|
747
|
-
import { ValidationError as
|
|
790
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core";
|
|
748
791
|
var keysOf = () => (keys) => keys;
|
|
749
792
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
750
793
|
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
751
794
|
if (options === void 0 || options === null) return void 0;
|
|
752
795
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
753
|
-
return new
|
|
796
|
+
return new ValidationError4(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
754
797
|
}
|
|
755
798
|
const bag = options;
|
|
756
799
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
757
800
|
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
758
801
|
if (unknown.length > 0) {
|
|
759
|
-
return new
|
|
802
|
+
return new ValidationError4(
|
|
760
803
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
761
804
|
);
|
|
762
805
|
}
|
|
763
806
|
const audit = bag.audit;
|
|
764
807
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
765
|
-
return new
|
|
808
|
+
return new ValidationError4(
|
|
766
809
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
767
810
|
);
|
|
768
811
|
}
|
|
769
812
|
for (const name of booleans) {
|
|
770
813
|
const v = bag[name];
|
|
771
814
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
772
|
-
return new
|
|
815
|
+
return new ValidationError4(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
773
816
|
}
|
|
774
817
|
}
|
|
775
818
|
return void 0;
|
|
@@ -789,16 +832,12 @@ function checkOptions(options, op, known, booleans, misplaced) {
|
|
|
789
832
|
}
|
|
790
833
|
|
|
791
834
|
// src/index.ts
|
|
835
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
792
836
|
import {
|
|
793
|
-
MemoryStorage,
|
|
794
|
-
LocalFsStorage,
|
|
795
|
-
destroySegment,
|
|
796
|
-
eraseNamespace,
|
|
797
837
|
excludingReservedRows as excludingReservedRows2,
|
|
798
838
|
readRetentionPolicy,
|
|
799
|
-
InProcessKeystore,
|
|
800
839
|
CloudRoaringError,
|
|
801
|
-
ValidationError as
|
|
840
|
+
ValidationError as ValidationError6,
|
|
802
841
|
WriteConflictError as WriteConflictError2,
|
|
803
842
|
IntegrityError as IntegrityError3,
|
|
804
843
|
NotFoundError as NotFoundError2,
|
|
@@ -806,11 +845,11 @@ import {
|
|
|
806
845
|
CapabilityError,
|
|
807
846
|
TransientError,
|
|
808
847
|
KeyUnavailableError,
|
|
809
|
-
BudgetExceededError,
|
|
848
|
+
BudgetExceededError as BudgetExceededError2,
|
|
810
849
|
LeaseExpiredError as LeaseExpiredError2,
|
|
811
850
|
LeaseLimitError,
|
|
812
851
|
StaleOperandError,
|
|
813
|
-
errorKind,
|
|
852
|
+
errorKind as errorKind2,
|
|
814
853
|
CountingMetricsSink,
|
|
815
854
|
RecordingAuditSink
|
|
816
855
|
} from "@cloudbitmaps/core";
|
|
@@ -820,13 +859,25 @@ var SystemRng = class {
|
|
|
820
859
|
return Math.random();
|
|
821
860
|
}
|
|
822
861
|
};
|
|
823
|
-
function
|
|
862
|
+
function withWaits(deps) {
|
|
863
|
+
if (deps === null || typeof deps !== "object") return deps;
|
|
864
|
+
return { ...deps, clock: deps.clock ?? new SystemClock(), rng: deps.rng ?? new SystemRng() };
|
|
865
|
+
}
|
|
866
|
+
function destroySegment(ref, deps, options) {
|
|
867
|
+
return coreDestroySegment(ref, withWaits(deps), options);
|
|
868
|
+
}
|
|
869
|
+
function eraseNamespace(namespace, deps, options) {
|
|
870
|
+
return coreEraseNamespace(namespace, withWaits(deps), options);
|
|
871
|
+
}
|
|
872
|
+
function validateRef(ref, verb) {
|
|
824
873
|
if (typeof ref !== "object" || ref === null) {
|
|
825
|
-
throw new
|
|
874
|
+
throw new ValidationError5(
|
|
826
875
|
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
827
876
|
);
|
|
828
877
|
}
|
|
878
|
+
refuseUnknownRefKeys(ref, verb);
|
|
829
879
|
validateSegmentRef(ref);
|
|
880
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
830
881
|
}
|
|
831
882
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
832
883
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
@@ -841,15 +892,20 @@ function validateConcurrency(concurrency) {
|
|
|
841
892
|
function requireScope(options, op) {
|
|
842
893
|
if (options.namespace !== void 0) {
|
|
843
894
|
if (typeof options.namespace !== "string") {
|
|
844
|
-
throw new
|
|
895
|
+
throw new ValidationError5(
|
|
845
896
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
846
897
|
);
|
|
847
898
|
}
|
|
848
899
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
849
900
|
}
|
|
850
901
|
refuseReservedNamespace(options.namespace);
|
|
902
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
903
|
+
throw new ValidationError5(
|
|
904
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
851
907
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
852
|
-
throw new
|
|
908
|
+
throw new ValidationError5(
|
|
853
909
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
854
910
|
);
|
|
855
911
|
}
|
|
@@ -896,7 +952,7 @@ var CALL_KEYS = {
|
|
|
896
952
|
"metadata",
|
|
897
953
|
"keep"
|
|
898
954
|
]),
|
|
899
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
955
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
900
956
|
segments: keysOf()(["namespace"]),
|
|
901
957
|
subjectReport: keysOf()([
|
|
902
958
|
"namespace",
|
|
@@ -957,16 +1013,16 @@ var originOf;
|
|
|
957
1013
|
function assertOneStore(verb, role, h, origin) {
|
|
958
1014
|
if (h instanceof Segment) {
|
|
959
1015
|
if (originOf(h) === origin) return;
|
|
960
|
-
throw new
|
|
1016
|
+
throw new ValidationError5(
|
|
961
1017
|
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
962
1018
|
);
|
|
963
1019
|
}
|
|
964
1020
|
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
965
|
-
throw new
|
|
1021
|
+
throw new ValidationError5(
|
|
966
1022
|
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
967
1023
|
);
|
|
968
1024
|
}
|
|
969
|
-
throw new
|
|
1025
|
+
throw new ValidationError5(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
970
1026
|
}
|
|
971
1027
|
var MATERIALIZE_MANY_KEYS = [
|
|
972
1028
|
"operands",
|
|
@@ -1035,6 +1091,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1035
1091
|
// no underlying driver to write through).
|
|
1036
1092
|
storageDriver;
|
|
1037
1093
|
registry;
|
|
1094
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
1095
|
+
backend;
|
|
1038
1096
|
keystore;
|
|
1039
1097
|
requireEncryption;
|
|
1040
1098
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1054,7 +1112,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1054
1112
|
*/
|
|
1055
1113
|
static rejectUnknownOptions(options) {
|
|
1056
1114
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1057
|
-
throw new
|
|
1115
|
+
throw new ValidationError5(
|
|
1058
1116
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1059
1117
|
);
|
|
1060
1118
|
}
|
|
@@ -1072,7 +1130,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1072
1130
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1073
1131
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1074
1132
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1075
|
-
throw new
|
|
1133
|
+
throw new ValidationError5(
|
|
1076
1134
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1077
1135
|
);
|
|
1078
1136
|
}
|
|
@@ -1082,7 +1140,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1082
1140
|
if (unknown.length === 0) return;
|
|
1083
1141
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1084
1142
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1085
|
-
throw new
|
|
1143
|
+
throw new ValidationError5(
|
|
1086
1144
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1087
1145
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1088
1146
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1100,23 +1158,51 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1100
1158
|
];
|
|
1101
1159
|
const entropy = options.seams?.entropy;
|
|
1102
1160
|
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1103
|
-
throw new
|
|
1161
|
+
throw new ValidationError5(
|
|
1104
1162
|
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1105
1163
|
);
|
|
1106
1164
|
}
|
|
1107
1165
|
for (const [seam, name, needs] of seams) {
|
|
1108
|
-
if (seam === void 0
|
|
1109
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1110
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1111
|
-
throw new
|
|
1166
|
+
if (seam === void 0) continue;
|
|
1167
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
1168
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
1169
|
+
throw new ValidationError5(
|
|
1112
1170
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1113
1171
|
);
|
|
1114
1172
|
}
|
|
1115
1173
|
}
|
|
1116
1174
|
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
1177
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
1178
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
1179
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
1180
|
+
*/
|
|
1181
|
+
static rejectBadValues(options) {
|
|
1182
|
+
const cache = options.cache;
|
|
1183
|
+
if (cache !== void 0) {
|
|
1184
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
1185
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
1186
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
1187
|
+
const ttl = cache.ttlMs;
|
|
1188
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
1189
|
+
throw new ValidationError5(
|
|
1190
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
1191
|
+
);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
1198
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
1199
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
1200
|
+
* calls, throws `ValidationError`.
|
|
1201
|
+
*/
|
|
1117
1202
|
constructor(options) {
|
|
1118
1203
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1119
1204
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1205
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1120
1206
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1121
1207
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1122
1208
|
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
@@ -1147,6 +1233,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1147
1233
|
this.metrics = metrics;
|
|
1148
1234
|
this.storageDriver = wired.driver;
|
|
1149
1235
|
this.registry = wired.registry;
|
|
1236
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
1237
|
+
this.backend = wired.backend;
|
|
1150
1238
|
this.keystore = options.encryption?.keystore;
|
|
1151
1239
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1152
1240
|
}
|
|
@@ -1190,17 +1278,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1190
1278
|
*/
|
|
1191
1279
|
segment(name, options) {
|
|
1192
1280
|
if (typeof name === "object" && name !== null) {
|
|
1193
|
-
throw new
|
|
1281
|
+
throw new ValidationError5(
|
|
1194
1282
|
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1195
1283
|
);
|
|
1196
1284
|
}
|
|
1197
1285
|
if (options !== void 0 && options !== null) {
|
|
1198
1286
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1199
|
-
throw new
|
|
1287
|
+
throw new ValidationError5("segment: options must be an object such as { namespace }");
|
|
1200
1288
|
}
|
|
1201
1289
|
const proto = Object.getPrototypeOf(options);
|
|
1202
1290
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1203
|
-
throw new
|
|
1291
|
+
throw new ValidationError5(
|
|
1204
1292
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1205
1293
|
);
|
|
1206
1294
|
}
|
|
@@ -1208,7 +1296,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1208
1296
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1209
1297
|
);
|
|
1210
1298
|
if (unknown.length > 0) {
|
|
1211
|
-
throw new
|
|
1299
|
+
throw new ValidationError5(
|
|
1212
1300
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1213
1301
|
);
|
|
1214
1302
|
}
|
|
@@ -1290,7 +1378,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1290
1378
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1291
1379
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1292
1380
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1293
|
-
keep: options?.keep
|
|
1381
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1294
1382
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1295
1383
|
};
|
|
1296
1384
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1318,20 +1406,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1318
1406
|
/** The one body of the forms above. */
|
|
1319
1407
|
async materializeMany(options) {
|
|
1320
1408
|
if (typeof options !== "object" || options === null) {
|
|
1321
|
-
throw new
|
|
1409
|
+
throw new ValidationError5(
|
|
1322
1410
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1323
1411
|
);
|
|
1324
1412
|
}
|
|
1325
1413
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1326
1414
|
const unknown = unknownManyKeys(options);
|
|
1327
1415
|
if (unknown.length > 0) {
|
|
1328
|
-
throw new
|
|
1416
|
+
throw new ValidationError5(
|
|
1329
1417
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1330
1418
|
);
|
|
1331
1419
|
}
|
|
1332
1420
|
const dryRunIn = options.dryRun;
|
|
1333
1421
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1334
|
-
throw new
|
|
1422
|
+
throw new ValidationError5("materializeMany: dryRun must be a boolean");
|
|
1335
1423
|
}
|
|
1336
1424
|
const dryRun = dryRunIn === true;
|
|
1337
1425
|
const operandsIn = options.operands;
|
|
@@ -1343,7 +1431,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1343
1431
|
const feedIn = options.feed;
|
|
1344
1432
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1345
1433
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1346
|
-
throw new
|
|
1434
|
+
throw new ValidationError5(
|
|
1347
1435
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1348
1436
|
);
|
|
1349
1437
|
}
|
|
@@ -1353,15 +1441,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1353
1441
|
const budgetOption = options.budget;
|
|
1354
1442
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1355
1443
|
if (typeof pinOption !== "boolean")
|
|
1356
|
-
throw new
|
|
1444
|
+
throw new ValidationError5("materializeMany: pin must be a boolean");
|
|
1357
1445
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1358
|
-
throw new
|
|
1446
|
+
throw new ValidationError5("materializeMany: allowAbsentOperands must be a boolean");
|
|
1359
1447
|
}
|
|
1360
1448
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1361
|
-
throw new
|
|
1449
|
+
throw new ValidationError5("materializeMany: operands must be an object of name to segment");
|
|
1362
1450
|
}
|
|
1363
1451
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1364
|
-
throw new
|
|
1452
|
+
throw new ValidationError5("materializeMany: outputs must be a non-empty array");
|
|
1365
1453
|
}
|
|
1366
1454
|
const names = Object.keys(operandsIn);
|
|
1367
1455
|
const handles = /* @__PURE__ */ new Map();
|
|
@@ -1372,14 +1460,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1372
1460
|
}
|
|
1373
1461
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1374
1462
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1375
|
-
throw new
|
|
1463
|
+
throw new ValidationError5(
|
|
1376
1464
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1377
1465
|
);
|
|
1378
1466
|
}
|
|
1379
1467
|
if (mayBeEmptyIn.length > 0) {
|
|
1380
1468
|
const first = mayBeEmptyIn[0];
|
|
1381
1469
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1382
|
-
throw new
|
|
1470
|
+
throw new ValidationError5(
|
|
1383
1471
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1384
1472
|
);
|
|
1385
1473
|
}
|
|
@@ -1389,13 +1477,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1389
1477
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1390
1478
|
outputsIn.forEach((raw, i) => {
|
|
1391
1479
|
if (typeof raw !== "object" || raw === null) {
|
|
1392
|
-
throw new
|
|
1480
|
+
throw new ValidationError5(`outputs[${i}] must be an object`);
|
|
1393
1481
|
}
|
|
1394
1482
|
const bad = Object.keys(raw).filter(
|
|
1395
1483
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1396
1484
|
);
|
|
1397
1485
|
if (bad.length > 0) {
|
|
1398
|
-
throw new
|
|
1486
|
+
throw new ValidationError5(
|
|
1399
1487
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1400
1488
|
);
|
|
1401
1489
|
}
|
|
@@ -1403,12 +1491,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1403
1491
|
const dest = spec.dest;
|
|
1404
1492
|
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1405
1493
|
if (operandKeys.has(dest.key())) {
|
|
1406
|
-
throw new
|
|
1494
|
+
throw new ValidationError5(
|
|
1407
1495
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1408
1496
|
);
|
|
1409
1497
|
}
|
|
1410
1498
|
if (destKeys.has(dest.key())) {
|
|
1411
|
-
throw new
|
|
1499
|
+
throw new ValidationError5(
|
|
1412
1500
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1413
1501
|
);
|
|
1414
1502
|
}
|
|
@@ -1481,13 +1569,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1481
1569
|
held.set(key, { at, handle });
|
|
1482
1570
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1483
1571
|
if (!samePin(prior.at, at)) {
|
|
1484
|
-
throw new
|
|
1572
|
+
throw new ValidationError5(
|
|
1485
1573
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1486
1574
|
);
|
|
1487
1575
|
}
|
|
1488
1576
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1489
1577
|
if (!pinOption) {
|
|
1490
|
-
throw new
|
|
1578
|
+
throw new ValidationError5(
|
|
1491
1579
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1492
1580
|
);
|
|
1493
1581
|
}
|
|
@@ -1575,7 +1663,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1575
1663
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1576
1664
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1577
1665
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1578
|
-
throw new
|
|
1666
|
+
throw new ValidationError5(
|
|
1579
1667
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1580
1668
|
);
|
|
1581
1669
|
}
|
|
@@ -1625,13 +1713,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1625
1713
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1626
1714
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1627
1715
|
*
|
|
1716
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1717
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1718
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1719
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1720
|
+
*
|
|
1628
1721
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1629
1722
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1630
1723
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1631
1724
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1632
1725
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
1726
|
+
*
|
|
1727
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
1728
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
1729
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
1730
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
1731
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
1732
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
1733
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
1734
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
1735
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
1736
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
1737
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
1738
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
1739
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1633
1740
|
*/
|
|
1634
|
-
async subjectReport(id, options
|
|
1741
|
+
async subjectReport(id, options) {
|
|
1635
1742
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1636
1743
|
options ??= {};
|
|
1637
1744
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1646,26 +1753,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1646
1753
|
budget,
|
|
1647
1754
|
"subjectReport"
|
|
1648
1755
|
);
|
|
1756
|
+
let opens = 0;
|
|
1757
|
+
const charge = () => {
|
|
1758
|
+
opens += 1;
|
|
1759
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
1760
|
+
throw new BudgetExceededError(
|
|
1761
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
1762
|
+
{
|
|
1763
|
+
option: "budget.maxRequests",
|
|
1764
|
+
limit: budget.maxRequests,
|
|
1765
|
+
spent: budget.maxRequests + 1
|
|
1766
|
+
}
|
|
1767
|
+
);
|
|
1768
|
+
}
|
|
1769
|
+
};
|
|
1649
1770
|
const membership = await mapWithConcurrency(
|
|
1650
1771
|
recs,
|
|
1651
1772
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1652
1773
|
async (rec) => {
|
|
1653
|
-
if (rec.status === "destroyed") return null;
|
|
1654
1774
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1775
|
+
if (rec.status === "destroyed") {
|
|
1776
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
1777
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
1778
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
1779
|
+
return held === null ? null : { tombstone: true, ref };
|
|
1780
|
+
}
|
|
1655
1781
|
if (this.crbmSource !== void 0) {
|
|
1656
1782
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1657
|
-
const
|
|
1658
|
-
if (rowVersionOfLive(held) !==
|
|
1783
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1784
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1659
1785
|
}
|
|
1660
|
-
return await this.engine.has(ref, id) ? {
|
|
1786
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1661
1787
|
}
|
|
1662
1788
|
);
|
|
1663
|
-
const
|
|
1664
|
-
|
|
1789
|
+
const listed = membership.filter((m) => m !== null);
|
|
1790
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
1791
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
1792
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1665
1793
|
}
|
|
1666
1794
|
/**
|
|
1667
1795
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1668
|
-
* from the bucket on return
|
|
1796
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
1797
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
1798
|
+
*
|
|
1799
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1800
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1801
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1802
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1669
1803
|
*
|
|
1670
1804
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1671
1805
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1730,15 +1864,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1730
1864
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1731
1865
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1732
1866
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1733
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1734
|
-
*
|
|
1867
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
1868
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
1869
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
1870
|
+
* that exhausts the budget is
|
|
1735
1871
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1736
1872
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1737
1873
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1738
1874
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1739
1875
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1740
1876
|
*/
|
|
1741
|
-
async eraseSubject(id, options
|
|
1877
|
+
async eraseSubject(id, options) {
|
|
1742
1878
|
this.erasureEpoch++;
|
|
1743
1879
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1744
1880
|
options ??= {};
|
|
@@ -1794,7 +1930,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1794
1930
|
segment: rec.segment,
|
|
1795
1931
|
namespace: rec.namespace,
|
|
1796
1932
|
erased: false,
|
|
1797
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
1933
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
1934
|
+
// the class lets a note be matched to the guide's table
|
|
1798
1935
|
};
|
|
1799
1936
|
} finally {
|
|
1800
1937
|
try {
|
|
@@ -1815,11 +1952,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1815
1952
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1816
1953
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1817
1954
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1955
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1818
1956
|
*
|
|
1819
1957
|
* Needs a backend.
|
|
1820
1958
|
*/
|
|
1821
1959
|
async generations(ref) {
|
|
1822
|
-
validateRef(ref);
|
|
1960
|
+
ref = validateRef(ref, "generations");
|
|
1823
1961
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1824
1962
|
}
|
|
1825
1963
|
/**
|
|
@@ -1849,7 +1987,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1849
1987
|
* ```
|
|
1850
1988
|
*/
|
|
1851
1989
|
async exists(ref) {
|
|
1852
|
-
validateRef(ref);
|
|
1990
|
+
ref = validateRef(ref, "exists");
|
|
1853
1991
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1854
1992
|
}
|
|
1855
1993
|
/**
|
|
@@ -1906,8 +2044,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1906
2044
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1907
2045
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1908
2046
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1909
|
-
* crypto-shredded
|
|
1910
|
-
*
|
|
2047
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
2048
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
2049
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1911
2050
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1912
2051
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1913
2052
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1918,8 +2057,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1918
2057
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1919
2058
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1920
2059
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1921
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1922
|
-
*
|
|
2060
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
2061
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
2062
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
2063
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1923
2064
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1924
2065
|
*
|
|
1925
2066
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1949,7 +2090,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1949
2090
|
* Needs a backend.
|
|
1950
2091
|
*/
|
|
1951
2092
|
async rollback(ref, toGeneration, options = {}) {
|
|
1952
|
-
validateRef(ref);
|
|
2093
|
+
ref = validateRef(ref, "rollback");
|
|
1953
2094
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1954
2095
|
options ??= {};
|
|
1955
2096
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1966,8 +2107,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1966
2107
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1967
2108
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1968
2109
|
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. A bare
|
|
1969
|
-
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A
|
|
1970
|
-
* `ValidationError`, since each byte would load as an id:
|
|
2110
|
+
* `RoaringBitmap32`, or a bitmap `deserializePortable` returned, passed as ids loads as `{ bitmap }`. A
|
|
2111
|
+
* `Uint8Array` or `Buffer` passed as ids is refused with `ValidationError`, since each byte would load as an id:
|
|
2112
|
+
* pass bytes as `{ serialized }`.
|
|
2113
|
+
*
|
|
2114
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
2115
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
2116
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
2117
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1971
2118
|
*
|
|
1972
2119
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1973
2120
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -2026,7 +2173,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2026
2173
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
2027
2174
|
* what the pass deleted by name, and one name may have been gone already.
|
|
2028
2175
|
*
|
|
2029
|
-
*
|
|
2176
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
2177
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
2178
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
2179
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
2180
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
2181
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
2182
|
+
*
|
|
2183
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
2030
2184
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
2031
2185
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
2032
2186
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2049,8 +2203,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2049
2203
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2050
2204
|
*/
|
|
2051
2205
|
async load(ref, input, options = {}) {
|
|
2052
|
-
validateRef(ref);
|
|
2053
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
2206
|
+
ref = validateRef(ref, "load");
|
|
2207
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
2208
|
+
if (typeof input === "object" && input !== null) {
|
|
2209
|
+
const reads = STREAM_SOURCES.get(input);
|
|
2210
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
2211
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
2212
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
2213
|
+
throw new ValidationError5(
|
|
2214
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
2215
|
+
);
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2054
2218
|
options ??= {};
|
|
2055
2219
|
const deps = this.lifecycleDeps("load");
|
|
2056
2220
|
try {
|
|
@@ -2060,11 +2224,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2060
2224
|
}
|
|
2061
2225
|
}
|
|
2062
2226
|
/**
|
|
2063
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
2227
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
2228
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2064
2229
|
*
|
|
2065
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2066
|
-
*
|
|
2067
|
-
*
|
|
2230
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
2231
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
2232
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
2233
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
2234
|
+
* encrypted segment it does both.
|
|
2068
2235
|
*
|
|
2069
2236
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2070
2237
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2105,9 +2272,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2105
2272
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2106
2273
|
*/
|
|
2107
2274
|
async dropSegment(ref, options) {
|
|
2108
|
-
validateRef(ref);
|
|
2275
|
+
ref = validateRef(ref, "dropSegment");
|
|
2109
2276
|
if (options === void 0 || options === null) {
|
|
2110
|
-
throw new
|
|
2277
|
+
throw new ValidationError5("dropSegment: options { confirmSegment } are required");
|
|
2111
2278
|
}
|
|
2112
2279
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2113
2280
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2122,8 +2289,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2122
2289
|
}
|
|
2123
2290
|
}
|
|
2124
2291
|
/**
|
|
2125
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2126
|
-
*
|
|
2292
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
2293
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
2294
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2127
2295
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2128
2296
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2129
2297
|
*
|
|
@@ -2145,10 +2313,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2145
2313
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2146
2314
|
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2147
2315
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2148
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
2316
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2149
2317
|
*/
|
|
2150
2318
|
async setRetention(ref, policy) {
|
|
2151
|
-
validateRef(ref);
|
|
2319
|
+
ref = validateRef(ref, "setRetention");
|
|
2152
2320
|
return setSegmentRetention(
|
|
2153
2321
|
ref,
|
|
2154
2322
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2162,16 +2330,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2162
2330
|
* "never expires" on a segment someone believes is expiring.
|
|
2163
2331
|
*/
|
|
2164
2332
|
async getRetention(ref) {
|
|
2165
|
-
validateRef(ref);
|
|
2333
|
+
ref = validateRef(ref, "getRetention");
|
|
2166
2334
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2167
2335
|
}
|
|
2168
2336
|
/**
|
|
2169
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2170
|
-
* when there was none
|
|
2337
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
2338
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
2339
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
2340
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2171
2341
|
* to the setter is how a typo becomes a deletion.
|
|
2172
2342
|
*/
|
|
2173
2343
|
async clearRetention(ref) {
|
|
2174
|
-
validateRef(ref);
|
|
2344
|
+
ref = validateRef(ref, "clearRetention");
|
|
2175
2345
|
return clearSegmentRetention(ref, {
|
|
2176
2346
|
registry: this.requireRegistry("clearRetention"),
|
|
2177
2347
|
clock: this.clock,
|
|
@@ -2196,20 +2366,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2196
2366
|
*
|
|
2197
2367
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2198
2368
|
* for (const e of swept.entries) {
|
|
2199
|
-
*
|
|
2369
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
2370
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2200
2371
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2201
2372
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2202
2373
|
* }
|
|
2203
2374
|
* }
|
|
2204
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
2375
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2205
2376
|
* ```
|
|
2206
2377
|
*
|
|
2207
2378
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2208
2379
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2209
2380
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2210
|
-
* `limit` (default 100) cut the pass short and more
|
|
2211
|
-
*
|
|
2212
|
-
*
|
|
2381
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
2382
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
2383
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
2384
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2213
2385
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2214
2386
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2215
2387
|
*
|
|
@@ -2219,9 +2391,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2219
2391
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2220
2392
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2221
2393
|
*
|
|
2222
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
2394
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2223
2395
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2224
|
-
* first, since nothing else
|
|
2396
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
2397
|
+
* only
|
|
2398
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2225
2399
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2226
2400
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2227
2401
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2242,7 +2416,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2242
2416
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2243
2417
|
const result = await retireExpired(
|
|
2244
2418
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2245
|
-
{ ...opts, now: opts.now
|
|
2419
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2246
2420
|
);
|
|
2247
2421
|
if (opts.dryRun !== true) {
|
|
2248
2422
|
for (const entry of result.entries) {
|
|
@@ -2289,7 +2463,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2289
2463
|
* ```
|
|
2290
2464
|
*/
|
|
2291
2465
|
invalidate(ref) {
|
|
2292
|
-
validateRef(ref);
|
|
2466
|
+
ref = validateRef(ref, "invalidate");
|
|
2293
2467
|
refuseReservedNamespace(ref.namespace);
|
|
2294
2468
|
this.engine.invalidate(ref);
|
|
2295
2469
|
}
|
|
@@ -2349,11 +2523,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2349
2523
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2350
2524
|
throw err;
|
|
2351
2525
|
}
|
|
2352
|
-
const pinnedAt = {
|
|
2353
|
-
generation: at?.generation ?? null,
|
|
2354
|
-
version: at?.version ?? null,
|
|
2355
|
-
fingerprint: at?.fingerprint ?? null
|
|
2356
|
-
};
|
|
2526
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2357
2527
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2358
2528
|
return makeSegment({
|
|
2359
2529
|
engine: this.engineWithPins(pins),
|
|
@@ -2423,8 +2593,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2423
2593
|
if (held.has(key)) {
|
|
2424
2594
|
const before = held.get(key);
|
|
2425
2595
|
if (!samePin(before, at)) {
|
|
2426
|
-
throw new
|
|
2427
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
2596
|
+
throw new ValidationError5(
|
|
2597
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2428
2598
|
);
|
|
2429
2599
|
}
|
|
2430
2600
|
} else {
|
|
@@ -2455,8 +2625,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2455
2625
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2456
2626
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2457
2627
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2458
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2459
|
-
*
|
|
2628
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
2629
|
+
* dropped) are skipped. Pair
|
|
2630
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2460
2631
|
*
|
|
2461
2632
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2462
2633
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2489,6 +2660,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2489
2660
|
options
|
|
2490
2661
|
);
|
|
2491
2662
|
}
|
|
2663
|
+
/**
|
|
2664
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
2665
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
2666
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
2667
|
+
*
|
|
2668
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
2669
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
2670
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
2671
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
2672
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
2673
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
2674
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
2675
|
+
*
|
|
2676
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
2677
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
2678
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
2679
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
2680
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
2681
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
2682
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
2683
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
2684
|
+
* timing.
|
|
2685
|
+
*
|
|
2686
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
2687
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
2688
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
2689
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
2690
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
2691
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
2692
|
+
*
|
|
2693
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
2694
|
+
*/
|
|
2695
|
+
async preflight() {
|
|
2696
|
+
const deps = this.lifecycleDeps("preflight");
|
|
2697
|
+
const backend = this.backend;
|
|
2698
|
+
let checkLifecycle;
|
|
2699
|
+
try {
|
|
2700
|
+
const method = backend?.checkLifecycle;
|
|
2701
|
+
if (typeof method === "function")
|
|
2702
|
+
checkLifecycle = method.bind(backend);
|
|
2703
|
+
} catch {
|
|
2704
|
+
}
|
|
2705
|
+
return runPreflight({
|
|
2706
|
+
storage: deps.storage,
|
|
2707
|
+
registry: deps.registry,
|
|
2708
|
+
clock: this.clock,
|
|
2709
|
+
entropy: this.entropy,
|
|
2710
|
+
checkLifecycle
|
|
2711
|
+
});
|
|
2712
|
+
}
|
|
2492
2713
|
/**
|
|
2493
2714
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2494
2715
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2497,8 +2718,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2497
2718
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2498
2719
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2499
2720
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2500
|
-
*
|
|
2501
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
2721
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
2722
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
2723
|
+
* check `failed`. The `export-segments`
|
|
2502
2724
|
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2503
2725
|
*/
|
|
2504
2726
|
async exportSegments(sink, options = {}) {
|
|
@@ -2533,12 +2755,23 @@ var readOptions = (options) => ({
|
|
|
2533
2755
|
});
|
|
2534
2756
|
var NO_SEGMENTS = [];
|
|
2535
2757
|
function segmentList(list, what) {
|
|
2536
|
-
if (!Array.isArray(list)) throw new
|
|
2758
|
+
if (!Array.isArray(list)) throw new ValidationError5(`${what} must be an array of segments`);
|
|
2537
2759
|
return list;
|
|
2538
2760
|
}
|
|
2761
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
2762
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
2763
|
+
function readsFrom(stream, origin, keys) {
|
|
2764
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
2765
|
+
return stream;
|
|
2766
|
+
}
|
|
2539
2767
|
function operandsLabel(verb) {
|
|
2540
2768
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2541
2769
|
}
|
|
2770
|
+
function refuseNoOperands(verb) {
|
|
2771
|
+
throw new ValidationError5(
|
|
2772
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
2773
|
+
);
|
|
2774
|
+
}
|
|
2542
2775
|
var AS_IDS = {
|
|
2543
2776
|
failing,
|
|
2544
2777
|
guard: guardIds,
|
|
@@ -2555,9 +2788,9 @@ var CombineChunks = class {
|
|
|
2555
2788
|
var AS_CHUNKS = {
|
|
2556
2789
|
guard: guardItems,
|
|
2557
2790
|
failing: (err) => refusing(err)(),
|
|
2558
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2559
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2560
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
2791
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
2792
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
2793
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2561
2794
|
};
|
|
2562
2795
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2563
2796
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2616,7 +2849,7 @@ var Segment = class _Segment {
|
|
|
2616
2849
|
*/
|
|
2617
2850
|
constructor(parts) {
|
|
2618
2851
|
if (!minting) {
|
|
2619
|
-
throw new
|
|
2852
|
+
throw new ValidationError5(
|
|
2620
2853
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2621
2854
|
);
|
|
2622
2855
|
}
|
|
@@ -2655,7 +2888,7 @@ var Segment = class _Segment {
|
|
|
2655
2888
|
*
|
|
2656
2889
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2657
2890
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2658
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
2891
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2659
2892
|
*
|
|
2660
2893
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2661
2894
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2730,11 +2963,11 @@ var Segment = class _Segment {
|
|
|
2730
2963
|
leaseUntilOf(op, options) {
|
|
2731
2964
|
if (options === void 0 || options === null) return void 0;
|
|
2732
2965
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2733
|
-
throw new
|
|
2966
|
+
throw new ValidationError5(`${op}: options must be an object such as { leaseUntil }`);
|
|
2734
2967
|
}
|
|
2735
2968
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2736
2969
|
if (unknown.length > 0) {
|
|
2737
|
-
throw new
|
|
2970
|
+
throw new ValidationError5(
|
|
2738
2971
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2739
2972
|
);
|
|
2740
2973
|
}
|
|
@@ -2742,17 +2975,17 @@ var Segment = class _Segment {
|
|
|
2742
2975
|
if (until === void 0) return void 0;
|
|
2743
2976
|
const now = this.clock.now();
|
|
2744
2977
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2745
|
-
throw new
|
|
2978
|
+
throw new ValidationError5(
|
|
2746
2979
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2747
2980
|
);
|
|
2748
2981
|
}
|
|
2749
2982
|
if (until <= now) {
|
|
2750
|
-
throw new
|
|
2983
|
+
throw new ValidationError5(
|
|
2751
2984
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2752
2985
|
);
|
|
2753
2986
|
}
|
|
2754
2987
|
if (until > now + MAX_LEASE_MS) {
|
|
2755
|
-
throw new
|
|
2988
|
+
throw new ValidationError5(
|
|
2756
2989
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2757
2990
|
);
|
|
2758
2991
|
}
|
|
@@ -2830,10 +3063,10 @@ var Segment = class _Segment {
|
|
|
2830
3063
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2831
3064
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2832
3065
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2833
|
-
* than `generation` and `
|
|
3066
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2834
3067
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2835
3068
|
*
|
|
2836
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
3069
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2837
3070
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2838
3071
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2839
3072
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2846,18 +3079,32 @@ var Segment = class _Segment {
|
|
|
2846
3079
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2847
3080
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2848
3081
|
* is in `options`, the second.
|
|
3082
|
+
*
|
|
3083
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
3084
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
3085
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2849
3086
|
*/
|
|
2850
3087
|
async pinAt(at, options) {
|
|
2851
3088
|
this.assertLease();
|
|
2852
|
-
const { generation, fingerprint } = at ?? {};
|
|
2853
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
3089
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
3090
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2854
3091
|
if (unknown.length > 0) {
|
|
2855
|
-
throw new
|
|
2856
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
3092
|
+
throw new ValidationError5(
|
|
3093
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
3094
|
+
);
|
|
3095
|
+
}
|
|
3096
|
+
if (generation === null) {
|
|
3097
|
+
throw new ValidationError5(
|
|
3098
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
3099
|
+
);
|
|
3100
|
+
}
|
|
3101
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
3102
|
+
throw new ValidationError5(
|
|
3103
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2857
3104
|
);
|
|
2858
3105
|
}
|
|
2859
3106
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2860
|
-
throw new
|
|
3107
|
+
throw new ValidationError5(
|
|
2861
3108
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2862
3109
|
);
|
|
2863
3110
|
}
|
|
@@ -2874,9 +3121,11 @@ var Segment = class _Segment {
|
|
|
2874
3121
|
return segmentKey(this.ref);
|
|
2875
3122
|
}
|
|
2876
3123
|
/**
|
|
2877
|
-
* Refuse an `*Into`
|
|
2878
|
-
* segment of another store
|
|
2879
|
-
*
|
|
3124
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
3125
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
3126
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
3127
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
3128
|
+
* first. Called by the three `*Into` verbs only.
|
|
2880
3129
|
*
|
|
2881
3130
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2882
3131
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
@@ -2886,8 +3135,28 @@ var Segment = class _Segment {
|
|
|
2886
3135
|
this.assertSegments(verb, "the destination", [dest]);
|
|
2887
3136
|
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
2888
3137
|
this.assertSegments(verb, "an exclude", excludes);
|
|
3138
|
+
const base = verb.slice(0, -"Into".length);
|
|
3139
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
3140
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
2889
3141
|
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
2890
3142
|
}
|
|
3143
|
+
/**
|
|
3144
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
3145
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
3146
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
3147
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
3148
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
3149
|
+
*/
|
|
3150
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
3151
|
+
const key = dest.key();
|
|
3152
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
3153
|
+
if (role === void 0) return;
|
|
3154
|
+
const { namespace, segment } = dest.ref;
|
|
3155
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
3156
|
+
throw new ValidationError5(
|
|
3157
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
3158
|
+
);
|
|
3159
|
+
}
|
|
2891
3160
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2892
3161
|
guarded(out, handles, result) {
|
|
2893
3162
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
@@ -2999,7 +3268,11 @@ var Segment = class _Segment {
|
|
|
2999
3268
|
engine.iterate(ref, bounds),
|
|
3000
3269
|
() => engine.iterateBatches(ref, bounds)
|
|
3001
3270
|
);
|
|
3002
|
-
return
|
|
3271
|
+
return readsFrom(
|
|
3272
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
3273
|
+
this.origin,
|
|
3274
|
+
[this.key()]
|
|
3275
|
+
);
|
|
3003
3276
|
}
|
|
3004
3277
|
/**
|
|
3005
3278
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -3048,7 +3321,11 @@ var Segment = class _Segment {
|
|
|
3048
3321
|
n,
|
|
3049
3322
|
options == null ? void 0 : rangeOf(options)
|
|
3050
3323
|
);
|
|
3051
|
-
return
|
|
3324
|
+
return readsFrom(
|
|
3325
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
3326
|
+
this.origin,
|
|
3327
|
+
[this.key()]
|
|
3328
|
+
);
|
|
3052
3329
|
}
|
|
3053
3330
|
/**
|
|
3054
3331
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3087,7 +3364,8 @@ var Segment = class _Segment {
|
|
|
3087
3364
|
*
|
|
3088
3365
|
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3089
3366
|
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3090
|
-
* first read, before any request.
|
|
3367
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
3368
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3091
3369
|
*/
|
|
3092
3370
|
intersect(others, options) {
|
|
3093
3371
|
const bad = optionsError(
|
|
@@ -3119,6 +3397,15 @@ var Segment = class _Segment {
|
|
|
3119
3397
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3120
3398
|
*
|
|
3121
3399
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
3400
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
3401
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
3402
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
3403
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
3404
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
3405
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
3406
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
3407
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
3408
|
+
* once, at one instant.
|
|
3122
3409
|
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3123
3410
|
*/
|
|
3124
3411
|
async intersectInto(dest, others, options) {
|
|
@@ -3133,7 +3420,8 @@ var Segment = class _Segment {
|
|
|
3133
3420
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3134
3421
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3135
3422
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3136
|
-
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes
|
|
3423
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
3424
|
+
* empty `others` reads `this` itself.
|
|
3137
3425
|
*/
|
|
3138
3426
|
union(others, options) {
|
|
3139
3427
|
const bad = optionsError(
|
|
@@ -3201,6 +3489,7 @@ var Segment = class _Segment {
|
|
|
3201
3489
|
try {
|
|
3202
3490
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3203
3491
|
const others = segmentList(operands, operandsLabel(verb));
|
|
3492
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3204
3493
|
handles = [this, ...others, ...exclude];
|
|
3205
3494
|
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3206
3495
|
this.assertSegments(verb, "an exclude", exclude);
|
|
@@ -3214,17 +3503,21 @@ var Segment = class _Segment {
|
|
|
3214
3503
|
} catch (err) {
|
|
3215
3504
|
return out.failing(err);
|
|
3216
3505
|
}
|
|
3506
|
+
const keys = handles.map((h) => h.key());
|
|
3507
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3217
3508
|
if (verb === "andNot") {
|
|
3218
3509
|
const refs2 = operands.map((o) => o.ref);
|
|
3219
3510
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3220
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3511
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3221
3512
|
}
|
|
3222
3513
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3223
3514
|
const opts = this.refsIn(options, exclude);
|
|
3224
|
-
return
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3515
|
+
return known(
|
|
3516
|
+
this.guarded(
|
|
3517
|
+
out,
|
|
3518
|
+
handles,
|
|
3519
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3520
|
+
)
|
|
3228
3521
|
);
|
|
3229
3522
|
}
|
|
3230
3523
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
@@ -3250,7 +3543,7 @@ var Segment = class _Segment {
|
|
|
3250
3543
|
}
|
|
3251
3544
|
};
|
|
3252
3545
|
export {
|
|
3253
|
-
BudgetExceededError,
|
|
3546
|
+
BudgetExceededError2 as BudgetExceededError,
|
|
3254
3547
|
CapabilityError,
|
|
3255
3548
|
CloudRoaring,
|
|
3256
3549
|
CloudRoaringError,
|
|
@@ -3268,12 +3561,12 @@ export {
|
|
|
3268
3561
|
StaleOperandError,
|
|
3269
3562
|
TransientError,
|
|
3270
3563
|
UnsupportedError3 as UnsupportedError,
|
|
3271
|
-
|
|
3564
|
+
ValidationError6 as ValidationError,
|
|
3272
3565
|
WriteConflictError2 as WriteConflictError,
|
|
3273
3566
|
deserializePortable,
|
|
3274
3567
|
destroySegment,
|
|
3275
3568
|
eraseNamespace,
|
|
3276
|
-
errorKind,
|
|
3569
|
+
errorKind2 as errorKind,
|
|
3277
3570
|
excludingReservedRows2 as excludingReservedRows,
|
|
3278
3571
|
readRetentionPolicy
|
|
3279
3572
|
};
|