@cloudbitmaps/roaring 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +38 -23
- package/README.md +27 -9
- package/dist/bin/export-segments.js +683 -260
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +395 -171
- package/dist/index.js +633 -279
- package/dist/index.js.map +4 -4
- package/dist/roaring-codec.d.ts +23 -22
- package/package.json +4 -3
- package/dist/bin/export-segments.d.ts +0 -25
- package/dist/lease-guards.d.ts +0 -46
- package/dist/open-charging-storage.d.ts +0 -37
- package/dist/option-keys.d.ts +0 -19
- package/dist/options.d.ts +0 -30
- package/dist/portable/decode.d.ts +0 -54
- package/dist/portable/layout.d.ts +0 -52
- package/dist/reserved-namespace.d.ts +0 -3
- package/dist/system-clock.d.ts +0 -39
- package/dist/testing/conformance.d.ts +0 -118
- package/dist/testing/fuzz-codec.d.ts +0 -23
- package/dist/wire-store.d.ts +0 -38
|
@@ -21,17 +21,20 @@ import {
|
|
|
21
21
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
22
22
|
SegmentEngine,
|
|
23
23
|
LEASE_SKEW_MS,
|
|
24
|
+
BudgetExceededError,
|
|
24
25
|
LeaseExpiredError,
|
|
25
26
|
MAX_LEASE_MS,
|
|
26
27
|
NotFoundError,
|
|
27
|
-
UnsupportedError,
|
|
28
|
-
ValidationError as
|
|
28
|
+
UnsupportedError as UnsupportedError2,
|
|
29
|
+
ValidationError as ValidationError5,
|
|
30
|
+
errorKind,
|
|
29
31
|
WriteConflictError,
|
|
30
32
|
checkBudget,
|
|
31
33
|
collectWithinBudget,
|
|
32
34
|
excludingReservedRows,
|
|
33
35
|
dropSegment,
|
|
34
36
|
eraseIdFromSegment,
|
|
37
|
+
findIdUnderTombstone,
|
|
35
38
|
judgeLoad,
|
|
36
39
|
loadSegment,
|
|
37
40
|
loadSegmentChunks,
|
|
@@ -52,7 +55,12 @@ import {
|
|
|
52
55
|
splitId,
|
|
53
56
|
takeLease
|
|
54
57
|
} from "@cloudbitmaps/core";
|
|
55
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
pointerIdOf,
|
|
60
|
+
requireBool,
|
|
61
|
+
requirePositiveInt,
|
|
62
|
+
validateSegmentRef
|
|
63
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
56
64
|
|
|
57
65
|
// src/open-charging-storage.ts
|
|
58
66
|
var OpenChargingStorage = class {
|
|
@@ -68,6 +76,7 @@ var OpenChargingStorage = class {
|
|
|
68
76
|
}
|
|
69
77
|
inner;
|
|
70
78
|
charge;
|
|
79
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
71
80
|
free = /* @__PURE__ */ new Set();
|
|
72
81
|
capabilities() {
|
|
73
82
|
return this.inner.capabilities();
|
|
@@ -80,9 +89,15 @@ var OpenChargingStorage = class {
|
|
|
80
89
|
getRange(key, offset, length) {
|
|
81
90
|
return this.inner.getRange(key, offset, length);
|
|
82
91
|
}
|
|
83
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
94
|
+
* generation is charged at its first open only.
|
|
95
|
+
*/
|
|
84
96
|
async getTail(key, maxBytes) {
|
|
85
|
-
if (!this.free.has(key.generation))
|
|
97
|
+
if (!this.free.has(key.generation)) {
|
|
98
|
+
this.charge();
|
|
99
|
+
this.free.add(key.generation);
|
|
100
|
+
}
|
|
86
101
|
return this.inner.getTail(key, maxBytes);
|
|
87
102
|
}
|
|
88
103
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -96,7 +111,12 @@ var OpenChargingStorage = class {
|
|
|
96
111
|
|
|
97
112
|
// src/index.ts
|
|
98
113
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
99
|
-
import {
|
|
114
|
+
import {
|
|
115
|
+
destroySegment as coreDestroySegment,
|
|
116
|
+
eraseNamespace as coreEraseNamespace
|
|
117
|
+
} from "@cloudbitmaps/core";
|
|
118
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
119
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
100
120
|
|
|
101
121
|
// src/reserved-namespace.ts
|
|
102
122
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -111,7 +131,7 @@ function refuseReservedNamespace(namespace) {
|
|
|
111
131
|
|
|
112
132
|
// src/roaring-codec.ts
|
|
113
133
|
import roaring from "roaring";
|
|
114
|
-
import { IntegrityError as IntegrityError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
134
|
+
import { IntegrityError as IntegrityError2, ValidationError as ValidationError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
115
135
|
|
|
116
136
|
// src/portable/layout.ts
|
|
117
137
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -359,10 +379,6 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
359
379
|
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
360
380
|
}
|
|
361
381
|
}
|
|
362
|
-
/** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
|
|
363
|
-
unwrap() {
|
|
364
|
-
return this.bitmap;
|
|
365
|
-
}
|
|
366
382
|
serialize() {
|
|
367
383
|
return this.bitmap.serialize(SerializationFormat.portable);
|
|
368
384
|
}
|
|
@@ -467,7 +483,7 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
467
483
|
}
|
|
468
484
|
};
|
|
469
485
|
function bitmapAsLoadInput(input) {
|
|
470
|
-
return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
|
|
486
|
+
return input instanceof RoaringBitmap32 || input instanceof ReadonlyRoaringView ? { bitmap: input } : input;
|
|
471
487
|
}
|
|
472
488
|
var roaringCodec = {
|
|
473
489
|
empty: () => SafeBitmap.empty(),
|
|
@@ -475,6 +491,31 @@ var roaringCodec = {
|
|
|
475
491
|
safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
476
492
|
owns: (bitmap) => bitmap instanceof SafeBitmap
|
|
477
493
|
};
|
|
494
|
+
var ReadonlyRoaringView = class {
|
|
495
|
+
#bitmap;
|
|
496
|
+
constructor(bitmap) {
|
|
497
|
+
this.#bitmap = bitmap;
|
|
498
|
+
}
|
|
499
|
+
get size() {
|
|
500
|
+
return this.#bitmap.size;
|
|
501
|
+
}
|
|
502
|
+
has(id) {
|
|
503
|
+
return this.#bitmap.has(id);
|
|
504
|
+
}
|
|
505
|
+
[Symbol.iterator]() {
|
|
506
|
+
return this.#bitmap[Symbol.iterator]();
|
|
507
|
+
}
|
|
508
|
+
/** The portable bytes, for exactly the argument `'portable'`; any other call is a `ValidationError`. */
|
|
509
|
+
serialize(...args) {
|
|
510
|
+
if (args.length !== 1 || args[0] !== "portable") {
|
|
511
|
+
const got = args.length === 0 ? "no argument" : args.map((a) => typeof a === "string" ? `'${a}'` : typeof a).join(", ");
|
|
512
|
+
throw new ValidationError2(
|
|
513
|
+
`a decoded bitmap serializes only as serialize('portable'); got ${got}`
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
return this.#bitmap.serialize();
|
|
517
|
+
}
|
|
518
|
+
};
|
|
478
519
|
|
|
479
520
|
// src/system-clock.ts
|
|
480
521
|
var SystemClock = class {
|
|
@@ -517,6 +558,18 @@ var SystemClock = class {
|
|
|
517
558
|
}
|
|
518
559
|
};
|
|
519
560
|
|
|
561
|
+
// src/system-entropy.ts
|
|
562
|
+
import { UnsupportedError } from "@cloudbitmaps/core";
|
|
563
|
+
var systemEntropy = (length) => {
|
|
564
|
+
const c = globalThis.crypto;
|
|
565
|
+
if (c === void 0 || typeof c.getRandomValues !== "function") {
|
|
566
|
+
throw new UnsupportedError(
|
|
567
|
+
"this runtime has no Web Crypto (crypto.getRandomValues), which every load needs for its generation's write identity: run on a runtime that has it, or pass `seams.entropy`"
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
return c.getRandomValues(new Uint8Array(length));
|
|
571
|
+
};
|
|
572
|
+
|
|
520
573
|
// src/option-keys.ts
|
|
521
574
|
var OPTION_KEYS = {
|
|
522
575
|
top: ["storage", "cache", "encryption", "retry", "metrics", "budget", "seams"],
|
|
@@ -524,7 +577,7 @@ var OPTION_KEYS = {
|
|
|
524
577
|
encryption: ["keystore", "required"],
|
|
525
578
|
retry: ["maxAttempts", "baseDelayMs", "maxDelayMs", "backoffFactor", "jitter", "onRetry"],
|
|
526
579
|
budget: ["maxRequests"],
|
|
527
|
-
seams: ["clock", "rng"]
|
|
580
|
+
seams: ["clock", "rng", "entropy"]
|
|
528
581
|
};
|
|
529
582
|
|
|
530
583
|
// src/lease-guards.ts
|
|
@@ -602,17 +655,18 @@ import {
|
|
|
602
655
|
DEFAULT_RETRY_POLICY,
|
|
603
656
|
NOOP_METRICS,
|
|
604
657
|
RetryingStorageChunkSource,
|
|
605
|
-
ValidationError as
|
|
658
|
+
ValidationError as ValidationError3,
|
|
606
659
|
isStorageBackend
|
|
607
660
|
} from "@cloudbitmaps/core";
|
|
608
661
|
function resolveRetryPolicy(partial) {
|
|
609
|
-
const
|
|
662
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
663
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
610
664
|
return {
|
|
611
|
-
maxAttempts:
|
|
665
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
612
666
|
baseDelayMs,
|
|
613
|
-
maxDelayMs: partial.maxDelayMs
|
|
614
|
-
backoffFactor:
|
|
615
|
-
jitter:
|
|
667
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
668
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
669
|
+
jitter: orDefault("jitter")
|
|
616
670
|
};
|
|
617
671
|
}
|
|
618
672
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -625,7 +679,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
625
679
|
if (options.retry !== false) {
|
|
626
680
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
627
681
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
628
|
-
throw new
|
|
682
|
+
throw new ValidationError3("retry.onRetry must be a function");
|
|
629
683
|
}
|
|
630
684
|
retryOptions = {
|
|
631
685
|
clock,
|
|
@@ -649,13 +703,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
649
703
|
source: resolved.source,
|
|
650
704
|
driver: resolved.driver,
|
|
651
705
|
registry: resolved.registry,
|
|
706
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
652
707
|
retryOptions
|
|
653
708
|
};
|
|
654
709
|
}
|
|
655
710
|
function resolveStorageSource(options, clock) {
|
|
656
711
|
const storage = options.storage;
|
|
657
712
|
if (storage === null || typeof storage !== "object") {
|
|
658
|
-
throw new
|
|
713
|
+
throw new ValidationError3(
|
|
659
714
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
660
715
|
);
|
|
661
716
|
}
|
|
@@ -665,14 +720,14 @@ function resolveStorageSource(options, clock) {
|
|
|
665
720
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
666
721
|
const isBackend = isStorageBackend(storage);
|
|
667
722
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
668
|
-
throw new
|
|
723
|
+
throw new ValidationError3(
|
|
669
724
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
670
725
|
);
|
|
671
726
|
}
|
|
672
727
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
673
728
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
674
729
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
675
|
-
throw new
|
|
730
|
+
throw new ValidationError3(
|
|
676
731
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
677
732
|
);
|
|
678
733
|
}
|
|
@@ -681,11 +736,11 @@ function resolveStorageSource(options, clock) {
|
|
|
681
736
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
682
737
|
if (!storageOk || !registryOk) {
|
|
683
738
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
684
|
-
throw new
|
|
739
|
+
throw new ValidationError3(
|
|
685
740
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
686
741
|
);
|
|
687
742
|
}
|
|
688
|
-
throw new
|
|
743
|
+
throw new ValidationError3(
|
|
689
744
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
690
745
|
);
|
|
691
746
|
}
|
|
@@ -706,19 +761,19 @@ function resolveStorageSource(options, clock) {
|
|
|
706
761
|
};
|
|
707
762
|
}
|
|
708
763
|
if (hasGetChunk && hasPutImmutable) {
|
|
709
|
-
throw new
|
|
764
|
+
throw new ValidationError3(
|
|
710
765
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
711
766
|
);
|
|
712
767
|
}
|
|
713
768
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
714
|
-
throw new
|
|
769
|
+
throw new ValidationError3(
|
|
715
770
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
716
771
|
);
|
|
717
772
|
}
|
|
718
773
|
if (hasGetChunk) {
|
|
719
774
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
720
|
-
throw new
|
|
721
|
-
"keystore/requireEncryption apply only when the store builds its own read path;
|
|
775
|
+
throw new ValidationError3(
|
|
776
|
+
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
722
777
|
);
|
|
723
778
|
}
|
|
724
779
|
return { source: storage, driver: void 0, registry: void 0 };
|
|
@@ -739,31 +794,32 @@ function resolveStorageSource(options, clock) {
|
|
|
739
794
|
}
|
|
740
795
|
|
|
741
796
|
// src/options.ts
|
|
742
|
-
import { ValidationError as
|
|
797
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core";
|
|
743
798
|
var keysOf = () => (keys) => keys;
|
|
744
799
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
745
|
-
function optionsError(options, op, known, booleans = []) {
|
|
800
|
+
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
746
801
|
if (options === void 0 || options === null) return void 0;
|
|
747
802
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
748
|
-
return new
|
|
803
|
+
return new ValidationError4(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
749
804
|
}
|
|
750
805
|
const bag = options;
|
|
751
806
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
807
|
+
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
752
808
|
if (unknown.length > 0) {
|
|
753
|
-
return new
|
|
809
|
+
return new ValidationError4(
|
|
754
810
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
755
811
|
);
|
|
756
812
|
}
|
|
757
813
|
const audit = bag.audit;
|
|
758
814
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
759
|
-
return new
|
|
815
|
+
return new ValidationError4(
|
|
760
816
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
761
817
|
);
|
|
762
818
|
}
|
|
763
819
|
for (const name of booleans) {
|
|
764
820
|
const v = bag[name];
|
|
765
821
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
766
|
-
return new
|
|
822
|
+
return new ValidationError4(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
767
823
|
}
|
|
768
824
|
}
|
|
769
825
|
return void 0;
|
|
@@ -777,58 +833,32 @@ function picked(options, keys) {
|
|
|
777
833
|
}
|
|
778
834
|
return out;
|
|
779
835
|
}
|
|
780
|
-
function checkOptions(options, op, known, booleans) {
|
|
781
|
-
const err = optionsError(options, op, known, booleans);
|
|
836
|
+
function checkOptions(options, op, known, booleans, misplaced) {
|
|
837
|
+
const err = optionsError(options, op, known, booleans, misplaced);
|
|
782
838
|
if (err !== void 0) throw err;
|
|
783
839
|
}
|
|
784
840
|
|
|
785
841
|
// src/index.ts
|
|
842
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
786
843
|
import {
|
|
787
|
-
MemoryStorage,
|
|
788
|
-
LocalFsStorage,
|
|
789
|
-
CrbmStorageChunkSource as CrbmStorageChunkSource3,
|
|
790
|
-
destroySegment,
|
|
791
|
-
eraseNamespace,
|
|
792
844
|
excludingReservedRows as excludingReservedRows2,
|
|
793
845
|
readRetentionPolicy,
|
|
794
|
-
MIN_EXPIRES_AT_MS,
|
|
795
|
-
LEASE_SKEW_MS as LEASE_SKEW_MS2,
|
|
796
|
-
MAX_LEASE_MS as MAX_LEASE_MS2,
|
|
797
|
-
MAX_LEASES_PER_SEGMENT,
|
|
798
|
-
InProcessKeystore,
|
|
799
|
-
NodeAead,
|
|
800
|
-
aadFor,
|
|
801
846
|
CloudRoaringError,
|
|
802
|
-
ValidationError as
|
|
847
|
+
ValidationError as ValidationError6,
|
|
803
848
|
WriteConflictError as WriteConflictError2,
|
|
804
849
|
IntegrityError as IntegrityError3,
|
|
805
850
|
NotFoundError as NotFoundError2,
|
|
806
|
-
UnsupportedError as
|
|
851
|
+
UnsupportedError as UnsupportedError3,
|
|
807
852
|
CapabilityError,
|
|
808
853
|
TransientError,
|
|
809
854
|
KeyUnavailableError,
|
|
810
|
-
BudgetExceededError,
|
|
855
|
+
BudgetExceededError as BudgetExceededError2,
|
|
811
856
|
LeaseExpiredError as LeaseExpiredError2,
|
|
812
857
|
LeaseLimitError,
|
|
813
858
|
StaleOperandError,
|
|
814
|
-
|
|
815
|
-
isWriteConflictError,
|
|
816
|
-
isTransientError,
|
|
817
|
-
isNotFoundError,
|
|
818
|
-
isIntegrityError,
|
|
819
|
-
isValidationError,
|
|
820
|
-
isLeaseExpiredError,
|
|
821
|
-
isLeaseLimitError,
|
|
822
|
-
isStaleOperandError,
|
|
823
|
-
isUnsupportedError,
|
|
824
|
-
isCapabilityError,
|
|
825
|
-
isBudgetExceededError,
|
|
826
|
-
isKeyUnavailableError,
|
|
827
|
-
CrbmReader,
|
|
828
|
-
BufferReader,
|
|
859
|
+
errorKind as errorKind2,
|
|
829
860
|
CountingMetricsSink,
|
|
830
|
-
RecordingAuditSink
|
|
831
|
-
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY3
|
|
861
|
+
RecordingAuditSink
|
|
832
862
|
} from "@cloudbitmaps/core";
|
|
833
863
|
var SystemRng = class {
|
|
834
864
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -836,6 +866,16 @@ var SystemRng = class {
|
|
|
836
866
|
return Math.random();
|
|
837
867
|
}
|
|
838
868
|
};
|
|
869
|
+
function validateRef(ref, verb) {
|
|
870
|
+
if (typeof ref !== "object" || ref === null) {
|
|
871
|
+
throw new ValidationError5(
|
|
872
|
+
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
refuseUnknownRefKeys(ref, verb);
|
|
876
|
+
validateSegmentRef(ref);
|
|
877
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
878
|
+
}
|
|
839
879
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
840
880
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
841
881
|
function rowVersionOfLive(version) {
|
|
@@ -849,15 +889,20 @@ function validateConcurrency(concurrency) {
|
|
|
849
889
|
function requireScope(options, op) {
|
|
850
890
|
if (options.namespace !== void 0) {
|
|
851
891
|
if (typeof options.namespace !== "string") {
|
|
852
|
-
throw new
|
|
892
|
+
throw new ValidationError5(
|
|
853
893
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
854
894
|
);
|
|
855
895
|
}
|
|
856
896
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
857
897
|
}
|
|
858
898
|
refuseReservedNamespace(options.namespace);
|
|
899
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
900
|
+
throw new ValidationError5(
|
|
901
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
859
904
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
860
|
-
throw new
|
|
905
|
+
throw new ValidationError5(
|
|
861
906
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
862
907
|
);
|
|
863
908
|
}
|
|
@@ -904,7 +949,7 @@ var CALL_KEYS = {
|
|
|
904
949
|
"metadata",
|
|
905
950
|
"keep"
|
|
906
951
|
]),
|
|
907
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
952
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
908
953
|
segments: keysOf()(["namespace"]),
|
|
909
954
|
subjectReport: keysOf()([
|
|
910
955
|
"namespace",
|
|
@@ -945,13 +990,37 @@ var CALL_KEYS = {
|
|
|
945
990
|
checkConsistency: keysOf()([
|
|
946
991
|
"namespace",
|
|
947
992
|
"concurrency",
|
|
993
|
+
"maxScanSegments",
|
|
948
994
|
"summaries",
|
|
949
995
|
"generations"
|
|
950
996
|
]),
|
|
951
|
-
exportSegments: keysOf()(["format", "
|
|
997
|
+
exportSegments: keysOf()(["format", "namespace", "ndjsonBatchBytes"])
|
|
998
|
+
};
|
|
999
|
+
var outside = (wider, known) => wider.filter((k) => !known.includes(k));
|
|
1000
|
+
var MISPLACED = {
|
|
1001
|
+
range: outside(CALL_KEYS.into, CALL_KEYS.range),
|
|
1002
|
+
andNot: outside(CALL_KEYS.into, CALL_KEYS.andNot),
|
|
1003
|
+
combine: outside(CALL_KEYS.into, CALL_KEYS.combine),
|
|
1004
|
+
andNotInto: outside(CALL_KEYS.into, CALL_KEYS.andNotInto),
|
|
1005
|
+
exportSegments: ["codec"]
|
|
952
1006
|
};
|
|
953
1007
|
var SEGMENT_BRAND = /* @__PURE__ */ Symbol.for("cloudbitmaps.roaring.segment/1");
|
|
954
1008
|
var viewOf;
|
|
1009
|
+
var originOf;
|
|
1010
|
+
function assertOneStore(verb, role, h, origin) {
|
|
1011
|
+
if (h instanceof Segment) {
|
|
1012
|
+
if (originOf(h) === origin) return;
|
|
1013
|
+
throw new ValidationError5(
|
|
1014
|
+
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
1015
|
+
);
|
|
1016
|
+
}
|
|
1017
|
+
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
1018
|
+
throw new ValidationError5(
|
|
1019
|
+
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
1020
|
+
);
|
|
1021
|
+
}
|
|
1022
|
+
throw new ValidationError5(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
1023
|
+
}
|
|
955
1024
|
var MATERIALIZE_MANY_KEYS = [
|
|
956
1025
|
"operands",
|
|
957
1026
|
"dryRun",
|
|
@@ -968,6 +1037,17 @@ var MATERIALIZE_MANY_KEYS = [
|
|
|
968
1037
|
"feed",
|
|
969
1038
|
"mayBeEmpty"
|
|
970
1039
|
];
|
|
1040
|
+
var MATERIALIZE_MANY_MISPLACED = outside(CALL_KEYS.into, MATERIALIZE_MANY_KEYS);
|
|
1041
|
+
function unknownManyKeys(options) {
|
|
1042
|
+
const unknown = Object.keys(options).filter(
|
|
1043
|
+
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1044
|
+
);
|
|
1045
|
+
const bag = options;
|
|
1046
|
+
for (const k of MATERIALIZE_MANY_MISPLACED) {
|
|
1047
|
+
if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
1048
|
+
}
|
|
1049
|
+
return unknown;
|
|
1050
|
+
}
|
|
971
1051
|
var MATERIALIZE_MANY_OUTPUT_KEYS = [
|
|
972
1052
|
"dest",
|
|
973
1053
|
"expr",
|
|
@@ -993,18 +1073,23 @@ var NOT_A_GROUP = {
|
|
|
993
1073
|
};
|
|
994
1074
|
var CloudRoaring = class _CloudRoaring {
|
|
995
1075
|
engine;
|
|
1076
|
+
/** This store's identity: every handle it makes carries this object, and a combine compares it by reference. */
|
|
1077
|
+
origin = Object.freeze({});
|
|
996
1078
|
cache;
|
|
997
1079
|
/** The store's retry options, for a pinned engine to read through as the store's own does; undefined when off. */
|
|
998
1080
|
retryOptions;
|
|
999
1081
|
crbmSource;
|
|
1000
1082
|
clock;
|
|
1001
1083
|
rng;
|
|
1084
|
+
entropy;
|
|
1002
1085
|
metrics;
|
|
1003
1086
|
// The store's own drivers, kept so the lifecycle helpers and the `*Into` verbs reuse them instead of making
|
|
1004
1087
|
// you re-pass deps. `storageDriver` is set only when `storage` was a raw IStorageDriver (a pre-built StorageChunkSource has
|
|
1005
1088
|
// no underlying driver to write through).
|
|
1006
1089
|
storageDriver;
|
|
1007
1090
|
registry;
|
|
1091
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
1092
|
+
backend;
|
|
1008
1093
|
keystore;
|
|
1009
1094
|
requireEncryption;
|
|
1010
1095
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1024,7 +1109,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1024
1109
|
*/
|
|
1025
1110
|
static rejectUnknownOptions(options) {
|
|
1026
1111
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1027
|
-
throw new
|
|
1112
|
+
throw new ValidationError5(
|
|
1028
1113
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1029
1114
|
);
|
|
1030
1115
|
}
|
|
@@ -1042,7 +1127,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1042
1127
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1043
1128
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1044
1129
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1045
|
-
throw new
|
|
1130
|
+
throw new ValidationError5(
|
|
1046
1131
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1047
1132
|
);
|
|
1048
1133
|
}
|
|
@@ -1052,7 +1137,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1052
1137
|
if (unknown.length === 0) return;
|
|
1053
1138
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1054
1139
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1055
|
-
throw new
|
|
1140
|
+
throw new ValidationError5(
|
|
1056
1141
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1057
1142
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1058
1143
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1068,21 +1153,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1068
1153
|
[options.seams?.rng, "seams.rng", ["next"]],
|
|
1069
1154
|
[options.encryption?.keystore, "encryption.keystore", ["createDek", "openDek"]]
|
|
1070
1155
|
];
|
|
1156
|
+
const entropy = options.seams?.entropy;
|
|
1157
|
+
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1158
|
+
throw new ValidationError5(
|
|
1159
|
+
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1160
|
+
);
|
|
1161
|
+
}
|
|
1071
1162
|
for (const [seam, name, needs] of seams) {
|
|
1072
|
-
if (seam === void 0
|
|
1073
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1074
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1075
|
-
throw new
|
|
1163
|
+
if (seam === void 0) continue;
|
|
1164
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
1165
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
1166
|
+
throw new ValidationError5(
|
|
1076
1167
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1077
1168
|
);
|
|
1078
1169
|
}
|
|
1079
1170
|
}
|
|
1080
1171
|
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
1174
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
1175
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
1176
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
1177
|
+
*/
|
|
1178
|
+
static rejectBadValues(options) {
|
|
1179
|
+
const cache = options.cache;
|
|
1180
|
+
if (cache !== void 0) {
|
|
1181
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
1182
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
1183
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
1184
|
+
const ttl = cache.ttlMs;
|
|
1185
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
1186
|
+
throw new ValidationError5(
|
|
1187
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
1192
|
+
}
|
|
1193
|
+
/**
|
|
1194
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
1195
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
1196
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
1197
|
+
* calls, throws `ValidationError`.
|
|
1198
|
+
*/
|
|
1081
1199
|
constructor(options) {
|
|
1082
1200
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1083
1201
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1202
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1084
1203
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1085
1204
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1205
|
+
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
1086
1206
|
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS2);
|
|
1087
1207
|
const cache = new BoundedLru({
|
|
1088
1208
|
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
@@ -1106,9 +1226,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1106
1226
|
this.crbmSource = wired.source instanceof CrbmStorageChunkSource2 ? wired.source : void 0;
|
|
1107
1227
|
this.clock = clock;
|
|
1108
1228
|
this.rng = rng;
|
|
1229
|
+
this.entropy = entropy;
|
|
1109
1230
|
this.metrics = metrics;
|
|
1110
1231
|
this.storageDriver = wired.driver;
|
|
1111
1232
|
this.registry = wired.registry;
|
|
1233
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
1234
|
+
this.backend = wired.backend;
|
|
1112
1235
|
this.keystore = options.encryption?.keystore;
|
|
1113
1236
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1114
1237
|
}
|
|
@@ -1120,12 +1243,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1120
1243
|
*/
|
|
1121
1244
|
lifecycleDeps(op) {
|
|
1122
1245
|
if (this.storageDriver === void 0) {
|
|
1123
|
-
throw new
|
|
1246
|
+
throw new UnsupportedError2(
|
|
1124
1247
|
`${op} needs the store built with a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A pre-built StorageChunkSource is read-only: it has no IStorageDriver underneath to write generations through. Build the store on one of those backend classes to write.`
|
|
1125
1248
|
);
|
|
1126
1249
|
}
|
|
1127
1250
|
if (this.registry === void 0) {
|
|
1128
|
-
throw new
|
|
1251
|
+
throw new UnsupportedError2(
|
|
1129
1252
|
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
1130
1253
|
);
|
|
1131
1254
|
}
|
|
@@ -1134,6 +1257,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1134
1257
|
registry: this.registry,
|
|
1135
1258
|
clock: this.clock,
|
|
1136
1259
|
rng: this.rng,
|
|
1260
|
+
entropy: this.entropy,
|
|
1137
1261
|
codec: roaringCodec,
|
|
1138
1262
|
// facade injects the flagship codec
|
|
1139
1263
|
keystore: this.keystore,
|
|
@@ -1150,13 +1274,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1150
1274
|
* spread of options keeps working.
|
|
1151
1275
|
*/
|
|
1152
1276
|
segment(name, options) {
|
|
1277
|
+
if (typeof name === "object" && name !== null) {
|
|
1278
|
+
throw new ValidationError5(
|
|
1279
|
+
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1153
1282
|
if (options !== void 0 && options !== null) {
|
|
1154
1283
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1155
|
-
throw new
|
|
1284
|
+
throw new ValidationError5("segment: options must be an object such as { namespace }");
|
|
1156
1285
|
}
|
|
1157
1286
|
const proto = Object.getPrototypeOf(options);
|
|
1158
1287
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1159
|
-
throw new
|
|
1288
|
+
throw new ValidationError5(
|
|
1160
1289
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1161
1290
|
);
|
|
1162
1291
|
}
|
|
@@ -1164,7 +1293,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1164
1293
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1165
1294
|
);
|
|
1166
1295
|
if (unknown.length > 0) {
|
|
1167
|
-
throw new
|
|
1296
|
+
throw new ValidationError5(
|
|
1168
1297
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1169
1298
|
);
|
|
1170
1299
|
}
|
|
@@ -1174,6 +1303,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1174
1303
|
refuseReservedNamespace(ref.namespace);
|
|
1175
1304
|
return makeSegment({
|
|
1176
1305
|
engine: this.engine,
|
|
1306
|
+
origin: this.origin,
|
|
1177
1307
|
ref,
|
|
1178
1308
|
clock: this.clock,
|
|
1179
1309
|
metrics: this.metrics,
|
|
@@ -1245,7 +1375,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1245
1375
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1246
1376
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1247
1377
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1248
|
-
keep: options?.keep
|
|
1378
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1249
1379
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1250
1380
|
};
|
|
1251
1381
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1273,22 +1403,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1273
1403
|
/** The one body of the forms above. */
|
|
1274
1404
|
async materializeMany(options) {
|
|
1275
1405
|
if (typeof options !== "object" || options === null) {
|
|
1276
|
-
throw new
|
|
1406
|
+
throw new ValidationError5(
|
|
1277
1407
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1278
1408
|
);
|
|
1279
1409
|
}
|
|
1280
1410
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1281
|
-
const unknown =
|
|
1282
|
-
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1283
|
-
);
|
|
1411
|
+
const unknown = unknownManyKeys(options);
|
|
1284
1412
|
if (unknown.length > 0) {
|
|
1285
|
-
throw new
|
|
1413
|
+
throw new ValidationError5(
|
|
1286
1414
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1287
1415
|
);
|
|
1288
1416
|
}
|
|
1289
1417
|
const dryRunIn = options.dryRun;
|
|
1290
1418
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1291
|
-
throw new
|
|
1419
|
+
throw new ValidationError5("materializeMany: dryRun must be a boolean");
|
|
1292
1420
|
}
|
|
1293
1421
|
const dryRun = dryRunIn === true;
|
|
1294
1422
|
const operandsIn = options.operands;
|
|
@@ -1300,7 +1428,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1300
1428
|
const feedIn = options.feed;
|
|
1301
1429
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1302
1430
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1303
|
-
throw new
|
|
1431
|
+
throw new ValidationError5(
|
|
1304
1432
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1305
1433
|
);
|
|
1306
1434
|
}
|
|
@@ -1310,37 +1438,33 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1310
1438
|
const budgetOption = options.budget;
|
|
1311
1439
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1312
1440
|
if (typeof pinOption !== "boolean")
|
|
1313
|
-
throw new
|
|
1441
|
+
throw new ValidationError5("materializeMany: pin must be a boolean");
|
|
1314
1442
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1315
|
-
throw new
|
|
1443
|
+
throw new ValidationError5("materializeMany: allowAbsentOperands must be a boolean");
|
|
1316
1444
|
}
|
|
1317
1445
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1318
|
-
throw new
|
|
1446
|
+
throw new ValidationError5("materializeMany: operands must be an object of name to segment");
|
|
1319
1447
|
}
|
|
1320
1448
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1321
|
-
throw new
|
|
1449
|
+
throw new ValidationError5("materializeMany: outputs must be a non-empty array");
|
|
1322
1450
|
}
|
|
1323
1451
|
const names = Object.keys(operandsIn);
|
|
1324
1452
|
const handles = /* @__PURE__ */ new Map();
|
|
1325
1453
|
for (const name of names) {
|
|
1326
1454
|
const handle = operandsIn[name];
|
|
1327
|
-
|
|
1328
|
-
throw new ValidationError4(
|
|
1329
|
-
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1330
|
-
);
|
|
1331
|
-
}
|
|
1455
|
+
assertOneStore("materializeMany", `operands["${name}"]`, handle, this.origin);
|
|
1332
1456
|
handles.set(name, handle);
|
|
1333
1457
|
}
|
|
1334
1458
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1335
1459
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1336
|
-
throw new
|
|
1460
|
+
throw new ValidationError5(
|
|
1337
1461
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1338
1462
|
);
|
|
1339
1463
|
}
|
|
1340
1464
|
if (mayBeEmptyIn.length > 0) {
|
|
1341
1465
|
const first = mayBeEmptyIn[0];
|
|
1342
1466
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1343
|
-
throw new
|
|
1467
|
+
throw new ValidationError5(
|
|
1344
1468
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1345
1469
|
);
|
|
1346
1470
|
}
|
|
@@ -1350,28 +1474,26 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1350
1474
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1351
1475
|
outputsIn.forEach((raw, i) => {
|
|
1352
1476
|
if (typeof raw !== "object" || raw === null) {
|
|
1353
|
-
throw new
|
|
1477
|
+
throw new ValidationError5(`outputs[${i}] must be an object`);
|
|
1354
1478
|
}
|
|
1355
1479
|
const bad = Object.keys(raw).filter(
|
|
1356
1480
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1357
1481
|
);
|
|
1358
1482
|
if (bad.length > 0) {
|
|
1359
|
-
throw new
|
|
1483
|
+
throw new ValidationError5(
|
|
1360
1484
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1361
1485
|
);
|
|
1362
1486
|
}
|
|
1363
1487
|
const spec = raw;
|
|
1364
1488
|
const dest = spec.dest;
|
|
1365
|
-
|
|
1366
|
-
throw new ValidationError4(`outputs[${i}].dest must be a segment from store.segment()`);
|
|
1367
|
-
}
|
|
1489
|
+
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1368
1490
|
if (operandKeys.has(dest.key())) {
|
|
1369
|
-
throw new
|
|
1491
|
+
throw new ValidationError5(
|
|
1370
1492
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1371
1493
|
);
|
|
1372
1494
|
}
|
|
1373
1495
|
if (destKeys.has(dest.key())) {
|
|
1374
|
-
throw new
|
|
1496
|
+
throw new ValidationError5(
|
|
1375
1497
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1376
1498
|
);
|
|
1377
1499
|
}
|
|
@@ -1431,7 +1553,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1431
1553
|
this.lifecycleDeps("materializeMany");
|
|
1432
1554
|
const crbm = this.crbmSource;
|
|
1433
1555
|
if (crbm === void 0) {
|
|
1434
|
-
throw new
|
|
1556
|
+
throw new UnsupportedError2(
|
|
1435
1557
|
"materializeMany needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage`"
|
|
1436
1558
|
);
|
|
1437
1559
|
}
|
|
@@ -1444,13 +1566,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1444
1566
|
held.set(key, { at, handle });
|
|
1445
1567
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1446
1568
|
if (!samePin(prior.at, at)) {
|
|
1447
|
-
throw new
|
|
1569
|
+
throw new ValidationError5(
|
|
1448
1570
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1449
1571
|
);
|
|
1450
1572
|
}
|
|
1451
1573
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1452
1574
|
if (!pinOption) {
|
|
1453
|
-
throw new
|
|
1575
|
+
throw new ValidationError5(
|
|
1454
1576
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1455
1577
|
);
|
|
1456
1578
|
}
|
|
@@ -1538,7 +1660,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1538
1660
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1539
1661
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1540
1662
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1541
|
-
throw new
|
|
1663
|
+
throw new ValidationError5(
|
|
1542
1664
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1543
1665
|
);
|
|
1544
1666
|
}
|
|
@@ -1588,13 +1710,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1588
1710
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1589
1711
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1590
1712
|
*
|
|
1713
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1714
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1715
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1716
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1717
|
+
*
|
|
1591
1718
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1592
1719
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1593
1720
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1594
1721
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1595
1722
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
1723
|
+
*
|
|
1724
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
1725
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
1726
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
1727
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
1728
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
1729
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
1730
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
1731
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
1732
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
1733
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
1734
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
1735
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
1736
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1596
1737
|
*/
|
|
1597
|
-
async subjectReport(id, options
|
|
1738
|
+
async subjectReport(id, options) {
|
|
1598
1739
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1599
1740
|
options ??= {};
|
|
1600
1741
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1609,26 +1750,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1609
1750
|
budget,
|
|
1610
1751
|
"subjectReport"
|
|
1611
1752
|
);
|
|
1753
|
+
let opens = 0;
|
|
1754
|
+
const charge = () => {
|
|
1755
|
+
opens += 1;
|
|
1756
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
1757
|
+
throw new BudgetExceededError(
|
|
1758
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
1759
|
+
{
|
|
1760
|
+
option: "budget.maxRequests",
|
|
1761
|
+
limit: budget.maxRequests,
|
|
1762
|
+
spent: budget.maxRequests + 1
|
|
1763
|
+
}
|
|
1764
|
+
);
|
|
1765
|
+
}
|
|
1766
|
+
};
|
|
1612
1767
|
const membership = await mapWithConcurrency(
|
|
1613
1768
|
recs,
|
|
1614
1769
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1615
1770
|
async (rec) => {
|
|
1616
|
-
if (rec.status === "destroyed") return null;
|
|
1617
1771
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1772
|
+
if (rec.status === "destroyed") {
|
|
1773
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
1774
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
1775
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
1776
|
+
return held === null ? null : { tombstone: true, ref };
|
|
1777
|
+
}
|
|
1618
1778
|
if (this.crbmSource !== void 0) {
|
|
1619
1779
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1620
|
-
const
|
|
1621
|
-
if (rowVersionOfLive(held) !==
|
|
1780
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1781
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1622
1782
|
}
|
|
1623
|
-
return await this.engine.has(ref, id) ? {
|
|
1783
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1624
1784
|
}
|
|
1625
1785
|
);
|
|
1626
|
-
const
|
|
1627
|
-
|
|
1786
|
+
const listed = membership.filter((m) => m !== null);
|
|
1787
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
1788
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
1789
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1628
1790
|
}
|
|
1629
1791
|
/**
|
|
1630
1792
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1631
|
-
* from the bucket on return
|
|
1793
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
1794
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
1795
|
+
*
|
|
1796
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1797
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1798
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1799
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1632
1800
|
*
|
|
1633
1801
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1634
1802
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1693,15 +1861,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1693
1861
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1694
1862
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1695
1863
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1696
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1697
|
-
*
|
|
1864
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
1865
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
1866
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
1867
|
+
* that exhausts the budget is
|
|
1698
1868
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1699
1869
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1700
1870
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1701
1871
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1702
1872
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1703
1873
|
*/
|
|
1704
|
-
async eraseSubject(id, options
|
|
1874
|
+
async eraseSubject(id, options) {
|
|
1705
1875
|
this.erasureEpoch++;
|
|
1706
1876
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1707
1877
|
options ??= {};
|
|
@@ -1757,7 +1927,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1757
1927
|
segment: rec.segment,
|
|
1758
1928
|
namespace: rec.namespace,
|
|
1759
1929
|
erased: false,
|
|
1760
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
1930
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
1931
|
+
// the class lets a note be matched to the guide's table
|
|
1761
1932
|
};
|
|
1762
1933
|
} finally {
|
|
1763
1934
|
try {
|
|
@@ -1778,11 +1949,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1778
1949
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1779
1950
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1780
1951
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1952
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1781
1953
|
*
|
|
1782
1954
|
* Needs a backend.
|
|
1783
1955
|
*/
|
|
1784
1956
|
async generations(ref) {
|
|
1785
|
-
|
|
1957
|
+
ref = validateRef(ref, "generations");
|
|
1786
1958
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1787
1959
|
}
|
|
1788
1960
|
/**
|
|
@@ -1812,7 +1984,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1812
1984
|
* ```
|
|
1813
1985
|
*/
|
|
1814
1986
|
async exists(ref) {
|
|
1815
|
-
|
|
1987
|
+
ref = validateRef(ref, "exists");
|
|
1816
1988
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1817
1989
|
}
|
|
1818
1990
|
/**
|
|
@@ -1869,8 +2041,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1869
2041
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1870
2042
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1871
2043
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1872
|
-
* crypto-shredded
|
|
1873
|
-
*
|
|
2044
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
2045
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
2046
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1874
2047
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1875
2048
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1876
2049
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1881,8 +2054,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1881
2054
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1882
2055
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1883
2056
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1884
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1885
|
-
*
|
|
2057
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
2058
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
2059
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
2060
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1886
2061
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1887
2062
|
*
|
|
1888
2063
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1912,7 +2087,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1912
2087
|
* Needs a backend.
|
|
1913
2088
|
*/
|
|
1914
2089
|
async rollback(ref, toGeneration, options = {}) {
|
|
1915
|
-
|
|
2090
|
+
ref = validateRef(ref, "rollback");
|
|
1916
2091
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1917
2092
|
options ??= {};
|
|
1918
2093
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1928,9 +2103,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1928
2103
|
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
1929
2104
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1930
2105
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1931
|
-
* containers with no per-id work, and gives the generation
|
|
1932
|
-
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A
|
|
1933
|
-
* `ValidationError`, since each byte would load as an id:
|
|
2106
|
+
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. A bare
|
|
2107
|
+
* `RoaringBitmap32`, or a bitmap `deserializePortable` returned, passed as ids loads as `{ bitmap }`. A
|
|
2108
|
+
* `Uint8Array` or `Buffer` passed as ids is refused with `ValidationError`, since each byte would load as an id:
|
|
2109
|
+
* pass bytes as `{ serialized }`.
|
|
2110
|
+
*
|
|
2111
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
2112
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
2113
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
2114
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1934
2115
|
*
|
|
1935
2116
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1936
2117
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -1974,7 +2155,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1974
2155
|
* its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1975
2156
|
* encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
|
|
1976
2157
|
* load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
|
|
1977
|
-
* writer's object, and
|
|
2158
|
+
* writer's object, and the row does not say whether that write took this number, so it leaves the orphan rather than risk
|
|
1978
2159
|
* deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
|
|
1979
2160
|
* generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
|
|
1980
2161
|
* with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
|
|
@@ -1989,7 +2170,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1989
2170
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
1990
2171
|
* what the pass deleted by name, and one name may have been gone already.
|
|
1991
2172
|
*
|
|
1992
|
-
*
|
|
2173
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
2174
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
2175
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
2176
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
2177
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
2178
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
2179
|
+
*
|
|
2180
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
1993
2181
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
1994
2182
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
1995
2183
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2012,8 +2200,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2012
2200
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2013
2201
|
*/
|
|
2014
2202
|
async load(ref, input, options = {}) {
|
|
2015
|
-
|
|
2016
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
2203
|
+
ref = validateRef(ref, "load");
|
|
2204
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
2205
|
+
if (typeof input === "object" && input !== null) {
|
|
2206
|
+
const reads = STREAM_SOURCES.get(input);
|
|
2207
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
2208
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
2209
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
2210
|
+
throw new ValidationError5(
|
|
2211
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
2212
|
+
);
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2017
2215
|
options ??= {};
|
|
2018
2216
|
const deps = this.lifecycleDeps("load");
|
|
2019
2217
|
try {
|
|
@@ -2023,11 +2221,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2023
2221
|
}
|
|
2024
2222
|
}
|
|
2025
2223
|
/**
|
|
2026
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
2224
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
2225
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2027
2226
|
*
|
|
2028
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2029
|
-
*
|
|
2030
|
-
*
|
|
2227
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
2228
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
2229
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
2230
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
2231
|
+
* encrypted segment it does both.
|
|
2031
2232
|
*
|
|
2032
2233
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2033
2234
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2058,7 +2259,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2058
2259
|
* segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
|
|
2059
2260
|
* cannot be read because of a transient fault, until a retry 500 ms apart reaches it again. A reader that never
|
|
2060
2261
|
* touched it sees empty at once. **That bound needs a registry and `cache.genTtlMs > 0`.** A store with no registry
|
|
2061
|
-
* (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a
|
|
2262
|
+
* (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a `StorageChunkSource` of your own, has no timed
|
|
2062
2263
|
* refresh. It notices the drop only when a read has to fetch
|
|
2063
2264
|
* from a deleted generation or its reader cache evicts the segment, and until then a chunk it has cached
|
|
2064
2265
|
* answers without reaching storage, so it can keep answering `true` for a dropped segment indefinitely. Tell
|
|
@@ -2068,9 +2269,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2068
2269
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2069
2270
|
*/
|
|
2070
2271
|
async dropSegment(ref, options) {
|
|
2071
|
-
|
|
2272
|
+
ref = validateRef(ref, "dropSegment");
|
|
2072
2273
|
if (options === void 0 || options === null) {
|
|
2073
|
-
throw new
|
|
2274
|
+
throw new ValidationError5("dropSegment: options { confirmSegment } are required");
|
|
2074
2275
|
}
|
|
2075
2276
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2076
2277
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2085,8 +2286,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2085
2286
|
}
|
|
2086
2287
|
}
|
|
2087
2288
|
/**
|
|
2088
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2089
|
-
*
|
|
2289
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
2290
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
2291
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2090
2292
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2091
2293
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2092
2294
|
*
|
|
@@ -2106,12 +2308,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2106
2308
|
* enumerable by the sweep; the first load then publishes onto that row.
|
|
2107
2309
|
*
|
|
2108
2310
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2109
|
-
* buckets is normal. A value below `
|
|
2311
|
+
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2110
2312
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2111
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
2313
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2112
2314
|
*/
|
|
2113
2315
|
async setRetention(ref, policy) {
|
|
2114
|
-
|
|
2316
|
+
ref = validateRef(ref, "setRetention");
|
|
2115
2317
|
return setSegmentRetention(
|
|
2116
2318
|
ref,
|
|
2117
2319
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2125,16 +2327,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2125
2327
|
* "never expires" on a segment someone believes is expiring.
|
|
2126
2328
|
*/
|
|
2127
2329
|
async getRetention(ref) {
|
|
2128
|
-
|
|
2330
|
+
ref = validateRef(ref, "getRetention");
|
|
2129
2331
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2130
2332
|
}
|
|
2131
2333
|
/**
|
|
2132
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2133
|
-
* when there was none
|
|
2334
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
2335
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
2336
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
2337
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2134
2338
|
* to the setter is how a typo becomes a deletion.
|
|
2135
2339
|
*/
|
|
2136
2340
|
async clearRetention(ref) {
|
|
2137
|
-
|
|
2341
|
+
ref = validateRef(ref, "clearRetention");
|
|
2138
2342
|
return clearSegmentRetention(ref, {
|
|
2139
2343
|
registry: this.requireRegistry("clearRetention"),
|
|
2140
2344
|
clock: this.clock,
|
|
@@ -2159,20 +2363,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2159
2363
|
*
|
|
2160
2364
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2161
2365
|
* for (const e of swept.entries) {
|
|
2162
|
-
*
|
|
2366
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
2367
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2163
2368
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2164
2369
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2165
2370
|
* }
|
|
2166
2371
|
* }
|
|
2167
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
2372
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2168
2373
|
* ```
|
|
2169
2374
|
*
|
|
2170
2375
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2171
2376
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2172
2377
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2173
|
-
* `limit` (default 100) cut the pass short and more
|
|
2174
|
-
*
|
|
2175
|
-
*
|
|
2378
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
2379
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
2380
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
2381
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2176
2382
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2177
2383
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2178
2384
|
*
|
|
@@ -2182,9 +2388,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2182
2388
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2183
2389
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2184
2390
|
*
|
|
2185
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
2391
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2186
2392
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2187
|
-
* first, since nothing else
|
|
2393
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
2394
|
+
* only
|
|
2395
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2188
2396
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2189
2397
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2190
2398
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2205,7 +2413,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2205
2413
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2206
2414
|
const result = await retireExpired(
|
|
2207
2415
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2208
|
-
{ ...opts, now: opts.now
|
|
2416
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2209
2417
|
);
|
|
2210
2418
|
if (opts.dryRun !== true) {
|
|
2211
2419
|
for (const entry of result.entries) {
|
|
@@ -2234,7 +2442,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2234
2442
|
* A pin answers with no timed bound, until it is told or both caches let it go.
|
|
2235
2443
|
* - **Another process.** Erasing on one box invalidates nothing on the others; each store bounds its own
|
|
2236
2444
|
* staleness by `cache.genTtlMs`, and a store with no registry (a bare `IStorageDriver`), with
|
|
2237
|
-
* `cache.genTtlMs: 0`, or on a
|
|
2445
|
+
* `cache.genTtlMs: 0`, or on a `StorageChunkSource` of your own has no bound at all: it converges only when a read happens to miss its caches. If a compliance deadline depends on every
|
|
2238
2446
|
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
2239
2447
|
*
|
|
2240
2448
|
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
@@ -2252,7 +2460,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2252
2460
|
* ```
|
|
2253
2461
|
*/
|
|
2254
2462
|
invalidate(ref) {
|
|
2255
|
-
|
|
2463
|
+
ref = validateRef(ref, "invalidate");
|
|
2256
2464
|
refuseReservedNamespace(ref.namespace);
|
|
2257
2465
|
this.engine.invalidate(ref);
|
|
2258
2466
|
}
|
|
@@ -2267,8 +2475,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2267
2475
|
async pinSegment(ref, leaseUntil, named) {
|
|
2268
2476
|
const crbm = this.crbmSource;
|
|
2269
2477
|
if (crbm === void 0) {
|
|
2270
|
-
throw new
|
|
2271
|
-
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (
|
|
2478
|
+
throw new UnsupportedError2(
|
|
2479
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (the store cannot pin a StorageChunkSource of your own)"
|
|
2272
2480
|
);
|
|
2273
2481
|
}
|
|
2274
2482
|
let hold;
|
|
@@ -2276,7 +2484,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2276
2484
|
if (leaseUntil !== void 0) {
|
|
2277
2485
|
const registry = this.requireRegistry("pin({ leaseUntil })");
|
|
2278
2486
|
if (registry.capabilities().canWrite === false) {
|
|
2279
|
-
throw new
|
|
2487
|
+
throw new UnsupportedError2(
|
|
2280
2488
|
"pin({ leaseUntil }) needs a registry that can write a row, and this store's cannot in this runtime"
|
|
2281
2489
|
);
|
|
2282
2490
|
}
|
|
@@ -2312,14 +2520,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2312
2520
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2313
2521
|
throw err;
|
|
2314
2522
|
}
|
|
2315
|
-
const pinnedAt = {
|
|
2316
|
-
generation: at?.generation ?? null,
|
|
2317
|
-
version: at?.version ?? null,
|
|
2318
|
-
fingerprint: at?.fingerprint ?? null
|
|
2319
|
-
};
|
|
2523
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2320
2524
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2321
2525
|
return makeSegment({
|
|
2322
2526
|
engine: this.engineWithPins(pins),
|
|
2527
|
+
origin: this.origin,
|
|
2323
2528
|
ref,
|
|
2324
2529
|
clock: this.clock,
|
|
2325
2530
|
metrics: this.metrics,
|
|
@@ -2347,8 +2552,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2347
2552
|
engineWithPins(pins) {
|
|
2348
2553
|
const crbm = this.crbmSource;
|
|
2349
2554
|
if (crbm === void 0) {
|
|
2350
|
-
throw new
|
|
2351
|
-
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (
|
|
2555
|
+
throw new UnsupportedError2(
|
|
2556
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (the store cannot pin a StorageChunkSource of your own)"
|
|
2352
2557
|
);
|
|
2353
2558
|
}
|
|
2354
2559
|
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
@@ -2385,8 +2590,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2385
2590
|
if (held.has(key)) {
|
|
2386
2591
|
const before = held.get(key);
|
|
2387
2592
|
if (!samePin(before, at)) {
|
|
2388
|
-
throw new
|
|
2389
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
2593
|
+
throw new ValidationError5(
|
|
2594
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2390
2595
|
);
|
|
2391
2596
|
}
|
|
2392
2597
|
} else {
|
|
@@ -2404,7 +2609,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2404
2609
|
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
2405
2610
|
requireRegistry(op) {
|
|
2406
2611
|
if (this.registry === void 0) {
|
|
2407
|
-
throw new
|
|
2612
|
+
throw new UnsupportedError2(
|
|
2408
2613
|
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
2409
2614
|
);
|
|
2410
2615
|
}
|
|
@@ -2417,8 +2622,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2417
2622
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2418
2623
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2419
2624
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2420
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2421
|
-
*
|
|
2625
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
2626
|
+
* dropped) are skipped. Pair
|
|
2627
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2422
2628
|
*
|
|
2423
2629
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2424
2630
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2435,9 +2641,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2435
2641
|
* be used is counted in `generationsUnchecked`, and none of its objects is read.
|
|
2436
2642
|
*
|
|
2437
2643
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
2438
|
-
* `BudgetExceededError` past that rather than report a partial scan as a whole one.
|
|
2439
|
-
*
|
|
2440
|
-
*
|
|
2644
|
+
* `BudgetExceededError` (`option: 'maxScanSegments'`) past that rather than report a partial scan as a whole one.
|
|
2645
|
+
* Narrow the scan with `namespace`, or raise `maxScanSegments` when the fleet really is that large and the memory
|
|
2646
|
+
* is there.
|
|
2441
2647
|
*/
|
|
2442
2648
|
async checkConsistency(options = {}) {
|
|
2443
2649
|
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
@@ -2451,6 +2657,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2451
2657
|
options
|
|
2452
2658
|
);
|
|
2453
2659
|
}
|
|
2660
|
+
/**
|
|
2661
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
2662
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
2663
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
2664
|
+
*
|
|
2665
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
2666
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
2667
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
2668
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
2669
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
2670
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
2671
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
2672
|
+
*
|
|
2673
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
2674
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
2675
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
2676
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
2677
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
2678
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
2679
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
2680
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
2681
|
+
* timing.
|
|
2682
|
+
*
|
|
2683
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
2684
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
2685
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
2686
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
2687
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
2688
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
2689
|
+
*
|
|
2690
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
2691
|
+
*/
|
|
2692
|
+
async preflight() {
|
|
2693
|
+
const deps = this.lifecycleDeps("preflight");
|
|
2694
|
+
const backend = this.backend;
|
|
2695
|
+
let checkLifecycle;
|
|
2696
|
+
try {
|
|
2697
|
+
const method = backend?.checkLifecycle;
|
|
2698
|
+
if (typeof method === "function")
|
|
2699
|
+
checkLifecycle = method.bind(backend);
|
|
2700
|
+
} catch {
|
|
2701
|
+
}
|
|
2702
|
+
return runPreflight({
|
|
2703
|
+
storage: deps.storage,
|
|
2704
|
+
registry: deps.registry,
|
|
2705
|
+
clock: this.clock,
|
|
2706
|
+
entropy: this.entropy,
|
|
2707
|
+
checkLifecycle
|
|
2708
|
+
});
|
|
2709
|
+
}
|
|
2454
2710
|
/**
|
|
2455
2711
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2456
2712
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2459,15 +2715,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2459
2715
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2460
2716
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2461
2717
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2462
|
-
*
|
|
2463
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
2464
|
-
*
|
|
2718
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
2719
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
2720
|
+
* check `failed`. The `export-segments`
|
|
2721
|
+
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2465
2722
|
*/
|
|
2466
2723
|
async exportSegments(sink, options = {}) {
|
|
2467
|
-
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments);
|
|
2724
|
+
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments, [], MISPLACED.exportSegments);
|
|
2468
2725
|
const opts = picked(options, CALL_KEYS.exportSegments);
|
|
2469
2726
|
const registry = this.requireRegistry("exportSegments");
|
|
2470
|
-
return runExport(this, registry, sink, { ...opts, codec:
|
|
2727
|
+
return runExport(this, registry, sink, { ...opts, codec: roaringCodec });
|
|
2471
2728
|
}
|
|
2472
2729
|
};
|
|
2473
2730
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
@@ -2495,12 +2752,23 @@ var readOptions = (options) => ({
|
|
|
2495
2752
|
});
|
|
2496
2753
|
var NO_SEGMENTS = [];
|
|
2497
2754
|
function segmentList(list, what) {
|
|
2498
|
-
if (!Array.isArray(list)) throw new
|
|
2755
|
+
if (!Array.isArray(list)) throw new ValidationError5(`${what} must be an array of segments`);
|
|
2499
2756
|
return list;
|
|
2500
2757
|
}
|
|
2758
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
2759
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
2760
|
+
function readsFrom(stream, origin, keys) {
|
|
2761
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
2762
|
+
return stream;
|
|
2763
|
+
}
|
|
2501
2764
|
function operandsLabel(verb) {
|
|
2502
2765
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2503
2766
|
}
|
|
2767
|
+
function refuseNoOperands(verb) {
|
|
2768
|
+
throw new ValidationError5(
|
|
2769
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
2770
|
+
);
|
|
2771
|
+
}
|
|
2504
2772
|
var AS_IDS = {
|
|
2505
2773
|
failing,
|
|
2506
2774
|
guard: guardIds,
|
|
@@ -2517,9 +2785,9 @@ var CombineChunks = class {
|
|
|
2517
2785
|
var AS_CHUNKS = {
|
|
2518
2786
|
guard: guardItems,
|
|
2519
2787
|
failing: (err) => refusing(err)(),
|
|
2520
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2521
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2522
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
2788
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
2789
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
2790
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2523
2791
|
};
|
|
2524
2792
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2525
2793
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2535,6 +2803,8 @@ var minting = false;
|
|
|
2535
2803
|
var Segment = class _Segment {
|
|
2536
2804
|
metricsOn;
|
|
2537
2805
|
engine;
|
|
2806
|
+
/** The store that made this handle, by reference; a combine takes only handles of its own store. */
|
|
2807
|
+
origin;
|
|
2538
2808
|
ref;
|
|
2539
2809
|
clock;
|
|
2540
2810
|
metrics;
|
|
@@ -2568,6 +2838,7 @@ var Segment = class _Segment {
|
|
|
2568
2838
|
pinnedAt: segment.pinnedAt,
|
|
2569
2839
|
leaseError: () => segment.leaseError()
|
|
2570
2840
|
});
|
|
2841
|
+
originOf = (segment) => segment.origin;
|
|
2571
2842
|
}
|
|
2572
2843
|
/**
|
|
2573
2844
|
* Not constructible: a handle comes from {@link CloudRoaring.segment}, which wires it to the store's engine,
|
|
@@ -2575,11 +2846,12 @@ var Segment = class _Segment {
|
|
|
2575
2846
|
*/
|
|
2576
2847
|
constructor(parts) {
|
|
2577
2848
|
if (!minting) {
|
|
2578
|
-
throw new
|
|
2849
|
+
throw new ValidationError5(
|
|
2579
2850
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2580
2851
|
);
|
|
2581
2852
|
}
|
|
2582
2853
|
this.engine = parts.engine;
|
|
2854
|
+
this.origin = parts.origin;
|
|
2583
2855
|
this.ref = parts.ref;
|
|
2584
2856
|
this.clock = parts.clock;
|
|
2585
2857
|
this.metrics = parts.metrics;
|
|
@@ -2613,7 +2885,7 @@ var Segment = class _Segment {
|
|
|
2613
2885
|
*
|
|
2614
2886
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2615
2887
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2616
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
2888
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2617
2889
|
*
|
|
2618
2890
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2619
2891
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2653,9 +2925,7 @@ var Segment = class _Segment {
|
|
|
2653
2925
|
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
2654
2926
|
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
2655
2927
|
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
2656
|
-
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store
|
|
2657
|
-
* operand reads it through that store's reader and chunk cache, which only that store's own writes and its
|
|
2658
|
-
* `invalidate` reach. Where the
|
|
2928
|
+
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store refuses the pin. Where the
|
|
2659
2929
|
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
2660
2930
|
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
2661
2931
|
*
|
|
@@ -2675,7 +2945,7 @@ var Segment = class _Segment {
|
|
|
2675
2945
|
* or misfiled one.
|
|
2676
2946
|
*
|
|
2677
2947
|
* Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
|
|
2678
|
-
* {@link UnsupportedError} on a store wired with a
|
|
2948
|
+
* {@link UnsupportedError} on a store wired with a `StorageChunkSource` of your own, which it cannot pin.
|
|
2679
2949
|
*/
|
|
2680
2950
|
async pin(options) {
|
|
2681
2951
|
this.assertLease();
|
|
@@ -2690,11 +2960,11 @@ var Segment = class _Segment {
|
|
|
2690
2960
|
leaseUntilOf(op, options) {
|
|
2691
2961
|
if (options === void 0 || options === null) return void 0;
|
|
2692
2962
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2693
|
-
throw new
|
|
2963
|
+
throw new ValidationError5(`${op}: options must be an object such as { leaseUntil }`);
|
|
2694
2964
|
}
|
|
2695
2965
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2696
2966
|
if (unknown.length > 0) {
|
|
2697
|
-
throw new
|
|
2967
|
+
throw new ValidationError5(
|
|
2698
2968
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2699
2969
|
);
|
|
2700
2970
|
}
|
|
@@ -2702,17 +2972,17 @@ var Segment = class _Segment {
|
|
|
2702
2972
|
if (until === void 0) return void 0;
|
|
2703
2973
|
const now = this.clock.now();
|
|
2704
2974
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2705
|
-
throw new
|
|
2975
|
+
throw new ValidationError5(
|
|
2706
2976
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2707
2977
|
);
|
|
2708
2978
|
}
|
|
2709
2979
|
if (until <= now) {
|
|
2710
|
-
throw new
|
|
2980
|
+
throw new ValidationError5(
|
|
2711
2981
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2712
2982
|
);
|
|
2713
2983
|
}
|
|
2714
2984
|
if (until > now + MAX_LEASE_MS) {
|
|
2715
|
-
throw new
|
|
2985
|
+
throw new ValidationError5(
|
|
2716
2986
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2717
2987
|
);
|
|
2718
2988
|
}
|
|
@@ -2763,18 +3033,12 @@ var Segment = class _Segment {
|
|
|
2763
3033
|
return void 0;
|
|
2764
3034
|
}
|
|
2765
3035
|
/**
|
|
2766
|
-
* Refuse with a {@link ValidationError}
|
|
2767
|
-
*
|
|
3036
|
+
* Refuse with a {@link ValidationError} any of `handles` that this store did not make: a value that is not a segment,
|
|
3037
|
+
* a segment of another store (even one over the same backend) and a segment of another installed copy of the
|
|
3038
|
+
* package. It runs first in every combine, before the lease check reads the handles and before any request.
|
|
2768
3039
|
*/
|
|
2769
|
-
assertSegments(handles) {
|
|
2770
|
-
for (const h of handles)
|
|
2771
|
-
if (h instanceof _Segment || typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
2772
|
-
continue;
|
|
2773
|
-
}
|
|
2774
|
-
throw new ValidationError4(
|
|
2775
|
-
"an operand must be a segment from `store.segment()` of a compatible copy of @cloudbitmaps/roaring; install one version of it"
|
|
2776
|
-
);
|
|
2777
|
-
}
|
|
3040
|
+
assertSegments(verb, role, handles) {
|
|
3041
|
+
for (const h of handles) assertOneStore(verb, role, h, this.origin);
|
|
2778
3042
|
}
|
|
2779
3043
|
/**
|
|
2780
3044
|
* Throw the first lease error among `handles`: this handle and every operand and exclude of a call. It runs before
|
|
@@ -2796,10 +3060,10 @@ var Segment = class _Segment {
|
|
|
2796
3060
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2797
3061
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2798
3062
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2799
|
-
* than `generation` and `
|
|
3063
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2800
3064
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2801
3065
|
*
|
|
2802
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
3066
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2803
3067
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2804
3068
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2805
3069
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2812,18 +3076,32 @@ var Segment = class _Segment {
|
|
|
2812
3076
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2813
3077
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2814
3078
|
* is in `options`, the second.
|
|
3079
|
+
*
|
|
3080
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
3081
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
3082
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2815
3083
|
*/
|
|
2816
3084
|
async pinAt(at, options) {
|
|
2817
3085
|
this.assertLease();
|
|
2818
|
-
const { generation, fingerprint } = at ?? {};
|
|
2819
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
3086
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
3087
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2820
3088
|
if (unknown.length > 0) {
|
|
2821
|
-
throw new
|
|
2822
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
3089
|
+
throw new ValidationError5(
|
|
3090
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
3091
|
+
);
|
|
3092
|
+
}
|
|
3093
|
+
if (generation === null) {
|
|
3094
|
+
throw new ValidationError5(
|
|
3095
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
3096
|
+
);
|
|
3097
|
+
}
|
|
3098
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
3099
|
+
throw new ValidationError5(
|
|
3100
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2823
3101
|
);
|
|
2824
3102
|
}
|
|
2825
3103
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2826
|
-
throw new
|
|
3104
|
+
throw new ValidationError5(
|
|
2827
3105
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2828
3106
|
);
|
|
2829
3107
|
}
|
|
@@ -2840,17 +3118,41 @@ var Segment = class _Segment {
|
|
|
2840
3118
|
return segmentKey(this.ref);
|
|
2841
3119
|
}
|
|
2842
3120
|
/**
|
|
2843
|
-
* Refuse an `*Into`
|
|
2844
|
-
*
|
|
2845
|
-
*
|
|
3121
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
3122
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
3123
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
3124
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
3125
|
+
* first. Called by the three `*Into` verbs only.
|
|
2846
3126
|
*
|
|
2847
3127
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2848
3128
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
2849
3129
|
* awaiting.
|
|
2850
3130
|
*/
|
|
2851
|
-
assertIntoHandles(dest, operands) {
|
|
2852
|
-
this.assertSegments([dest
|
|
2853
|
-
this.
|
|
3131
|
+
assertIntoHandles(verb, dest, operands, excludes) {
|
|
3132
|
+
this.assertSegments(verb, "the destination", [dest]);
|
|
3133
|
+
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
3134
|
+
this.assertSegments(verb, "an exclude", excludes);
|
|
3135
|
+
const base = verb.slice(0, -"Into".length);
|
|
3136
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
3137
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
3138
|
+
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
3139
|
+
}
|
|
3140
|
+
/**
|
|
3141
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
3142
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
3143
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
3144
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
3145
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
3146
|
+
*/
|
|
3147
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
3148
|
+
const key = dest.key();
|
|
3149
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
3150
|
+
if (role === void 0) return;
|
|
3151
|
+
const { namespace, segment } = dest.ref;
|
|
3152
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
3153
|
+
throw new ValidationError5(
|
|
3154
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
3155
|
+
);
|
|
2854
3156
|
}
|
|
2855
3157
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2856
3158
|
guarded(out, handles, result) {
|
|
@@ -2953,7 +3255,7 @@ var Segment = class _Segment {
|
|
|
2953
3255
|
iterate(options) {
|
|
2954
3256
|
const lease = this.leaseError();
|
|
2955
3257
|
if (lease !== void 0) return failing(lease);
|
|
2956
|
-
const bad = optionsError(options, "iterate", CALL_KEYS.range);
|
|
3258
|
+
const bad = optionsError(options, "iterate", CALL_KEYS.range, [], MISPLACED.range);
|
|
2957
3259
|
if (bad !== void 0) return failing(bad);
|
|
2958
3260
|
const range = options == null ? void 0 : rangeOf(options);
|
|
2959
3261
|
const none = range === void 0 || range.after === void 0 && range.through === void 0;
|
|
@@ -2963,7 +3265,11 @@ var Segment = class _Segment {
|
|
|
2963
3265
|
engine.iterate(ref, bounds),
|
|
2964
3266
|
() => engine.iterateBatches(ref, bounds)
|
|
2965
3267
|
);
|
|
2966
|
-
return
|
|
3268
|
+
return readsFrom(
|
|
3269
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
3270
|
+
this.origin,
|
|
3271
|
+
[this.key()]
|
|
3272
|
+
);
|
|
2967
3273
|
}
|
|
2968
3274
|
/**
|
|
2969
3275
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -2998,11 +3304,11 @@ var Segment = class _Segment {
|
|
|
2998
3304
|
everyNth(n, options) {
|
|
2999
3305
|
const lease = this.leaseError();
|
|
3000
3306
|
if (lease !== void 0) return refusing(lease)();
|
|
3001
|
-
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
3307
|
+
const bad = optionsError(options, "everyNth", CALL_KEYS.range, [], MISPLACED.range);
|
|
3002
3308
|
if (bad !== void 0) return refusing(bad)();
|
|
3003
3309
|
if (this.pinnedAt === void 0) {
|
|
3004
3310
|
return refusing(
|
|
3005
|
-
new
|
|
3311
|
+
new UnsupportedError2(
|
|
3006
3312
|
"everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
|
|
3007
3313
|
)
|
|
3008
3314
|
)();
|
|
@@ -3012,7 +3318,11 @@ var Segment = class _Segment {
|
|
|
3012
3318
|
n,
|
|
3013
3319
|
options == null ? void 0 : rangeOf(options)
|
|
3014
3320
|
);
|
|
3015
|
-
return
|
|
3321
|
+
return readsFrom(
|
|
3322
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
3323
|
+
this.origin,
|
|
3324
|
+
[this.key()]
|
|
3325
|
+
);
|
|
3016
3326
|
}
|
|
3017
3327
|
/**
|
|
3018
3328
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3048,9 +3358,20 @@ var Segment = class _Segment {
|
|
|
3048
3358
|
* requests held ahead per operand: a positive integer, 32 by default).
|
|
3049
3359
|
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
|
|
3050
3360
|
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
3361
|
+
*
|
|
3362
|
+
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3363
|
+
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3364
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
3365
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3051
3366
|
*/
|
|
3052
3367
|
intersect(others, options) {
|
|
3053
|
-
const bad = optionsError(
|
|
3368
|
+
const bad = optionsError(
|
|
3369
|
+
options,
|
|
3370
|
+
"intersect",
|
|
3371
|
+
CALL_KEYS.combine,
|
|
3372
|
+
["allowAbsentOperands"],
|
|
3373
|
+
MISPLACED.combine
|
|
3374
|
+
);
|
|
3054
3375
|
if (bad !== void 0) return failing(bad);
|
|
3055
3376
|
return this.combineAs(AS_IDS, "intersect", others, options ?? void 0);
|
|
3056
3377
|
}
|
|
@@ -3073,6 +3394,16 @@ var Segment = class _Segment {
|
|
|
3073
3394
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3074
3395
|
*
|
|
3075
3396
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
3397
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
3398
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
3399
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
3400
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
3401
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
3402
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
3403
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
3404
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
3405
|
+
* once, at one instant.
|
|
3406
|
+
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3076
3407
|
*/
|
|
3077
3408
|
async intersectInto(dest, others, options) {
|
|
3078
3409
|
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
@@ -3086,10 +3417,17 @@ var Segment = class _Segment {
|
|
|
3086
3417
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3087
3418
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3088
3419
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3089
|
-
* segment once (`unionInto`, or a load) is the cheaper shape.
|
|
3420
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
3421
|
+
* empty `others` reads `this` itself.
|
|
3090
3422
|
*/
|
|
3091
3423
|
union(others, options) {
|
|
3092
|
-
const bad = optionsError(
|
|
3424
|
+
const bad = optionsError(
|
|
3425
|
+
options,
|
|
3426
|
+
"union",
|
|
3427
|
+
CALL_KEYS.combine,
|
|
3428
|
+
["allowAbsentOperands"],
|
|
3429
|
+
MISPLACED.combine
|
|
3430
|
+
);
|
|
3093
3431
|
if (bad !== void 0) return failing(bad);
|
|
3094
3432
|
return this.combineAs(AS_IDS, "union", others, options ?? void 0);
|
|
3095
3433
|
}
|
|
@@ -3108,18 +3446,34 @@ var Segment = class _Segment {
|
|
|
3108
3446
|
*
|
|
3109
3447
|
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
3110
3448
|
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
3449
|
+
*
|
|
3450
|
+
* It takes the options {@link intersect} takes but `exclude`, since `excludes` is its subtraction: `exclude` or a write
|
|
3451
|
+
* option fails the stream with {@link ValidationError} when it is first read, before any request.
|
|
3111
3452
|
*/
|
|
3112
3453
|
andNot(excludes, options) {
|
|
3113
|
-
const bad = optionsError(
|
|
3454
|
+
const bad = optionsError(
|
|
3455
|
+
options,
|
|
3456
|
+
"andNot",
|
|
3457
|
+
CALL_KEYS.andNot,
|
|
3458
|
+
["allowAbsentOperands"],
|
|
3459
|
+
MISPLACED.andNot
|
|
3460
|
+
);
|
|
3114
3461
|
if (bad !== void 0) return failing(bad);
|
|
3115
3462
|
return this.combineAs(AS_IDS, "andNot", excludes, options ?? void 0);
|
|
3116
3463
|
}
|
|
3117
|
-
/**
|
|
3464
|
+
/**
|
|
3465
|
+
* Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. It takes every field
|
|
3466
|
+
* of {@link CombineOptions} but `exclude`, since `excludes` is its subtraction: `exclude` is refused with
|
|
3467
|
+
* {@link ValidationError} before any request.
|
|
3468
|
+
*/
|
|
3118
3469
|
async andNotInto(dest, excludes, options) {
|
|
3119
|
-
checkOptions(
|
|
3120
|
-
|
|
3121
|
-
"
|
|
3122
|
-
|
|
3470
|
+
checkOptions(
|
|
3471
|
+
options,
|
|
3472
|
+
"andNotInto",
|
|
3473
|
+
CALL_KEYS.andNotInto,
|
|
3474
|
+
["allowEmpty", "allowAbsentOperands"],
|
|
3475
|
+
MISPLACED.andNotInto
|
|
3476
|
+
);
|
|
3123
3477
|
return this.combineInto("andNotInto", dest, excludes, options);
|
|
3124
3478
|
}
|
|
3125
3479
|
/**
|
|
@@ -3131,8 +3485,11 @@ var Segment = class _Segment {
|
|
|
3131
3485
|
let handles;
|
|
3132
3486
|
try {
|
|
3133
3487
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3134
|
-
|
|
3135
|
-
|
|
3488
|
+
const others = segmentList(operands, operandsLabel(verb));
|
|
3489
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3490
|
+
handles = [this, ...others, ...exclude];
|
|
3491
|
+
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3492
|
+
this.assertSegments(verb, "an exclude", exclude);
|
|
3136
3493
|
this.assertLeases(handles);
|
|
3137
3494
|
} catch (err) {
|
|
3138
3495
|
return out.failing(err);
|
|
@@ -3143,24 +3500,33 @@ var Segment = class _Segment {
|
|
|
3143
3500
|
} catch (err) {
|
|
3144
3501
|
return out.failing(err);
|
|
3145
3502
|
}
|
|
3503
|
+
const keys = handles.map((h) => h.key());
|
|
3504
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3146
3505
|
if (verb === "andNot") {
|
|
3147
3506
|
const refs2 = operands.map((o) => o.ref);
|
|
3148
3507
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3149
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3508
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3150
3509
|
}
|
|
3151
3510
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3152
3511
|
const opts = this.refsIn(options, exclude);
|
|
3153
|
-
return
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3512
|
+
return known(
|
|
3513
|
+
this.guarded(
|
|
3514
|
+
out,
|
|
3515
|
+
handles,
|
|
3516
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3517
|
+
)
|
|
3157
3518
|
);
|
|
3158
3519
|
}
|
|
3159
3520
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
3160
3521
|
combineInto(name, dest, operands, options) {
|
|
3161
3522
|
const verb = name.slice(0, -"Into".length);
|
|
3162
|
-
const
|
|
3163
|
-
|
|
3523
|
+
const listed = segmentList(
|
|
3524
|
+
operands,
|
|
3525
|
+
verb === "andNot" ? `${name}: \`excludes\`` : `${name}: \`others\``
|
|
3526
|
+
);
|
|
3527
|
+
const excludes = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3528
|
+
const all = excludes.length > 0 ? [...listed, ...excludes] : listed;
|
|
3529
|
+
this.assertIntoHandles(name, dest, listed, excludes);
|
|
3164
3530
|
this.combineEngine([this, ...all]);
|
|
3165
3531
|
return this.timed(
|
|
3166
3532
|
name,
|
|
@@ -3175,18 +3541,64 @@ var Segment = class _Segment {
|
|
|
3175
3541
|
};
|
|
3176
3542
|
|
|
3177
3543
|
// src/bin/export-segments.ts
|
|
3544
|
+
var USAGE = `Usage: export-segments [--help]
|
|
3545
|
+
|
|
3546
|
+
Exports every registered segment's current generation from a local-filesystem store to a directory.
|
|
3547
|
+
Settings come from the environment:
|
|
3548
|
+
|
|
3549
|
+
CR_EXPORT_ROOT (required) the LocalFsStorage root, holding storage/ and registry/
|
|
3550
|
+
CR_EXPORT_OUT (required) the output directory for the dump
|
|
3551
|
+
CR_EXPORT_FORMAT roaring | ndjson (default: roaring)
|
|
3552
|
+
CR_EXPORT_NAMESPACE (optional) scope the export to one namespace
|
|
3553
|
+
|
|
3554
|
+
Example:
|
|
3555
|
+
CR_EXPORT_ROOT=./.cloudbitmaps CR_EXPORT_OUT=./dump export-segments
|
|
3556
|
+
`;
|
|
3557
|
+
function answerArgs(args) {
|
|
3558
|
+
const rest = args[0] === "--" ? args.slice(1) : args;
|
|
3559
|
+
if (rest.length === 0) return void 0;
|
|
3560
|
+
if (rest.length === 1 && (rest[0] === "--help" || rest[0] === "-h")) {
|
|
3561
|
+
return { stream: "stdout", text: USAGE, code: 0 };
|
|
3562
|
+
}
|
|
3563
|
+
return {
|
|
3564
|
+
stream: "stderr",
|
|
3565
|
+
text: `export-segments takes no arguments; it reads its settings from the environment
|
|
3566
|
+
|
|
3567
|
+
${USAGE}`,
|
|
3568
|
+
code: 2
|
|
3569
|
+
};
|
|
3570
|
+
}
|
|
3571
|
+
var EXPORT_VARIABLES = [
|
|
3572
|
+
"CR_EXPORT_ROOT",
|
|
3573
|
+
"CR_EXPORT_OUT",
|
|
3574
|
+
"CR_EXPORT_FORMAT",
|
|
3575
|
+
"CR_EXPORT_NAMESPACE"
|
|
3576
|
+
];
|
|
3178
3577
|
function parseConfig(env) {
|
|
3578
|
+
const unknown = Object.keys(env).filter(
|
|
3579
|
+
(name) => name.startsWith("CR_EXPORT_") && !EXPORT_VARIABLES.includes(name)
|
|
3580
|
+
);
|
|
3581
|
+
if (unknown.length > 0) {
|
|
3582
|
+
const which = unknown.length > 1 ? `${unknown.join(", ")} are not settings export-segments takes.` : `${unknown[0]} is not a setting export-segments takes.`;
|
|
3583
|
+
throw new ValidationError6(
|
|
3584
|
+
`${which} It takes ${EXPORT_VARIABLES.join(", ")}; run export-segments --help`
|
|
3585
|
+
);
|
|
3586
|
+
}
|
|
3179
3587
|
const root = env.CR_EXPORT_ROOT;
|
|
3180
3588
|
if (root === void 0 || root === "") {
|
|
3181
|
-
throw new
|
|
3589
|
+
throw new ValidationError6(
|
|
3590
|
+
"CR_EXPORT_ROOT is not set: the directory that holds the store's storage/ and registry/. export-segments also needs CR_EXPORT_OUT; run export-segments --help"
|
|
3591
|
+
);
|
|
3182
3592
|
}
|
|
3183
3593
|
const out = env.CR_EXPORT_OUT;
|
|
3184
3594
|
if (out === void 0 || out === "") {
|
|
3185
|
-
throw new
|
|
3595
|
+
throw new ValidationError6(
|
|
3596
|
+
"CR_EXPORT_OUT is not set: the directory to write the dump to. export-segments also needs CR_EXPORT_ROOT; run export-segments --help"
|
|
3597
|
+
);
|
|
3186
3598
|
}
|
|
3187
3599
|
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3188
3600
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3189
|
-
throw new
|
|
3601
|
+
throw new ValidationError6(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3190
3602
|
}
|
|
3191
3603
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
3192
3604
|
return { root, out, format, namespace };
|
|
@@ -3201,7 +3613,7 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3201
3613
|
}
|
|
3202
3614
|
const found = await lstat(dir);
|
|
3203
3615
|
const refuse = (why) => {
|
|
3204
|
-
throw new
|
|
3616
|
+
throw new ValidationError6(`the export directory is not one this tool made: ${why}`);
|
|
3205
3617
|
};
|
|
3206
3618
|
if (found.isSymbolicLink()) refuse("a namespace directory is a symbolic link");
|
|
3207
3619
|
if (!found.isDirectory()) refuse("a namespace path is not a directory");
|
|
@@ -3216,7 +3628,7 @@ function fsSink(out) {
|
|
|
3216
3628
|
async open(ref, ext) {
|
|
3217
3629
|
const part = namespacePathPart(ref.namespace);
|
|
3218
3630
|
if (part.toLowerCase() === MANIFEST_NAME) {
|
|
3219
|
-
throw new
|
|
3631
|
+
throw new ValidationError6(
|
|
3220
3632
|
`a namespace named ${MANIFEST_NAME} (in any letter case) cannot be exported to a directory: its folder would be the manifest's path`
|
|
3221
3633
|
);
|
|
3222
3634
|
}
|
|
@@ -3261,10 +3673,10 @@ async function requireStoreRoot(root) {
|
|
|
3261
3673
|
try {
|
|
3262
3674
|
entries = await readdir(root);
|
|
3263
3675
|
} catch {
|
|
3264
|
-
throw new
|
|
3676
|
+
throw new ValidationError6("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3265
3677
|
}
|
|
3266
3678
|
if (entries.length === 0) {
|
|
3267
|
-
throw new
|
|
3679
|
+
throw new ValidationError6(
|
|
3268
3680
|
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3269
3681
|
);
|
|
3270
3682
|
}
|
|
@@ -3280,7 +3692,7 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3280
3692
|
const store = new CloudRoaring({ storage });
|
|
3281
3693
|
const priorManifest = await lstat(join(config.out, MANIFEST_NAME)).catch(() => null);
|
|
3282
3694
|
if (priorManifest?.isDirectory() === true) {
|
|
3283
|
-
throw new
|
|
3695
|
+
throw new ValidationError6(
|
|
3284
3696
|
`${join(config.out, MANIFEST_NAME)} is a directory, so the manifest cannot be written there: remove or rename it, or export to a fresh directory`
|
|
3285
3697
|
);
|
|
3286
3698
|
}
|
|
@@ -3326,6 +3738,15 @@ function invokedAsCli() {
|
|
|
3326
3738
|
}
|
|
3327
3739
|
}
|
|
3328
3740
|
if (invokedAsCli()) {
|
|
3741
|
+
const answer = answerArgs(process.argv.slice(2));
|
|
3742
|
+
if (answer !== void 0) {
|
|
3743
|
+
process[answer.stream].write(answer.text);
|
|
3744
|
+
process.exitCode = answer.code;
|
|
3745
|
+
} else {
|
|
3746
|
+
runCli();
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
function runCli() {
|
|
3329
3750
|
main().then((manifest) => {
|
|
3330
3751
|
if (manifest.failed.length > 0) {
|
|
3331
3752
|
process.stderr.write(
|
|
@@ -3341,6 +3762,8 @@ if (invokedAsCli()) {
|
|
|
3341
3762
|
});
|
|
3342
3763
|
}
|
|
3343
3764
|
export {
|
|
3765
|
+
USAGE,
|
|
3766
|
+
answerArgs,
|
|
3344
3767
|
fsSink,
|
|
3345
3768
|
main,
|
|
3346
3769
|
parseConfig
|