@cloudbitmaps/roaring 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +38 -23
- package/README.md +27 -9
- package/dist/bin/export-segments.js +683 -260
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +395 -171
- package/dist/index.js +633 -279
- package/dist/index.js.map +4 -4
- package/dist/roaring-codec.d.ts +23 -22
- package/package.json +4 -3
- package/dist/bin/export-segments.d.ts +0 -25
- package/dist/lease-guards.d.ts +0 -46
- package/dist/open-charging-storage.d.ts +0 -37
- package/dist/option-keys.d.ts +0 -19
- package/dist/options.d.ts +0 -30
- package/dist/portable/decode.d.ts +0 -54
- package/dist/portable/layout.d.ts +0 -52
- package/dist/reserved-namespace.d.ts +0 -3
- package/dist/system-clock.d.ts +0 -39
- package/dist/testing/conformance.d.ts +0 -118
- package/dist/testing/fuzz-codec.d.ts +0 -23
- package/dist/wire-store.d.ts +0 -38
package/dist/index.js
CHANGED
|
@@ -11,17 +11,20 @@ import {
|
|
|
11
11
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
12
12
|
SegmentEngine,
|
|
13
13
|
LEASE_SKEW_MS,
|
|
14
|
+
BudgetExceededError,
|
|
14
15
|
LeaseExpiredError,
|
|
15
16
|
MAX_LEASE_MS,
|
|
16
17
|
NotFoundError,
|
|
17
|
-
UnsupportedError,
|
|
18
|
-
ValidationError as
|
|
18
|
+
UnsupportedError as UnsupportedError2,
|
|
19
|
+
ValidationError as ValidationError5,
|
|
20
|
+
errorKind,
|
|
19
21
|
WriteConflictError,
|
|
20
22
|
checkBudget,
|
|
21
23
|
collectWithinBudget,
|
|
22
24
|
excludingReservedRows,
|
|
23
25
|
dropSegment,
|
|
24
26
|
eraseIdFromSegment,
|
|
27
|
+
findIdUnderTombstone,
|
|
25
28
|
judgeLoad,
|
|
26
29
|
loadSegment,
|
|
27
30
|
loadSegmentChunks,
|
|
@@ -42,7 +45,12 @@ import {
|
|
|
42
45
|
splitId,
|
|
43
46
|
takeLease
|
|
44
47
|
} from "@cloudbitmaps/core";
|
|
45
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
pointerIdOf,
|
|
50
|
+
requireBool,
|
|
51
|
+
requirePositiveInt,
|
|
52
|
+
validateSegmentRef
|
|
53
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
46
54
|
|
|
47
55
|
// src/open-charging-storage.ts
|
|
48
56
|
var OpenChargingStorage = class {
|
|
@@ -58,6 +66,7 @@ var OpenChargingStorage = class {
|
|
|
58
66
|
}
|
|
59
67
|
inner;
|
|
60
68
|
charge;
|
|
69
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
61
70
|
free = /* @__PURE__ */ new Set();
|
|
62
71
|
capabilities() {
|
|
63
72
|
return this.inner.capabilities();
|
|
@@ -70,9 +79,15 @@ var OpenChargingStorage = class {
|
|
|
70
79
|
getRange(key, offset, length) {
|
|
71
80
|
return this.inner.getRange(key, offset, length);
|
|
72
81
|
}
|
|
73
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
84
|
+
* generation is charged at its first open only.
|
|
85
|
+
*/
|
|
74
86
|
async getTail(key, maxBytes) {
|
|
75
|
-
if (!this.free.has(key.generation))
|
|
87
|
+
if (!this.free.has(key.generation)) {
|
|
88
|
+
this.charge();
|
|
89
|
+
this.free.add(key.generation);
|
|
90
|
+
}
|
|
76
91
|
return this.inner.getTail(key, maxBytes);
|
|
77
92
|
}
|
|
78
93
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -86,7 +101,12 @@ var OpenChargingStorage = class {
|
|
|
86
101
|
|
|
87
102
|
// src/index.ts
|
|
88
103
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
89
|
-
import {
|
|
104
|
+
import {
|
|
105
|
+
destroySegment as coreDestroySegment,
|
|
106
|
+
eraseNamespace as coreEraseNamespace
|
|
107
|
+
} from "@cloudbitmaps/core";
|
|
108
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
109
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
90
110
|
|
|
91
111
|
// src/reserved-namespace.ts
|
|
92
112
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -101,7 +121,7 @@ function refuseReservedNamespace(namespace) {
|
|
|
101
121
|
|
|
102
122
|
// src/roaring-codec.ts
|
|
103
123
|
import roaring from "roaring";
|
|
104
|
-
import { IntegrityError as IntegrityError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
124
|
+
import { IntegrityError as IntegrityError2, ValidationError as ValidationError2, decodeSerialized } from "@cloudbitmaps/core";
|
|
105
125
|
|
|
106
126
|
// src/portable/layout.ts
|
|
107
127
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -349,10 +369,6 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
349
369
|
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
350
370
|
}
|
|
351
371
|
}
|
|
352
|
-
/** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
|
|
353
|
-
unwrap() {
|
|
354
|
-
return this.bitmap;
|
|
355
|
-
}
|
|
356
372
|
serialize() {
|
|
357
373
|
return this.bitmap.serialize(SerializationFormat.portable);
|
|
358
374
|
}
|
|
@@ -457,7 +473,7 @@ var SafeBitmap = class _SafeBitmap {
|
|
|
457
473
|
}
|
|
458
474
|
};
|
|
459
475
|
function bitmapAsLoadInput(input) {
|
|
460
|
-
return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
|
|
476
|
+
return input instanceof RoaringBitmap32 || input instanceof ReadonlyRoaringView ? { bitmap: input } : input;
|
|
461
477
|
}
|
|
462
478
|
var roaringCodec = {
|
|
463
479
|
empty: () => SafeBitmap.empty(),
|
|
@@ -465,8 +481,33 @@ var roaringCodec = {
|
|
|
465
481
|
safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
466
482
|
owns: (bitmap) => bitmap instanceof SafeBitmap
|
|
467
483
|
};
|
|
484
|
+
var ReadonlyRoaringView = class {
|
|
485
|
+
#bitmap;
|
|
486
|
+
constructor(bitmap) {
|
|
487
|
+
this.#bitmap = bitmap;
|
|
488
|
+
}
|
|
489
|
+
get size() {
|
|
490
|
+
return this.#bitmap.size;
|
|
491
|
+
}
|
|
492
|
+
has(id) {
|
|
493
|
+
return this.#bitmap.has(id);
|
|
494
|
+
}
|
|
495
|
+
[Symbol.iterator]() {
|
|
496
|
+
return this.#bitmap[Symbol.iterator]();
|
|
497
|
+
}
|
|
498
|
+
/** The portable bytes, for exactly the argument `'portable'`; any other call is a `ValidationError`. */
|
|
499
|
+
serialize(...args) {
|
|
500
|
+
if (args.length !== 1 || args[0] !== "portable") {
|
|
501
|
+
const got = args.length === 0 ? "no argument" : args.map((a) => typeof a === "string" ? `'${a}'` : typeof a).join(", ");
|
|
502
|
+
throw new ValidationError2(
|
|
503
|
+
`a decoded bitmap serializes only as serialize('portable'); got ${got}`
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
return this.#bitmap.serialize();
|
|
507
|
+
}
|
|
508
|
+
};
|
|
468
509
|
function deserializePortable(bytes) {
|
|
469
|
-
return decodeSerialized(bytes, roaringCodec)
|
|
510
|
+
return new ReadonlyRoaringView(decodeSerialized(bytes, roaringCodec));
|
|
470
511
|
}
|
|
471
512
|
|
|
472
513
|
// src/system-clock.ts
|
|
@@ -510,6 +551,18 @@ var SystemClock = class {
|
|
|
510
551
|
}
|
|
511
552
|
};
|
|
512
553
|
|
|
554
|
+
// src/system-entropy.ts
|
|
555
|
+
import { UnsupportedError } from "@cloudbitmaps/core";
|
|
556
|
+
var systemEntropy = (length) => {
|
|
557
|
+
const c = globalThis.crypto;
|
|
558
|
+
if (c === void 0 || typeof c.getRandomValues !== "function") {
|
|
559
|
+
throw new UnsupportedError(
|
|
560
|
+
"this runtime has no Web Crypto (crypto.getRandomValues), which every load needs for its generation's write identity: run on a runtime that has it, or pass `seams.entropy`"
|
|
561
|
+
);
|
|
562
|
+
}
|
|
563
|
+
return c.getRandomValues(new Uint8Array(length));
|
|
564
|
+
};
|
|
565
|
+
|
|
513
566
|
// src/option-keys.ts
|
|
514
567
|
var OPTION_KEYS = {
|
|
515
568
|
top: ["storage", "cache", "encryption", "retry", "metrics", "budget", "seams"],
|
|
@@ -517,7 +570,7 @@ var OPTION_KEYS = {
|
|
|
517
570
|
encryption: ["keystore", "required"],
|
|
518
571
|
retry: ["maxAttempts", "baseDelayMs", "maxDelayMs", "backoffFactor", "jitter", "onRetry"],
|
|
519
572
|
budget: ["maxRequests"],
|
|
520
|
-
seams: ["clock", "rng"]
|
|
573
|
+
seams: ["clock", "rng", "entropy"]
|
|
521
574
|
};
|
|
522
575
|
|
|
523
576
|
// src/lease-guards.ts
|
|
@@ -595,17 +648,18 @@ import {
|
|
|
595
648
|
DEFAULT_RETRY_POLICY,
|
|
596
649
|
NOOP_METRICS,
|
|
597
650
|
RetryingStorageChunkSource,
|
|
598
|
-
ValidationError as
|
|
651
|
+
ValidationError as ValidationError3,
|
|
599
652
|
isStorageBackend
|
|
600
653
|
} from "@cloudbitmaps/core";
|
|
601
654
|
function resolveRetryPolicy(partial) {
|
|
602
|
-
const
|
|
655
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
656
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
603
657
|
return {
|
|
604
|
-
maxAttempts:
|
|
658
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
605
659
|
baseDelayMs,
|
|
606
|
-
maxDelayMs: partial.maxDelayMs
|
|
607
|
-
backoffFactor:
|
|
608
|
-
jitter:
|
|
660
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
661
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
662
|
+
jitter: orDefault("jitter")
|
|
609
663
|
};
|
|
610
664
|
}
|
|
611
665
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -618,7 +672,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
618
672
|
if (options.retry !== false) {
|
|
619
673
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
620
674
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
621
|
-
throw new
|
|
675
|
+
throw new ValidationError3("retry.onRetry must be a function");
|
|
622
676
|
}
|
|
623
677
|
retryOptions = {
|
|
624
678
|
clock,
|
|
@@ -642,13 +696,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
642
696
|
source: resolved.source,
|
|
643
697
|
driver: resolved.driver,
|
|
644
698
|
registry: resolved.registry,
|
|
699
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
645
700
|
retryOptions
|
|
646
701
|
};
|
|
647
702
|
}
|
|
648
703
|
function resolveStorageSource(options, clock) {
|
|
649
704
|
const storage = options.storage;
|
|
650
705
|
if (storage === null || typeof storage !== "object") {
|
|
651
|
-
throw new
|
|
706
|
+
throw new ValidationError3(
|
|
652
707
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
653
708
|
);
|
|
654
709
|
}
|
|
@@ -658,14 +713,14 @@ function resolveStorageSource(options, clock) {
|
|
|
658
713
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
659
714
|
const isBackend = isStorageBackend(storage);
|
|
660
715
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
661
|
-
throw new
|
|
716
|
+
throw new ValidationError3(
|
|
662
717
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
663
718
|
);
|
|
664
719
|
}
|
|
665
720
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
666
721
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
667
722
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
668
|
-
throw new
|
|
723
|
+
throw new ValidationError3(
|
|
669
724
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
670
725
|
);
|
|
671
726
|
}
|
|
@@ -674,11 +729,11 @@ function resolveStorageSource(options, clock) {
|
|
|
674
729
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
675
730
|
if (!storageOk || !registryOk) {
|
|
676
731
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
677
|
-
throw new
|
|
732
|
+
throw new ValidationError3(
|
|
678
733
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
679
734
|
);
|
|
680
735
|
}
|
|
681
|
-
throw new
|
|
736
|
+
throw new ValidationError3(
|
|
682
737
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
683
738
|
);
|
|
684
739
|
}
|
|
@@ -699,19 +754,19 @@ function resolveStorageSource(options, clock) {
|
|
|
699
754
|
};
|
|
700
755
|
}
|
|
701
756
|
if (hasGetChunk && hasPutImmutable) {
|
|
702
|
-
throw new
|
|
757
|
+
throw new ValidationError3(
|
|
703
758
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
704
759
|
);
|
|
705
760
|
}
|
|
706
761
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
707
|
-
throw new
|
|
762
|
+
throw new ValidationError3(
|
|
708
763
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
709
764
|
);
|
|
710
765
|
}
|
|
711
766
|
if (hasGetChunk) {
|
|
712
767
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
713
|
-
throw new
|
|
714
|
-
"keystore/requireEncryption apply only when the store builds its own read path;
|
|
768
|
+
throw new ValidationError3(
|
|
769
|
+
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
715
770
|
);
|
|
716
771
|
}
|
|
717
772
|
return { source: storage, driver: void 0, registry: void 0 };
|
|
@@ -732,31 +787,32 @@ function resolveStorageSource(options, clock) {
|
|
|
732
787
|
}
|
|
733
788
|
|
|
734
789
|
// src/options.ts
|
|
735
|
-
import { ValidationError as
|
|
790
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core";
|
|
736
791
|
var keysOf = () => (keys) => keys;
|
|
737
792
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
738
|
-
function optionsError(options, op, known, booleans = []) {
|
|
793
|
+
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
739
794
|
if (options === void 0 || options === null) return void 0;
|
|
740
795
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
741
|
-
return new
|
|
796
|
+
return new ValidationError4(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
742
797
|
}
|
|
743
798
|
const bag = options;
|
|
744
799
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
800
|
+
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
745
801
|
if (unknown.length > 0) {
|
|
746
|
-
return new
|
|
802
|
+
return new ValidationError4(
|
|
747
803
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
748
804
|
);
|
|
749
805
|
}
|
|
750
806
|
const audit = bag.audit;
|
|
751
807
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
752
|
-
return new
|
|
808
|
+
return new ValidationError4(
|
|
753
809
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
754
810
|
);
|
|
755
811
|
}
|
|
756
812
|
for (const name of booleans) {
|
|
757
813
|
const v = bag[name];
|
|
758
814
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
759
|
-
return new
|
|
815
|
+
return new ValidationError4(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
760
816
|
}
|
|
761
817
|
}
|
|
762
818
|
return void 0;
|
|
@@ -770,58 +826,32 @@ function picked(options, keys) {
|
|
|
770
826
|
}
|
|
771
827
|
return out;
|
|
772
828
|
}
|
|
773
|
-
function checkOptions(options, op, known, booleans) {
|
|
774
|
-
const err = optionsError(options, op, known, booleans);
|
|
829
|
+
function checkOptions(options, op, known, booleans, misplaced) {
|
|
830
|
+
const err = optionsError(options, op, known, booleans, misplaced);
|
|
775
831
|
if (err !== void 0) throw err;
|
|
776
832
|
}
|
|
777
833
|
|
|
778
834
|
// src/index.ts
|
|
835
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
779
836
|
import {
|
|
780
|
-
MemoryStorage,
|
|
781
|
-
LocalFsStorage,
|
|
782
|
-
CrbmStorageChunkSource as CrbmStorageChunkSource3,
|
|
783
|
-
destroySegment,
|
|
784
|
-
eraseNamespace,
|
|
785
837
|
excludingReservedRows as excludingReservedRows2,
|
|
786
838
|
readRetentionPolicy,
|
|
787
|
-
MIN_EXPIRES_AT_MS,
|
|
788
|
-
LEASE_SKEW_MS as LEASE_SKEW_MS2,
|
|
789
|
-
MAX_LEASE_MS as MAX_LEASE_MS2,
|
|
790
|
-
MAX_LEASES_PER_SEGMENT,
|
|
791
|
-
InProcessKeystore,
|
|
792
|
-
NodeAead,
|
|
793
|
-
aadFor,
|
|
794
839
|
CloudRoaringError,
|
|
795
|
-
ValidationError as
|
|
840
|
+
ValidationError as ValidationError6,
|
|
796
841
|
WriteConflictError as WriteConflictError2,
|
|
797
842
|
IntegrityError as IntegrityError3,
|
|
798
843
|
NotFoundError as NotFoundError2,
|
|
799
|
-
UnsupportedError as
|
|
844
|
+
UnsupportedError as UnsupportedError3,
|
|
800
845
|
CapabilityError,
|
|
801
846
|
TransientError,
|
|
802
847
|
KeyUnavailableError,
|
|
803
|
-
BudgetExceededError,
|
|
848
|
+
BudgetExceededError as BudgetExceededError2,
|
|
804
849
|
LeaseExpiredError as LeaseExpiredError2,
|
|
805
850
|
LeaseLimitError,
|
|
806
851
|
StaleOperandError,
|
|
807
|
-
|
|
808
|
-
isWriteConflictError,
|
|
809
|
-
isTransientError,
|
|
810
|
-
isNotFoundError,
|
|
811
|
-
isIntegrityError,
|
|
812
|
-
isValidationError,
|
|
813
|
-
isLeaseExpiredError,
|
|
814
|
-
isLeaseLimitError,
|
|
815
|
-
isStaleOperandError,
|
|
816
|
-
isUnsupportedError,
|
|
817
|
-
isCapabilityError,
|
|
818
|
-
isBudgetExceededError,
|
|
819
|
-
isKeyUnavailableError,
|
|
820
|
-
CrbmReader,
|
|
821
|
-
BufferReader,
|
|
852
|
+
errorKind as errorKind2,
|
|
822
853
|
CountingMetricsSink,
|
|
823
|
-
RecordingAuditSink
|
|
824
|
-
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY3
|
|
854
|
+
RecordingAuditSink
|
|
825
855
|
} from "@cloudbitmaps/core";
|
|
826
856
|
var SystemRng = class {
|
|
827
857
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -829,6 +859,26 @@ var SystemRng = class {
|
|
|
829
859
|
return Math.random();
|
|
830
860
|
}
|
|
831
861
|
};
|
|
862
|
+
function withWaits(deps) {
|
|
863
|
+
if (deps === null || typeof deps !== "object") return deps;
|
|
864
|
+
return { ...deps, clock: deps.clock ?? new SystemClock(), rng: deps.rng ?? new SystemRng() };
|
|
865
|
+
}
|
|
866
|
+
function destroySegment(ref, deps, options) {
|
|
867
|
+
return coreDestroySegment(ref, withWaits(deps), options);
|
|
868
|
+
}
|
|
869
|
+
function eraseNamespace(namespace, deps, options) {
|
|
870
|
+
return coreEraseNamespace(namespace, withWaits(deps), options);
|
|
871
|
+
}
|
|
872
|
+
function validateRef(ref, verb) {
|
|
873
|
+
if (typeof ref !== "object" || ref === null) {
|
|
874
|
+
throw new ValidationError5(
|
|
875
|
+
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
refuseUnknownRefKeys(ref, verb);
|
|
879
|
+
validateSegmentRef(ref);
|
|
880
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
881
|
+
}
|
|
832
882
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
833
883
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
834
884
|
function rowVersionOfLive(version) {
|
|
@@ -842,15 +892,20 @@ function validateConcurrency(concurrency) {
|
|
|
842
892
|
function requireScope(options, op) {
|
|
843
893
|
if (options.namespace !== void 0) {
|
|
844
894
|
if (typeof options.namespace !== "string") {
|
|
845
|
-
throw new
|
|
895
|
+
throw new ValidationError5(
|
|
846
896
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
847
897
|
);
|
|
848
898
|
}
|
|
849
899
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
850
900
|
}
|
|
851
901
|
refuseReservedNamespace(options.namespace);
|
|
902
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
903
|
+
throw new ValidationError5(
|
|
904
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
852
907
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
853
|
-
throw new
|
|
908
|
+
throw new ValidationError5(
|
|
854
909
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
855
910
|
);
|
|
856
911
|
}
|
|
@@ -897,7 +952,7 @@ var CALL_KEYS = {
|
|
|
897
952
|
"metadata",
|
|
898
953
|
"keep"
|
|
899
954
|
]),
|
|
900
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
955
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
901
956
|
segments: keysOf()(["namespace"]),
|
|
902
957
|
subjectReport: keysOf()([
|
|
903
958
|
"namespace",
|
|
@@ -938,13 +993,37 @@ var CALL_KEYS = {
|
|
|
938
993
|
checkConsistency: keysOf()([
|
|
939
994
|
"namespace",
|
|
940
995
|
"concurrency",
|
|
996
|
+
"maxScanSegments",
|
|
941
997
|
"summaries",
|
|
942
998
|
"generations"
|
|
943
999
|
]),
|
|
944
|
-
exportSegments: keysOf()(["format", "
|
|
1000
|
+
exportSegments: keysOf()(["format", "namespace", "ndjsonBatchBytes"])
|
|
1001
|
+
};
|
|
1002
|
+
var outside = (wider, known) => wider.filter((k) => !known.includes(k));
|
|
1003
|
+
var MISPLACED = {
|
|
1004
|
+
range: outside(CALL_KEYS.into, CALL_KEYS.range),
|
|
1005
|
+
andNot: outside(CALL_KEYS.into, CALL_KEYS.andNot),
|
|
1006
|
+
combine: outside(CALL_KEYS.into, CALL_KEYS.combine),
|
|
1007
|
+
andNotInto: outside(CALL_KEYS.into, CALL_KEYS.andNotInto),
|
|
1008
|
+
exportSegments: ["codec"]
|
|
945
1009
|
};
|
|
946
1010
|
var SEGMENT_BRAND = /* @__PURE__ */ Symbol.for("cloudbitmaps.roaring.segment/1");
|
|
947
1011
|
var viewOf;
|
|
1012
|
+
var originOf;
|
|
1013
|
+
function assertOneStore(verb, role, h, origin) {
|
|
1014
|
+
if (h instanceof Segment) {
|
|
1015
|
+
if (originOf(h) === origin) return;
|
|
1016
|
+
throw new ValidationError5(
|
|
1017
|
+
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
1021
|
+
throw new ValidationError5(
|
|
1022
|
+
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
1023
|
+
);
|
|
1024
|
+
}
|
|
1025
|
+
throw new ValidationError5(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
1026
|
+
}
|
|
948
1027
|
var MATERIALIZE_MANY_KEYS = [
|
|
949
1028
|
"operands",
|
|
950
1029
|
"dryRun",
|
|
@@ -961,6 +1040,17 @@ var MATERIALIZE_MANY_KEYS = [
|
|
|
961
1040
|
"feed",
|
|
962
1041
|
"mayBeEmpty"
|
|
963
1042
|
];
|
|
1043
|
+
var MATERIALIZE_MANY_MISPLACED = outside(CALL_KEYS.into, MATERIALIZE_MANY_KEYS);
|
|
1044
|
+
function unknownManyKeys(options) {
|
|
1045
|
+
const unknown = Object.keys(options).filter(
|
|
1046
|
+
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1047
|
+
);
|
|
1048
|
+
const bag = options;
|
|
1049
|
+
for (const k of MATERIALIZE_MANY_MISPLACED) {
|
|
1050
|
+
if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
1051
|
+
}
|
|
1052
|
+
return unknown;
|
|
1053
|
+
}
|
|
964
1054
|
var MATERIALIZE_MANY_OUTPUT_KEYS = [
|
|
965
1055
|
"dest",
|
|
966
1056
|
"expr",
|
|
@@ -986,18 +1076,23 @@ var NOT_A_GROUP = {
|
|
|
986
1076
|
};
|
|
987
1077
|
var CloudRoaring = class _CloudRoaring {
|
|
988
1078
|
engine;
|
|
1079
|
+
/** This store's identity: every handle it makes carries this object, and a combine compares it by reference. */
|
|
1080
|
+
origin = Object.freeze({});
|
|
989
1081
|
cache;
|
|
990
1082
|
/** The store's retry options, for a pinned engine to read through as the store's own does; undefined when off. */
|
|
991
1083
|
retryOptions;
|
|
992
1084
|
crbmSource;
|
|
993
1085
|
clock;
|
|
994
1086
|
rng;
|
|
1087
|
+
entropy;
|
|
995
1088
|
metrics;
|
|
996
1089
|
// The store's own drivers, kept so the lifecycle helpers and the `*Into` verbs reuse them instead of making
|
|
997
1090
|
// you re-pass deps. `storageDriver` is set only when `storage` was a raw IStorageDriver (a pre-built StorageChunkSource has
|
|
998
1091
|
// no underlying driver to write through).
|
|
999
1092
|
storageDriver;
|
|
1000
1093
|
registry;
|
|
1094
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
1095
|
+
backend;
|
|
1001
1096
|
keystore;
|
|
1002
1097
|
requireEncryption;
|
|
1003
1098
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1017,7 +1112,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1017
1112
|
*/
|
|
1018
1113
|
static rejectUnknownOptions(options) {
|
|
1019
1114
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1020
|
-
throw new
|
|
1115
|
+
throw new ValidationError5(
|
|
1021
1116
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1022
1117
|
);
|
|
1023
1118
|
}
|
|
@@ -1035,7 +1130,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1035
1130
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1036
1131
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1037
1132
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1038
|
-
throw new
|
|
1133
|
+
throw new ValidationError5(
|
|
1039
1134
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1040
1135
|
);
|
|
1041
1136
|
}
|
|
@@ -1045,7 +1140,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1045
1140
|
if (unknown.length === 0) return;
|
|
1046
1141
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1047
1142
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1048
|
-
throw new
|
|
1143
|
+
throw new ValidationError5(
|
|
1049
1144
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1050
1145
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1051
1146
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1061,21 +1156,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1061
1156
|
[options.seams?.rng, "seams.rng", ["next"]],
|
|
1062
1157
|
[options.encryption?.keystore, "encryption.keystore", ["createDek", "openDek"]]
|
|
1063
1158
|
];
|
|
1159
|
+
const entropy = options.seams?.entropy;
|
|
1160
|
+
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1161
|
+
throw new ValidationError5(
|
|
1162
|
+
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1163
|
+
);
|
|
1164
|
+
}
|
|
1064
1165
|
for (const [seam, name, needs] of seams) {
|
|
1065
|
-
if (seam === void 0
|
|
1066
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1067
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1068
|
-
throw new
|
|
1166
|
+
if (seam === void 0) continue;
|
|
1167
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
1168
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
1169
|
+
throw new ValidationError5(
|
|
1069
1170
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1070
1171
|
);
|
|
1071
1172
|
}
|
|
1072
1173
|
}
|
|
1073
1174
|
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
1177
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
1178
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
1179
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
1180
|
+
*/
|
|
1181
|
+
static rejectBadValues(options) {
|
|
1182
|
+
const cache = options.cache;
|
|
1183
|
+
if (cache !== void 0) {
|
|
1184
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
1185
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
1186
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
1187
|
+
const ttl = cache.ttlMs;
|
|
1188
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
1189
|
+
throw new ValidationError5(
|
|
1190
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
1191
|
+
);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
1198
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
1199
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
1200
|
+
* calls, throws `ValidationError`.
|
|
1201
|
+
*/
|
|
1074
1202
|
constructor(options) {
|
|
1075
1203
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1076
1204
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1205
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1077
1206
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1078
1207
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1208
|
+
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
1079
1209
|
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS2);
|
|
1080
1210
|
const cache = new BoundedLru({
|
|
1081
1211
|
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
@@ -1099,9 +1229,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1099
1229
|
this.crbmSource = wired.source instanceof CrbmStorageChunkSource2 ? wired.source : void 0;
|
|
1100
1230
|
this.clock = clock;
|
|
1101
1231
|
this.rng = rng;
|
|
1232
|
+
this.entropy = entropy;
|
|
1102
1233
|
this.metrics = metrics;
|
|
1103
1234
|
this.storageDriver = wired.driver;
|
|
1104
1235
|
this.registry = wired.registry;
|
|
1236
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
1237
|
+
this.backend = wired.backend;
|
|
1105
1238
|
this.keystore = options.encryption?.keystore;
|
|
1106
1239
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1107
1240
|
}
|
|
@@ -1113,12 +1246,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1113
1246
|
*/
|
|
1114
1247
|
lifecycleDeps(op) {
|
|
1115
1248
|
if (this.storageDriver === void 0) {
|
|
1116
|
-
throw new
|
|
1249
|
+
throw new UnsupportedError2(
|
|
1117
1250
|
`${op} needs the store built with a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A pre-built StorageChunkSource is read-only: it has no IStorageDriver underneath to write generations through. Build the store on one of those backend classes to write.`
|
|
1118
1251
|
);
|
|
1119
1252
|
}
|
|
1120
1253
|
if (this.registry === void 0) {
|
|
1121
|
-
throw new
|
|
1254
|
+
throw new UnsupportedError2(
|
|
1122
1255
|
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
1123
1256
|
);
|
|
1124
1257
|
}
|
|
@@ -1127,6 +1260,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1127
1260
|
registry: this.registry,
|
|
1128
1261
|
clock: this.clock,
|
|
1129
1262
|
rng: this.rng,
|
|
1263
|
+
entropy: this.entropy,
|
|
1130
1264
|
codec: roaringCodec,
|
|
1131
1265
|
// facade injects the flagship codec
|
|
1132
1266
|
keystore: this.keystore,
|
|
@@ -1143,13 +1277,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1143
1277
|
* spread of options keeps working.
|
|
1144
1278
|
*/
|
|
1145
1279
|
segment(name, options) {
|
|
1280
|
+
if (typeof name === "object" && name !== null) {
|
|
1281
|
+
throw new ValidationError5(
|
|
1282
|
+
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1283
|
+
);
|
|
1284
|
+
}
|
|
1146
1285
|
if (options !== void 0 && options !== null) {
|
|
1147
1286
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1148
|
-
throw new
|
|
1287
|
+
throw new ValidationError5("segment: options must be an object such as { namespace }");
|
|
1149
1288
|
}
|
|
1150
1289
|
const proto = Object.getPrototypeOf(options);
|
|
1151
1290
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1152
|
-
throw new
|
|
1291
|
+
throw new ValidationError5(
|
|
1153
1292
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1154
1293
|
);
|
|
1155
1294
|
}
|
|
@@ -1157,7 +1296,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1157
1296
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1158
1297
|
);
|
|
1159
1298
|
if (unknown.length > 0) {
|
|
1160
|
-
throw new
|
|
1299
|
+
throw new ValidationError5(
|
|
1161
1300
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1162
1301
|
);
|
|
1163
1302
|
}
|
|
@@ -1167,6 +1306,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1167
1306
|
refuseReservedNamespace(ref.namespace);
|
|
1168
1307
|
return makeSegment({
|
|
1169
1308
|
engine: this.engine,
|
|
1309
|
+
origin: this.origin,
|
|
1170
1310
|
ref,
|
|
1171
1311
|
clock: this.clock,
|
|
1172
1312
|
metrics: this.metrics,
|
|
@@ -1238,7 +1378,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1238
1378
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1239
1379
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1240
1380
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1241
|
-
keep: options?.keep
|
|
1381
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1242
1382
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1243
1383
|
};
|
|
1244
1384
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1266,22 +1406,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1266
1406
|
/** The one body of the forms above. */
|
|
1267
1407
|
async materializeMany(options) {
|
|
1268
1408
|
if (typeof options !== "object" || options === null) {
|
|
1269
|
-
throw new
|
|
1409
|
+
throw new ValidationError5(
|
|
1270
1410
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1271
1411
|
);
|
|
1272
1412
|
}
|
|
1273
1413
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1274
|
-
const unknown =
|
|
1275
|
-
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1276
|
-
);
|
|
1414
|
+
const unknown = unknownManyKeys(options);
|
|
1277
1415
|
if (unknown.length > 0) {
|
|
1278
|
-
throw new
|
|
1416
|
+
throw new ValidationError5(
|
|
1279
1417
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1280
1418
|
);
|
|
1281
1419
|
}
|
|
1282
1420
|
const dryRunIn = options.dryRun;
|
|
1283
1421
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1284
|
-
throw new
|
|
1422
|
+
throw new ValidationError5("materializeMany: dryRun must be a boolean");
|
|
1285
1423
|
}
|
|
1286
1424
|
const dryRun = dryRunIn === true;
|
|
1287
1425
|
const operandsIn = options.operands;
|
|
@@ -1293,7 +1431,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1293
1431
|
const feedIn = options.feed;
|
|
1294
1432
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1295
1433
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1296
|
-
throw new
|
|
1434
|
+
throw new ValidationError5(
|
|
1297
1435
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1298
1436
|
);
|
|
1299
1437
|
}
|
|
@@ -1303,37 +1441,33 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1303
1441
|
const budgetOption = options.budget;
|
|
1304
1442
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1305
1443
|
if (typeof pinOption !== "boolean")
|
|
1306
|
-
throw new
|
|
1444
|
+
throw new ValidationError5("materializeMany: pin must be a boolean");
|
|
1307
1445
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1308
|
-
throw new
|
|
1446
|
+
throw new ValidationError5("materializeMany: allowAbsentOperands must be a boolean");
|
|
1309
1447
|
}
|
|
1310
1448
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1311
|
-
throw new
|
|
1449
|
+
throw new ValidationError5("materializeMany: operands must be an object of name to segment");
|
|
1312
1450
|
}
|
|
1313
1451
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1314
|
-
throw new
|
|
1452
|
+
throw new ValidationError5("materializeMany: outputs must be a non-empty array");
|
|
1315
1453
|
}
|
|
1316
1454
|
const names = Object.keys(operandsIn);
|
|
1317
1455
|
const handles = /* @__PURE__ */ new Map();
|
|
1318
1456
|
for (const name of names) {
|
|
1319
1457
|
const handle = operandsIn[name];
|
|
1320
|
-
|
|
1321
|
-
throw new ValidationError4(
|
|
1322
|
-
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1323
|
-
);
|
|
1324
|
-
}
|
|
1458
|
+
assertOneStore("materializeMany", `operands["${name}"]`, handle, this.origin);
|
|
1325
1459
|
handles.set(name, handle);
|
|
1326
1460
|
}
|
|
1327
1461
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1328
1462
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1329
|
-
throw new
|
|
1463
|
+
throw new ValidationError5(
|
|
1330
1464
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1331
1465
|
);
|
|
1332
1466
|
}
|
|
1333
1467
|
if (mayBeEmptyIn.length > 0) {
|
|
1334
1468
|
const first = mayBeEmptyIn[0];
|
|
1335
1469
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1336
|
-
throw new
|
|
1470
|
+
throw new ValidationError5(
|
|
1337
1471
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1338
1472
|
);
|
|
1339
1473
|
}
|
|
@@ -1343,28 +1477,26 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1343
1477
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1344
1478
|
outputsIn.forEach((raw, i) => {
|
|
1345
1479
|
if (typeof raw !== "object" || raw === null) {
|
|
1346
|
-
throw new
|
|
1480
|
+
throw new ValidationError5(`outputs[${i}] must be an object`);
|
|
1347
1481
|
}
|
|
1348
1482
|
const bad = Object.keys(raw).filter(
|
|
1349
1483
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1350
1484
|
);
|
|
1351
1485
|
if (bad.length > 0) {
|
|
1352
|
-
throw new
|
|
1486
|
+
throw new ValidationError5(
|
|
1353
1487
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1354
1488
|
);
|
|
1355
1489
|
}
|
|
1356
1490
|
const spec = raw;
|
|
1357
1491
|
const dest = spec.dest;
|
|
1358
|
-
|
|
1359
|
-
throw new ValidationError4(`outputs[${i}].dest must be a segment from store.segment()`);
|
|
1360
|
-
}
|
|
1492
|
+
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1361
1493
|
if (operandKeys.has(dest.key())) {
|
|
1362
|
-
throw new
|
|
1494
|
+
throw new ValidationError5(
|
|
1363
1495
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1364
1496
|
);
|
|
1365
1497
|
}
|
|
1366
1498
|
if (destKeys.has(dest.key())) {
|
|
1367
|
-
throw new
|
|
1499
|
+
throw new ValidationError5(
|
|
1368
1500
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1369
1501
|
);
|
|
1370
1502
|
}
|
|
@@ -1424,7 +1556,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1424
1556
|
this.lifecycleDeps("materializeMany");
|
|
1425
1557
|
const crbm = this.crbmSource;
|
|
1426
1558
|
if (crbm === void 0) {
|
|
1427
|
-
throw new
|
|
1559
|
+
throw new UnsupportedError2(
|
|
1428
1560
|
"materializeMany needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage`"
|
|
1429
1561
|
);
|
|
1430
1562
|
}
|
|
@@ -1437,13 +1569,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1437
1569
|
held.set(key, { at, handle });
|
|
1438
1570
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1439
1571
|
if (!samePin(prior.at, at)) {
|
|
1440
|
-
throw new
|
|
1572
|
+
throw new ValidationError5(
|
|
1441
1573
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1442
1574
|
);
|
|
1443
1575
|
}
|
|
1444
1576
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1445
1577
|
if (!pinOption) {
|
|
1446
|
-
throw new
|
|
1578
|
+
throw new ValidationError5(
|
|
1447
1579
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1448
1580
|
);
|
|
1449
1581
|
}
|
|
@@ -1531,7 +1663,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1531
1663
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1532
1664
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1533
1665
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1534
|
-
throw new
|
|
1666
|
+
throw new ValidationError5(
|
|
1535
1667
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1536
1668
|
);
|
|
1537
1669
|
}
|
|
@@ -1581,13 +1713,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1581
1713
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1582
1714
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1583
1715
|
*
|
|
1716
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1717
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1718
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1719
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1720
|
+
*
|
|
1584
1721
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1585
1722
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1586
1723
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1587
1724
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1588
1725
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
1726
|
+
*
|
|
1727
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
1728
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
1729
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
1730
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
1731
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
1732
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
1733
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
1734
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
1735
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
1736
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
1737
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
1738
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
1739
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1589
1740
|
*/
|
|
1590
|
-
async subjectReport(id, options
|
|
1741
|
+
async subjectReport(id, options) {
|
|
1591
1742
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1592
1743
|
options ??= {};
|
|
1593
1744
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1602,26 +1753,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1602
1753
|
budget,
|
|
1603
1754
|
"subjectReport"
|
|
1604
1755
|
);
|
|
1756
|
+
let opens = 0;
|
|
1757
|
+
const charge = () => {
|
|
1758
|
+
opens += 1;
|
|
1759
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
1760
|
+
throw new BudgetExceededError(
|
|
1761
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
1762
|
+
{
|
|
1763
|
+
option: "budget.maxRequests",
|
|
1764
|
+
limit: budget.maxRequests,
|
|
1765
|
+
spent: budget.maxRequests + 1
|
|
1766
|
+
}
|
|
1767
|
+
);
|
|
1768
|
+
}
|
|
1769
|
+
};
|
|
1605
1770
|
const membership = await mapWithConcurrency(
|
|
1606
1771
|
recs,
|
|
1607
1772
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1608
1773
|
async (rec) => {
|
|
1609
|
-
if (rec.status === "destroyed") return null;
|
|
1610
1774
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1775
|
+
if (rec.status === "destroyed") {
|
|
1776
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
1777
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
1778
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
1779
|
+
return held === null ? null : { tombstone: true, ref };
|
|
1780
|
+
}
|
|
1611
1781
|
if (this.crbmSource !== void 0) {
|
|
1612
1782
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1613
|
-
const
|
|
1614
|
-
if (rowVersionOfLive(held) !==
|
|
1783
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1784
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1615
1785
|
}
|
|
1616
|
-
return await this.engine.has(ref, id) ? {
|
|
1786
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1617
1787
|
}
|
|
1618
1788
|
);
|
|
1619
|
-
const
|
|
1620
|
-
|
|
1789
|
+
const listed = membership.filter((m) => m !== null);
|
|
1790
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
1791
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
1792
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1621
1793
|
}
|
|
1622
1794
|
/**
|
|
1623
1795
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1624
|
-
* from the bucket on return
|
|
1796
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
1797
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
1798
|
+
*
|
|
1799
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
1800
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
1801
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
1802
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1625
1803
|
*
|
|
1626
1804
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1627
1805
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1686,15 +1864,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1686
1864
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1687
1865
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1688
1866
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1689
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1690
|
-
*
|
|
1867
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
1868
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
1869
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
1870
|
+
* that exhausts the budget is
|
|
1691
1871
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1692
1872
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1693
1873
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1694
1874
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1695
1875
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1696
1876
|
*/
|
|
1697
|
-
async eraseSubject(id, options
|
|
1877
|
+
async eraseSubject(id, options) {
|
|
1698
1878
|
this.erasureEpoch++;
|
|
1699
1879
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1700
1880
|
options ??= {};
|
|
@@ -1750,7 +1930,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1750
1930
|
segment: rec.segment,
|
|
1751
1931
|
namespace: rec.namespace,
|
|
1752
1932
|
erased: false,
|
|
1753
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
1933
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
1934
|
+
// the class lets a note be matched to the guide's table
|
|
1754
1935
|
};
|
|
1755
1936
|
} finally {
|
|
1756
1937
|
try {
|
|
@@ -1771,11 +1952,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1771
1952
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1772
1953
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1773
1954
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1955
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1774
1956
|
*
|
|
1775
1957
|
* Needs a backend.
|
|
1776
1958
|
*/
|
|
1777
1959
|
async generations(ref) {
|
|
1778
|
-
|
|
1960
|
+
ref = validateRef(ref, "generations");
|
|
1779
1961
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1780
1962
|
}
|
|
1781
1963
|
/**
|
|
@@ -1805,7 +1987,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1805
1987
|
* ```
|
|
1806
1988
|
*/
|
|
1807
1989
|
async exists(ref) {
|
|
1808
|
-
|
|
1990
|
+
ref = validateRef(ref, "exists");
|
|
1809
1991
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1810
1992
|
}
|
|
1811
1993
|
/**
|
|
@@ -1862,8 +2044,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1862
2044
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1863
2045
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1864
2046
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1865
|
-
* crypto-shredded
|
|
1866
|
-
*
|
|
2047
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
2048
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
2049
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1867
2050
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1868
2051
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1869
2052
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1874,8 +2057,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1874
2057
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1875
2058
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1876
2059
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1877
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1878
|
-
*
|
|
2060
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
2061
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
2062
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
2063
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1879
2064
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1880
2065
|
*
|
|
1881
2066
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1905,7 +2090,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1905
2090
|
* Needs a backend.
|
|
1906
2091
|
*/
|
|
1907
2092
|
async rollback(ref, toGeneration, options = {}) {
|
|
1908
|
-
|
|
2093
|
+
ref = validateRef(ref, "rollback");
|
|
1909
2094
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1910
2095
|
options ??= {};
|
|
1911
2096
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1921,9 +2106,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1921
2106
|
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
1922
2107
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1923
2108
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1924
|
-
* containers with no per-id work, and gives the generation
|
|
1925
|
-
* `RoaringBitmap32` passed as ids loads as `{ bitmap }`. A
|
|
1926
|
-
* `ValidationError`, since each byte would load as an id:
|
|
2109
|
+
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. A bare
|
|
2110
|
+
* `RoaringBitmap32`, or a bitmap `deserializePortable` returned, passed as ids loads as `{ bitmap }`. A
|
|
2111
|
+
* `Uint8Array` or `Buffer` passed as ids is refused with `ValidationError`, since each byte would load as an id:
|
|
2112
|
+
* pass bytes as `{ serialized }`.
|
|
2113
|
+
*
|
|
2114
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
2115
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
2116
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
2117
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1927
2118
|
*
|
|
1928
2119
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1929
2120
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -1967,7 +2158,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1967
2158
|
* its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1968
2159
|
* encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
|
|
1969
2160
|
* load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
|
|
1970
|
-
* writer's object, and
|
|
2161
|
+
* writer's object, and the row does not say whether that write took this number, so it leaves the orphan rather than risk
|
|
1971
2162
|
* deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
|
|
1972
2163
|
* generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
|
|
1973
2164
|
* with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
|
|
@@ -1982,7 +2173,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1982
2173
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
1983
2174
|
* what the pass deleted by name, and one name may have been gone already.
|
|
1984
2175
|
*
|
|
1985
|
-
*
|
|
2176
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
2177
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
2178
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
2179
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
2180
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
2181
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
2182
|
+
*
|
|
2183
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
1986
2184
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
1987
2185
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
1988
2186
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2005,8 +2203,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2005
2203
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2006
2204
|
*/
|
|
2007
2205
|
async load(ref, input, options = {}) {
|
|
2008
|
-
|
|
2009
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
2206
|
+
ref = validateRef(ref, "load");
|
|
2207
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
2208
|
+
if (typeof input === "object" && input !== null) {
|
|
2209
|
+
const reads = STREAM_SOURCES.get(input);
|
|
2210
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
2211
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
2212
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
2213
|
+
throw new ValidationError5(
|
|
2214
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
2215
|
+
);
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2010
2218
|
options ??= {};
|
|
2011
2219
|
const deps = this.lifecycleDeps("load");
|
|
2012
2220
|
try {
|
|
@@ -2016,11 +2224,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2016
2224
|
}
|
|
2017
2225
|
}
|
|
2018
2226
|
/**
|
|
2019
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
2227
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
2228
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2020
2229
|
*
|
|
2021
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2022
|
-
*
|
|
2023
|
-
*
|
|
2230
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
2231
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
2232
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
2233
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
2234
|
+
* encrypted segment it does both.
|
|
2024
2235
|
*
|
|
2025
2236
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2026
2237
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2051,7 +2262,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2051
2262
|
* segment may answer from its cached generation + cached chunks until that window lapses, or, while the registry
|
|
2052
2263
|
* cannot be read because of a transient fault, until a retry 500 ms apart reaches it again. A reader that never
|
|
2053
2264
|
* touched it sees empty at once. **That bound needs a registry and `cache.genTtlMs > 0`.** A store with no registry
|
|
2054
|
-
* (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a
|
|
2265
|
+
* (a bare `IStorageDriver`), with `cache.genTtlMs: 0`, or on a `StorageChunkSource` of your own, has no timed
|
|
2055
2266
|
* refresh. It notices the drop only when a read has to fetch
|
|
2056
2267
|
* from a deleted generation or its reader cache evicts the segment, and until then a chunk it has cached
|
|
2057
2268
|
* answers without reaching storage, so it can keep answering `true` for a dropped segment indefinitely. Tell
|
|
@@ -2061,9 +2272,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2061
2272
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2062
2273
|
*/
|
|
2063
2274
|
async dropSegment(ref, options) {
|
|
2064
|
-
|
|
2275
|
+
ref = validateRef(ref, "dropSegment");
|
|
2065
2276
|
if (options === void 0 || options === null) {
|
|
2066
|
-
throw new
|
|
2277
|
+
throw new ValidationError5("dropSegment: options { confirmSegment } are required");
|
|
2067
2278
|
}
|
|
2068
2279
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2069
2280
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2078,8 +2289,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2078
2289
|
}
|
|
2079
2290
|
}
|
|
2080
2291
|
/**
|
|
2081
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2082
|
-
*
|
|
2292
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
2293
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
2294
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2083
2295
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2084
2296
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2085
2297
|
*
|
|
@@ -2099,12 +2311,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2099
2311
|
* enumerable by the sweep; the first load then publishes onto that row.
|
|
2100
2312
|
*
|
|
2101
2313
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2102
|
-
* buckets is normal. A value below `
|
|
2314
|
+
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2103
2315
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2104
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
2316
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2105
2317
|
*/
|
|
2106
2318
|
async setRetention(ref, policy) {
|
|
2107
|
-
|
|
2319
|
+
ref = validateRef(ref, "setRetention");
|
|
2108
2320
|
return setSegmentRetention(
|
|
2109
2321
|
ref,
|
|
2110
2322
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2118,16 +2330,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2118
2330
|
* "never expires" on a segment someone believes is expiring.
|
|
2119
2331
|
*/
|
|
2120
2332
|
async getRetention(ref) {
|
|
2121
|
-
|
|
2333
|
+
ref = validateRef(ref, "getRetention");
|
|
2122
2334
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2123
2335
|
}
|
|
2124
2336
|
/**
|
|
2125
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2126
|
-
* when there was none
|
|
2337
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
2338
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
2339
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
2340
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2127
2341
|
* to the setter is how a typo becomes a deletion.
|
|
2128
2342
|
*/
|
|
2129
2343
|
async clearRetention(ref) {
|
|
2130
|
-
|
|
2344
|
+
ref = validateRef(ref, "clearRetention");
|
|
2131
2345
|
return clearSegmentRetention(ref, {
|
|
2132
2346
|
registry: this.requireRegistry("clearRetention"),
|
|
2133
2347
|
clock: this.clock,
|
|
@@ -2152,20 +2366,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2152
2366
|
*
|
|
2153
2367
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2154
2368
|
* for (const e of swept.entries) {
|
|
2155
|
-
*
|
|
2369
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
2370
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2156
2371
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2157
2372
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2158
2373
|
* }
|
|
2159
2374
|
* }
|
|
2160
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
2375
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2161
2376
|
* ```
|
|
2162
2377
|
*
|
|
2163
2378
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2164
2379
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2165
2380
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2166
|
-
* `limit` (default 100) cut the pass short and more
|
|
2167
|
-
*
|
|
2168
|
-
*
|
|
2381
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
2382
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
2383
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
2384
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2169
2385
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2170
2386
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2171
2387
|
*
|
|
@@ -2175,9 +2391,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2175
2391
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2176
2392
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2177
2393
|
*
|
|
2178
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
2394
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2179
2395
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2180
|
-
* first, since nothing else
|
|
2396
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
2397
|
+
* only
|
|
2398
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2181
2399
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2182
2400
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2183
2401
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2198,7 +2416,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2198
2416
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2199
2417
|
const result = await retireExpired(
|
|
2200
2418
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2201
|
-
{ ...opts, now: opts.now
|
|
2419
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2202
2420
|
);
|
|
2203
2421
|
if (opts.dryRun !== true) {
|
|
2204
2422
|
for (const entry of result.entries) {
|
|
@@ -2227,7 +2445,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2227
2445
|
* A pin answers with no timed bound, until it is told or both caches let it go.
|
|
2228
2446
|
* - **Another process.** Erasing on one box invalidates nothing on the others; each store bounds its own
|
|
2229
2447
|
* staleness by `cache.genTtlMs`, and a store with no registry (a bare `IStorageDriver`), with
|
|
2230
|
-
* `cache.genTtlMs: 0`, or on a
|
|
2448
|
+
* `cache.genTtlMs: 0`, or on a `StorageChunkSource` of your own has no bound at all: it converges only when a read happens to miss its caches. If a compliance deadline depends on every
|
|
2231
2449
|
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
2232
2450
|
*
|
|
2233
2451
|
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
@@ -2245,7 +2463,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2245
2463
|
* ```
|
|
2246
2464
|
*/
|
|
2247
2465
|
invalidate(ref) {
|
|
2248
|
-
|
|
2466
|
+
ref = validateRef(ref, "invalidate");
|
|
2249
2467
|
refuseReservedNamespace(ref.namespace);
|
|
2250
2468
|
this.engine.invalidate(ref);
|
|
2251
2469
|
}
|
|
@@ -2260,8 +2478,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2260
2478
|
async pinSegment(ref, leaseUntil, named) {
|
|
2261
2479
|
const crbm = this.crbmSource;
|
|
2262
2480
|
if (crbm === void 0) {
|
|
2263
|
-
throw new
|
|
2264
|
-
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (
|
|
2481
|
+
throw new UnsupportedError2(
|
|
2482
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (the store cannot pin a StorageChunkSource of your own)"
|
|
2265
2483
|
);
|
|
2266
2484
|
}
|
|
2267
2485
|
let hold;
|
|
@@ -2269,7 +2487,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2269
2487
|
if (leaseUntil !== void 0) {
|
|
2270
2488
|
const registry = this.requireRegistry("pin({ leaseUntil })");
|
|
2271
2489
|
if (registry.capabilities().canWrite === false) {
|
|
2272
|
-
throw new
|
|
2490
|
+
throw new UnsupportedError2(
|
|
2273
2491
|
"pin({ leaseUntil }) needs a registry that can write a row, and this store's cannot in this runtime"
|
|
2274
2492
|
);
|
|
2275
2493
|
}
|
|
@@ -2305,14 +2523,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2305
2523
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2306
2524
|
throw err;
|
|
2307
2525
|
}
|
|
2308
|
-
const pinnedAt = {
|
|
2309
|
-
generation: at?.generation ?? null,
|
|
2310
|
-
version: at?.version ?? null,
|
|
2311
|
-
fingerprint: at?.fingerprint ?? null
|
|
2312
|
-
};
|
|
2526
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2313
2527
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2314
2528
|
return makeSegment({
|
|
2315
2529
|
engine: this.engineWithPins(pins),
|
|
2530
|
+
origin: this.origin,
|
|
2316
2531
|
ref,
|
|
2317
2532
|
clock: this.clock,
|
|
2318
2533
|
metrics: this.metrics,
|
|
@@ -2340,8 +2555,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2340
2555
|
engineWithPins(pins) {
|
|
2341
2556
|
const crbm = this.crbmSource;
|
|
2342
2557
|
if (crbm === void 0) {
|
|
2343
|
-
throw new
|
|
2344
|
-
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (
|
|
2558
|
+
throw new UnsupportedError2(
|
|
2559
|
+
"pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (the store cannot pin a StorageChunkSource of your own)"
|
|
2345
2560
|
);
|
|
2346
2561
|
}
|
|
2347
2562
|
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
@@ -2378,8 +2593,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2378
2593
|
if (held.has(key)) {
|
|
2379
2594
|
const before = held.get(key);
|
|
2380
2595
|
if (!samePin(before, at)) {
|
|
2381
|
-
throw new
|
|
2382
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
2596
|
+
throw new ValidationError5(
|
|
2597
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2383
2598
|
);
|
|
2384
2599
|
}
|
|
2385
2600
|
} else {
|
|
@@ -2397,7 +2612,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2397
2612
|
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
2398
2613
|
requireRegistry(op) {
|
|
2399
2614
|
if (this.registry === void 0) {
|
|
2400
|
-
throw new
|
|
2615
|
+
throw new UnsupportedError2(
|
|
2401
2616
|
`${op} needs a storage backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. A bare IStorageDriver has no generation pointer to publish through, and there is no separate \`registry\` option: a backend carries it.`
|
|
2402
2617
|
);
|
|
2403
2618
|
}
|
|
@@ -2410,8 +2625,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2410
2625
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2411
2626
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2412
2627
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2413
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2414
|
-
*
|
|
2628
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
2629
|
+
* dropped) are skipped. Pair
|
|
2630
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2415
2631
|
*
|
|
2416
2632
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2417
2633
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2428,9 +2644,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2428
2644
|
* be used is counted in `generationsUnchecked`, and none of its objects is read.
|
|
2429
2645
|
*
|
|
2430
2646
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
2431
|
-
* `BudgetExceededError` past that rather than report a partial scan as a whole one.
|
|
2432
|
-
*
|
|
2433
|
-
*
|
|
2647
|
+
* `BudgetExceededError` (`option: 'maxScanSegments'`) past that rather than report a partial scan as a whole one.
|
|
2648
|
+
* Narrow the scan with `namespace`, or raise `maxScanSegments` when the fleet really is that large and the memory
|
|
2649
|
+
* is there.
|
|
2434
2650
|
*/
|
|
2435
2651
|
async checkConsistency(options = {}) {
|
|
2436
2652
|
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
@@ -2444,6 +2660,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2444
2660
|
options
|
|
2445
2661
|
);
|
|
2446
2662
|
}
|
|
2663
|
+
/**
|
|
2664
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
2665
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
2666
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
2667
|
+
*
|
|
2668
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
2669
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
2670
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
2671
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
2672
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
2673
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
2674
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
2675
|
+
*
|
|
2676
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
2677
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
2678
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
2679
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
2680
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
2681
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
2682
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
2683
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
2684
|
+
* timing.
|
|
2685
|
+
*
|
|
2686
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
2687
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
2688
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
2689
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
2690
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
2691
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
2692
|
+
*
|
|
2693
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
2694
|
+
*/
|
|
2695
|
+
async preflight() {
|
|
2696
|
+
const deps = this.lifecycleDeps("preflight");
|
|
2697
|
+
const backend = this.backend;
|
|
2698
|
+
let checkLifecycle;
|
|
2699
|
+
try {
|
|
2700
|
+
const method = backend?.checkLifecycle;
|
|
2701
|
+
if (typeof method === "function")
|
|
2702
|
+
checkLifecycle = method.bind(backend);
|
|
2703
|
+
} catch {
|
|
2704
|
+
}
|
|
2705
|
+
return runPreflight({
|
|
2706
|
+
storage: deps.storage,
|
|
2707
|
+
registry: deps.registry,
|
|
2708
|
+
clock: this.clock,
|
|
2709
|
+
entropy: this.entropy,
|
|
2710
|
+
checkLifecycle
|
|
2711
|
+
});
|
|
2712
|
+
}
|
|
2447
2713
|
/**
|
|
2448
2714
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2449
2715
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2452,15 +2718,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2452
2718
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2453
2719
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2454
2720
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2455
|
-
*
|
|
2456
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
2457
|
-
*
|
|
2721
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
2722
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
2723
|
+
* check `failed`. The `export-segments`
|
|
2724
|
+
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2458
2725
|
*/
|
|
2459
2726
|
async exportSegments(sink, options = {}) {
|
|
2460
|
-
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments);
|
|
2727
|
+
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments, [], MISPLACED.exportSegments);
|
|
2461
2728
|
const opts = picked(options, CALL_KEYS.exportSegments);
|
|
2462
2729
|
const registry = this.requireRegistry("exportSegments");
|
|
2463
|
-
return runExport(this, registry, sink, { ...opts, codec:
|
|
2730
|
+
return runExport(this, registry, sink, { ...opts, codec: roaringCodec });
|
|
2464
2731
|
}
|
|
2465
2732
|
};
|
|
2466
2733
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
@@ -2488,12 +2755,23 @@ var readOptions = (options) => ({
|
|
|
2488
2755
|
});
|
|
2489
2756
|
var NO_SEGMENTS = [];
|
|
2490
2757
|
function segmentList(list, what) {
|
|
2491
|
-
if (!Array.isArray(list)) throw new
|
|
2758
|
+
if (!Array.isArray(list)) throw new ValidationError5(`${what} must be an array of segments`);
|
|
2492
2759
|
return list;
|
|
2493
2760
|
}
|
|
2761
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
2762
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
2763
|
+
function readsFrom(stream, origin, keys) {
|
|
2764
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
2765
|
+
return stream;
|
|
2766
|
+
}
|
|
2494
2767
|
function operandsLabel(verb) {
|
|
2495
2768
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2496
2769
|
}
|
|
2770
|
+
function refuseNoOperands(verb) {
|
|
2771
|
+
throw new ValidationError5(
|
|
2772
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
2773
|
+
);
|
|
2774
|
+
}
|
|
2497
2775
|
var AS_IDS = {
|
|
2498
2776
|
failing,
|
|
2499
2777
|
guard: guardIds,
|
|
@@ -2510,9 +2788,9 @@ var CombineChunks = class {
|
|
|
2510
2788
|
var AS_CHUNKS = {
|
|
2511
2789
|
guard: guardItems,
|
|
2512
2790
|
failing: (err) => refusing(err)(),
|
|
2513
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2514
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2515
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
2791
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
2792
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
2793
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2516
2794
|
};
|
|
2517
2795
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2518
2796
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2528,6 +2806,8 @@ var minting = false;
|
|
|
2528
2806
|
var Segment = class _Segment {
|
|
2529
2807
|
metricsOn;
|
|
2530
2808
|
engine;
|
|
2809
|
+
/** The store that made this handle, by reference; a combine takes only handles of its own store. */
|
|
2810
|
+
origin;
|
|
2531
2811
|
ref;
|
|
2532
2812
|
clock;
|
|
2533
2813
|
metrics;
|
|
@@ -2561,6 +2841,7 @@ var Segment = class _Segment {
|
|
|
2561
2841
|
pinnedAt: segment.pinnedAt,
|
|
2562
2842
|
leaseError: () => segment.leaseError()
|
|
2563
2843
|
});
|
|
2844
|
+
originOf = (segment) => segment.origin;
|
|
2564
2845
|
}
|
|
2565
2846
|
/**
|
|
2566
2847
|
* Not constructible: a handle comes from {@link CloudRoaring.segment}, which wires it to the store's engine,
|
|
@@ -2568,11 +2849,12 @@ var Segment = class _Segment {
|
|
|
2568
2849
|
*/
|
|
2569
2850
|
constructor(parts) {
|
|
2570
2851
|
if (!minting) {
|
|
2571
|
-
throw new
|
|
2852
|
+
throw new ValidationError5(
|
|
2572
2853
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2573
2854
|
);
|
|
2574
2855
|
}
|
|
2575
2856
|
this.engine = parts.engine;
|
|
2857
|
+
this.origin = parts.origin;
|
|
2576
2858
|
this.ref = parts.ref;
|
|
2577
2859
|
this.clock = parts.clock;
|
|
2578
2860
|
this.metrics = parts.metrics;
|
|
@@ -2606,7 +2888,7 @@ var Segment = class _Segment {
|
|
|
2606
2888
|
*
|
|
2607
2889
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2608
2890
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2609
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
2891
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2610
2892
|
*
|
|
2611
2893
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2612
2894
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2646,9 +2928,7 @@ var Segment = class _Segment {
|
|
|
2646
2928
|
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
2647
2929
|
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
2648
2930
|
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
2649
|
-
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store
|
|
2650
|
-
* operand reads it through that store's reader and chunk cache, which only that store's own writes and its
|
|
2651
|
-
* `invalidate` reach. Where the
|
|
2931
|
+
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store refuses the pin. Where the
|
|
2652
2932
|
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
2653
2933
|
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
2654
2934
|
*
|
|
@@ -2668,7 +2948,7 @@ var Segment = class _Segment {
|
|
|
2668
2948
|
* or misfiled one.
|
|
2669
2949
|
*
|
|
2670
2950
|
* Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
|
|
2671
|
-
* {@link UnsupportedError} on a store wired with a
|
|
2951
|
+
* {@link UnsupportedError} on a store wired with a `StorageChunkSource` of your own, which it cannot pin.
|
|
2672
2952
|
*/
|
|
2673
2953
|
async pin(options) {
|
|
2674
2954
|
this.assertLease();
|
|
@@ -2683,11 +2963,11 @@ var Segment = class _Segment {
|
|
|
2683
2963
|
leaseUntilOf(op, options) {
|
|
2684
2964
|
if (options === void 0 || options === null) return void 0;
|
|
2685
2965
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2686
|
-
throw new
|
|
2966
|
+
throw new ValidationError5(`${op}: options must be an object such as { leaseUntil }`);
|
|
2687
2967
|
}
|
|
2688
2968
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2689
2969
|
if (unknown.length > 0) {
|
|
2690
|
-
throw new
|
|
2970
|
+
throw new ValidationError5(
|
|
2691
2971
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2692
2972
|
);
|
|
2693
2973
|
}
|
|
@@ -2695,17 +2975,17 @@ var Segment = class _Segment {
|
|
|
2695
2975
|
if (until === void 0) return void 0;
|
|
2696
2976
|
const now = this.clock.now();
|
|
2697
2977
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2698
|
-
throw new
|
|
2978
|
+
throw new ValidationError5(
|
|
2699
2979
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2700
2980
|
);
|
|
2701
2981
|
}
|
|
2702
2982
|
if (until <= now) {
|
|
2703
|
-
throw new
|
|
2983
|
+
throw new ValidationError5(
|
|
2704
2984
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2705
2985
|
);
|
|
2706
2986
|
}
|
|
2707
2987
|
if (until > now + MAX_LEASE_MS) {
|
|
2708
|
-
throw new
|
|
2988
|
+
throw new ValidationError5(
|
|
2709
2989
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2710
2990
|
);
|
|
2711
2991
|
}
|
|
@@ -2756,18 +3036,12 @@ var Segment = class _Segment {
|
|
|
2756
3036
|
return void 0;
|
|
2757
3037
|
}
|
|
2758
3038
|
/**
|
|
2759
|
-
* Refuse with a {@link ValidationError}
|
|
2760
|
-
*
|
|
3039
|
+
* Refuse with a {@link ValidationError} any of `handles` that this store did not make: a value that is not a segment,
|
|
3040
|
+
* a segment of another store (even one over the same backend) and a segment of another installed copy of the
|
|
3041
|
+
* package. It runs first in every combine, before the lease check reads the handles and before any request.
|
|
2761
3042
|
*/
|
|
2762
|
-
assertSegments(handles) {
|
|
2763
|
-
for (const h of handles)
|
|
2764
|
-
if (h instanceof _Segment || typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
2765
|
-
continue;
|
|
2766
|
-
}
|
|
2767
|
-
throw new ValidationError4(
|
|
2768
|
-
"an operand must be a segment from `store.segment()` of a compatible copy of @cloudbitmaps/roaring; install one version of it"
|
|
2769
|
-
);
|
|
2770
|
-
}
|
|
3043
|
+
assertSegments(verb, role, handles) {
|
|
3044
|
+
for (const h of handles) assertOneStore(verb, role, h, this.origin);
|
|
2771
3045
|
}
|
|
2772
3046
|
/**
|
|
2773
3047
|
* Throw the first lease error among `handles`: this handle and every operand and exclude of a call. It runs before
|
|
@@ -2789,10 +3063,10 @@ var Segment = class _Segment {
|
|
|
2789
3063
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2790
3064
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2791
3065
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2792
|
-
* than `generation` and `
|
|
3066
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2793
3067
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2794
3068
|
*
|
|
2795
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
3069
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2796
3070
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2797
3071
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2798
3072
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2805,18 +3079,32 @@ var Segment = class _Segment {
|
|
|
2805
3079
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2806
3080
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2807
3081
|
* is in `options`, the second.
|
|
3082
|
+
*
|
|
3083
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
3084
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
3085
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2808
3086
|
*/
|
|
2809
3087
|
async pinAt(at, options) {
|
|
2810
3088
|
this.assertLease();
|
|
2811
|
-
const { generation, fingerprint } = at ?? {};
|
|
2812
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
3089
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
3090
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2813
3091
|
if (unknown.length > 0) {
|
|
2814
|
-
throw new
|
|
2815
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
3092
|
+
throw new ValidationError5(
|
|
3093
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
3094
|
+
);
|
|
3095
|
+
}
|
|
3096
|
+
if (generation === null) {
|
|
3097
|
+
throw new ValidationError5(
|
|
3098
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
3099
|
+
);
|
|
3100
|
+
}
|
|
3101
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
3102
|
+
throw new ValidationError5(
|
|
3103
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2816
3104
|
);
|
|
2817
3105
|
}
|
|
2818
3106
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2819
|
-
throw new
|
|
3107
|
+
throw new ValidationError5(
|
|
2820
3108
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2821
3109
|
);
|
|
2822
3110
|
}
|
|
@@ -2833,17 +3121,41 @@ var Segment = class _Segment {
|
|
|
2833
3121
|
return segmentKey(this.ref);
|
|
2834
3122
|
}
|
|
2835
3123
|
/**
|
|
2836
|
-
* Refuse an `*Into`
|
|
2837
|
-
*
|
|
2838
|
-
*
|
|
3124
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
3125
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
3126
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
3127
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
3128
|
+
* first. Called by the three `*Into` verbs only.
|
|
2839
3129
|
*
|
|
2840
3130
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2841
3131
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
2842
3132
|
* awaiting.
|
|
2843
3133
|
*/
|
|
2844
|
-
assertIntoHandles(dest, operands) {
|
|
2845
|
-
this.assertSegments([dest
|
|
2846
|
-
this.
|
|
3134
|
+
assertIntoHandles(verb, dest, operands, excludes) {
|
|
3135
|
+
this.assertSegments(verb, "the destination", [dest]);
|
|
3136
|
+
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
3137
|
+
this.assertSegments(verb, "an exclude", excludes);
|
|
3138
|
+
const base = verb.slice(0, -"Into".length);
|
|
3139
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
3140
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
3141
|
+
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
3142
|
+
}
|
|
3143
|
+
/**
|
|
3144
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
3145
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
3146
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
3147
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
3148
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
3149
|
+
*/
|
|
3150
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
3151
|
+
const key = dest.key();
|
|
3152
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
3153
|
+
if (role === void 0) return;
|
|
3154
|
+
const { namespace, segment } = dest.ref;
|
|
3155
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
3156
|
+
throw new ValidationError5(
|
|
3157
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
3158
|
+
);
|
|
2847
3159
|
}
|
|
2848
3160
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2849
3161
|
guarded(out, handles, result) {
|
|
@@ -2946,7 +3258,7 @@ var Segment = class _Segment {
|
|
|
2946
3258
|
iterate(options) {
|
|
2947
3259
|
const lease = this.leaseError();
|
|
2948
3260
|
if (lease !== void 0) return failing(lease);
|
|
2949
|
-
const bad = optionsError(options, "iterate", CALL_KEYS.range);
|
|
3261
|
+
const bad = optionsError(options, "iterate", CALL_KEYS.range, [], MISPLACED.range);
|
|
2950
3262
|
if (bad !== void 0) return failing(bad);
|
|
2951
3263
|
const range = options == null ? void 0 : rangeOf(options);
|
|
2952
3264
|
const none = range === void 0 || range.after === void 0 && range.through === void 0;
|
|
@@ -2956,7 +3268,11 @@ var Segment = class _Segment {
|
|
|
2956
3268
|
engine.iterate(ref, bounds),
|
|
2957
3269
|
() => engine.iterateBatches(ref, bounds)
|
|
2958
3270
|
);
|
|
2959
|
-
return
|
|
3271
|
+
return readsFrom(
|
|
3272
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
3273
|
+
this.origin,
|
|
3274
|
+
[this.key()]
|
|
3275
|
+
);
|
|
2960
3276
|
}
|
|
2961
3277
|
/**
|
|
2962
3278
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -2991,11 +3307,11 @@ var Segment = class _Segment {
|
|
|
2991
3307
|
everyNth(n, options) {
|
|
2992
3308
|
const lease = this.leaseError();
|
|
2993
3309
|
if (lease !== void 0) return refusing(lease)();
|
|
2994
|
-
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
3310
|
+
const bad = optionsError(options, "everyNth", CALL_KEYS.range, [], MISPLACED.range);
|
|
2995
3311
|
if (bad !== void 0) return refusing(bad)();
|
|
2996
3312
|
if (this.pinnedAt === void 0) {
|
|
2997
3313
|
return refusing(
|
|
2998
|
-
new
|
|
3314
|
+
new UnsupportedError2(
|
|
2999
3315
|
"everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
|
|
3000
3316
|
)
|
|
3001
3317
|
)();
|
|
@@ -3005,7 +3321,11 @@ var Segment = class _Segment {
|
|
|
3005
3321
|
n,
|
|
3006
3322
|
options == null ? void 0 : rangeOf(options)
|
|
3007
3323
|
);
|
|
3008
|
-
return
|
|
3324
|
+
return readsFrom(
|
|
3325
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
3326
|
+
this.origin,
|
|
3327
|
+
[this.key()]
|
|
3328
|
+
);
|
|
3009
3329
|
}
|
|
3010
3330
|
/**
|
|
3011
3331
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3041,9 +3361,20 @@ var Segment = class _Segment {
|
|
|
3041
3361
|
* requests held ahead per operand: a positive integer, 32 by default).
|
|
3042
3362
|
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
|
|
3043
3363
|
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
3364
|
+
*
|
|
3365
|
+
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3366
|
+
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3367
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
3368
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3044
3369
|
*/
|
|
3045
3370
|
intersect(others, options) {
|
|
3046
|
-
const bad = optionsError(
|
|
3371
|
+
const bad = optionsError(
|
|
3372
|
+
options,
|
|
3373
|
+
"intersect",
|
|
3374
|
+
CALL_KEYS.combine,
|
|
3375
|
+
["allowAbsentOperands"],
|
|
3376
|
+
MISPLACED.combine
|
|
3377
|
+
);
|
|
3047
3378
|
if (bad !== void 0) return failing(bad);
|
|
3048
3379
|
return this.combineAs(AS_IDS, "intersect", others, options ?? void 0);
|
|
3049
3380
|
}
|
|
@@ -3066,6 +3397,16 @@ var Segment = class _Segment {
|
|
|
3066
3397
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3067
3398
|
*
|
|
3068
3399
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
3400
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
3401
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
3402
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
3403
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
3404
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
3405
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
3406
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
3407
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
3408
|
+
* once, at one instant.
|
|
3409
|
+
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3069
3410
|
*/
|
|
3070
3411
|
async intersectInto(dest, others, options) {
|
|
3071
3412
|
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
@@ -3079,10 +3420,17 @@ var Segment = class _Segment {
|
|
|
3079
3420
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3080
3421
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3081
3422
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3082
|
-
* segment once (`unionInto`, or a load) is the cheaper shape.
|
|
3423
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
3424
|
+
* empty `others` reads `this` itself.
|
|
3083
3425
|
*/
|
|
3084
3426
|
union(others, options) {
|
|
3085
|
-
const bad = optionsError(
|
|
3427
|
+
const bad = optionsError(
|
|
3428
|
+
options,
|
|
3429
|
+
"union",
|
|
3430
|
+
CALL_KEYS.combine,
|
|
3431
|
+
["allowAbsentOperands"],
|
|
3432
|
+
MISPLACED.combine
|
|
3433
|
+
);
|
|
3086
3434
|
if (bad !== void 0) return failing(bad);
|
|
3087
3435
|
return this.combineAs(AS_IDS, "union", others, options ?? void 0);
|
|
3088
3436
|
}
|
|
@@ -3101,18 +3449,34 @@ var Segment = class _Segment {
|
|
|
3101
3449
|
*
|
|
3102
3450
|
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
3103
3451
|
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
3452
|
+
*
|
|
3453
|
+
* It takes the options {@link intersect} takes but `exclude`, since `excludes` is its subtraction: `exclude` or a write
|
|
3454
|
+
* option fails the stream with {@link ValidationError} when it is first read, before any request.
|
|
3104
3455
|
*/
|
|
3105
3456
|
andNot(excludes, options) {
|
|
3106
|
-
const bad = optionsError(
|
|
3457
|
+
const bad = optionsError(
|
|
3458
|
+
options,
|
|
3459
|
+
"andNot",
|
|
3460
|
+
CALL_KEYS.andNot,
|
|
3461
|
+
["allowAbsentOperands"],
|
|
3462
|
+
MISPLACED.andNot
|
|
3463
|
+
);
|
|
3107
3464
|
if (bad !== void 0) return failing(bad);
|
|
3108
3465
|
return this.combineAs(AS_IDS, "andNot", excludes, options ?? void 0);
|
|
3109
3466
|
}
|
|
3110
|
-
/**
|
|
3467
|
+
/**
|
|
3468
|
+
* Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. It takes every field
|
|
3469
|
+
* of {@link CombineOptions} but `exclude`, since `excludes` is its subtraction: `exclude` is refused with
|
|
3470
|
+
* {@link ValidationError} before any request.
|
|
3471
|
+
*/
|
|
3111
3472
|
async andNotInto(dest, excludes, options) {
|
|
3112
|
-
checkOptions(
|
|
3113
|
-
|
|
3114
|
-
"
|
|
3115
|
-
|
|
3473
|
+
checkOptions(
|
|
3474
|
+
options,
|
|
3475
|
+
"andNotInto",
|
|
3476
|
+
CALL_KEYS.andNotInto,
|
|
3477
|
+
["allowEmpty", "allowAbsentOperands"],
|
|
3478
|
+
MISPLACED.andNotInto
|
|
3479
|
+
);
|
|
3116
3480
|
return this.combineInto("andNotInto", dest, excludes, options);
|
|
3117
3481
|
}
|
|
3118
3482
|
/**
|
|
@@ -3124,8 +3488,11 @@ var Segment = class _Segment {
|
|
|
3124
3488
|
let handles;
|
|
3125
3489
|
try {
|
|
3126
3490
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3127
|
-
|
|
3128
|
-
|
|
3491
|
+
const others = segmentList(operands, operandsLabel(verb));
|
|
3492
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3493
|
+
handles = [this, ...others, ...exclude];
|
|
3494
|
+
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3495
|
+
this.assertSegments(verb, "an exclude", exclude);
|
|
3129
3496
|
this.assertLeases(handles);
|
|
3130
3497
|
} catch (err) {
|
|
3131
3498
|
return out.failing(err);
|
|
@@ -3136,24 +3503,33 @@ var Segment = class _Segment {
|
|
|
3136
3503
|
} catch (err) {
|
|
3137
3504
|
return out.failing(err);
|
|
3138
3505
|
}
|
|
3506
|
+
const keys = handles.map((h) => h.key());
|
|
3507
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3139
3508
|
if (verb === "andNot") {
|
|
3140
3509
|
const refs2 = operands.map((o) => o.ref);
|
|
3141
3510
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3142
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3511
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3143
3512
|
}
|
|
3144
3513
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3145
3514
|
const opts = this.refsIn(options, exclude);
|
|
3146
|
-
return
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3515
|
+
return known(
|
|
3516
|
+
this.guarded(
|
|
3517
|
+
out,
|
|
3518
|
+
handles,
|
|
3519
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3520
|
+
)
|
|
3150
3521
|
);
|
|
3151
3522
|
}
|
|
3152
3523
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
3153
3524
|
combineInto(name, dest, operands, options) {
|
|
3154
3525
|
const verb = name.slice(0, -"Into".length);
|
|
3155
|
-
const
|
|
3156
|
-
|
|
3526
|
+
const listed = segmentList(
|
|
3527
|
+
operands,
|
|
3528
|
+
verb === "andNot" ? `${name}: \`excludes\`` : `${name}: \`others\``
|
|
3529
|
+
);
|
|
3530
|
+
const excludes = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3531
|
+
const all = excludes.length > 0 ? [...listed, ...excludes] : listed;
|
|
3532
|
+
this.assertIntoHandles(name, dest, listed, excludes);
|
|
3157
3533
|
this.combineEngine([this, ...all]);
|
|
3158
3534
|
return this.timed(
|
|
3159
3535
|
name,
|
|
@@ -3167,53 +3543,31 @@ var Segment = class _Segment {
|
|
|
3167
3543
|
}
|
|
3168
3544
|
};
|
|
3169
3545
|
export {
|
|
3170
|
-
BudgetExceededError,
|
|
3171
|
-
BufferReader,
|
|
3546
|
+
BudgetExceededError2 as BudgetExceededError,
|
|
3172
3547
|
CapabilityError,
|
|
3173
3548
|
CloudRoaring,
|
|
3174
3549
|
CloudRoaringError,
|
|
3175
3550
|
CountingMetricsSink,
|
|
3176
|
-
CrbmReader,
|
|
3177
|
-
CrbmStorageChunkSource3 as CrbmStorageChunkSource,
|
|
3178
|
-
DEFAULT_RETRY_POLICY3 as DEFAULT_RETRY_POLICY,
|
|
3179
3551
|
InProcessKeystore,
|
|
3180
3552
|
IntegrityError3 as IntegrityError,
|
|
3181
3553
|
KeyUnavailableError,
|
|
3182
|
-
LEASE_SKEW_MS2 as LEASE_SKEW_MS,
|
|
3183
3554
|
LeaseExpiredError2 as LeaseExpiredError,
|
|
3184
3555
|
LeaseLimitError,
|
|
3185
3556
|
LocalFsStorage,
|
|
3186
|
-
MAX_LEASES_PER_SEGMENT,
|
|
3187
|
-
MAX_LEASE_MS2 as MAX_LEASE_MS,
|
|
3188
|
-
MIN_EXPIRES_AT_MS,
|
|
3189
3557
|
MemoryStorage,
|
|
3190
|
-
NodeAead,
|
|
3191
3558
|
NotFoundError2 as NotFoundError,
|
|
3192
3559
|
RecordingAuditSink,
|
|
3193
3560
|
Segment,
|
|
3194
3561
|
StaleOperandError,
|
|
3195
3562
|
TransientError,
|
|
3196
|
-
|
|
3197
|
-
|
|
3563
|
+
UnsupportedError3 as UnsupportedError,
|
|
3564
|
+
ValidationError6 as ValidationError,
|
|
3198
3565
|
WriteConflictError2 as WriteConflictError,
|
|
3199
|
-
aadFor,
|
|
3200
3566
|
deserializePortable,
|
|
3201
3567
|
destroySegment,
|
|
3202
3568
|
eraseNamespace,
|
|
3569
|
+
errorKind2 as errorKind,
|
|
3203
3570
|
excludingReservedRows2 as excludingReservedRows,
|
|
3204
|
-
isBudgetExceededError,
|
|
3205
|
-
isCapabilityError,
|
|
3206
|
-
isCloudRoaringError,
|
|
3207
|
-
isIntegrityError,
|
|
3208
|
-
isKeyUnavailableError,
|
|
3209
|
-
isLeaseExpiredError,
|
|
3210
|
-
isLeaseLimitError,
|
|
3211
|
-
isNotFoundError,
|
|
3212
|
-
isStaleOperandError,
|
|
3213
|
-
isTransientError,
|
|
3214
|
-
isUnsupportedError,
|
|
3215
|
-
isValidationError,
|
|
3216
|
-
isWriteConflictError,
|
|
3217
3571
|
readRetentionPolicy
|
|
3218
3572
|
};
|
|
3219
3573
|
//# sourceMappingURL=index.js.map
|