@cloudbitmaps/roaring 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +27 -11
- package/README.md +1 -1
- package/dist/bin/export-segments.d.ts +5 -1
- package/dist/bin/export-segments.js +497 -448
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +119 -47
- package/dist/index.js +474 -442
- package/dist/index.js.map +4 -4
- package/dist/lease-guards.d.ts +25 -13
- package/dist/open-charging-storage.d.ts +4 -2
- package/dist/options.d.ts +30 -0
- package/dist/portable/decode.d.ts +2 -1
- package/dist/portable/layout.d.ts +3 -2
- package/dist/testing/conformance.d.ts +2 -2
- package/dist/wire-store.d.ts +38 -0
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -3,20 +3,19 @@ import {
|
|
|
3
3
|
BoundedLru,
|
|
4
4
|
PinnedStorageChunkSource,
|
|
5
5
|
segmentKey,
|
|
6
|
-
CrbmStorageChunkSource,
|
|
6
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource2,
|
|
7
7
|
DEFAULT_BUDGET,
|
|
8
|
-
DEFAULT_RETRY_POLICY,
|
|
8
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2,
|
|
9
9
|
withRetry,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
RetryingStorageChunkSource,
|
|
10
|
+
NOOP_METRICS as NOOP_METRICS2,
|
|
11
|
+
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
13
12
|
SegmentEngine,
|
|
14
13
|
LEASE_SKEW_MS,
|
|
15
14
|
LeaseExpiredError,
|
|
16
15
|
MAX_LEASE_MS,
|
|
17
16
|
NotFoundError,
|
|
18
17
|
UnsupportedError,
|
|
19
|
-
ValidationError as
|
|
18
|
+
ValidationError as ValidationError4,
|
|
20
19
|
WriteConflictError,
|
|
21
20
|
checkBudget,
|
|
22
21
|
collectWithinBudget,
|
|
@@ -43,7 +42,7 @@ import {
|
|
|
43
42
|
splitId,
|
|
44
43
|
takeLease
|
|
45
44
|
} from "@cloudbitmaps/core";
|
|
46
|
-
import { pointerIdOf, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
45
|
+
import { pointerIdOf, requirePositiveInt, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
47
46
|
|
|
48
47
|
// src/open-charging-storage.ts
|
|
49
48
|
var OpenChargingStorage = class {
|
|
@@ -522,90 +521,265 @@ var OPTION_KEYS = {
|
|
|
522
521
|
};
|
|
523
522
|
|
|
524
523
|
// src/lease-guards.ts
|
|
525
|
-
var
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
return
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
check();
|
|
550
|
-
} catch (err) {
|
|
551
|
-
return stop(err);
|
|
552
|
-
}
|
|
553
|
-
chunk = key;
|
|
554
|
-
}
|
|
555
|
-
return r;
|
|
556
|
-
},
|
|
557
|
-
async return() {
|
|
558
|
-
await it.return?.();
|
|
559
|
-
return { done: true, value: void 0 };
|
|
524
|
+
var done = (value) => ({ done: true, value });
|
|
525
|
+
var guard = (inner, check, keyOf) => {
|
|
526
|
+
const it = inner[Symbol.asyncIterator]();
|
|
527
|
+
let first = true;
|
|
528
|
+
let over = false;
|
|
529
|
+
let last = -1;
|
|
530
|
+
const end = async (value) => {
|
|
531
|
+
over = true;
|
|
532
|
+
await it.return?.(value);
|
|
533
|
+
return value;
|
|
534
|
+
};
|
|
535
|
+
const stop = async (err) => {
|
|
536
|
+
await end();
|
|
537
|
+
throw err;
|
|
538
|
+
};
|
|
539
|
+
const deliver = (r) => {
|
|
540
|
+
if (over) return done();
|
|
541
|
+
if (r.done === true) return r;
|
|
542
|
+
const key = keyOf?.(r.value);
|
|
543
|
+
if (key === void 0 || key !== last) {
|
|
544
|
+
try {
|
|
545
|
+
check();
|
|
546
|
+
} catch (err) {
|
|
547
|
+
return stop(err);
|
|
560
548
|
}
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
await it.return?.();
|
|
574
|
-
throw err;
|
|
575
|
-
};
|
|
576
|
-
return {
|
|
577
|
-
async next() {
|
|
578
|
-
if (first) {
|
|
579
|
-
first = false;
|
|
580
|
-
try {
|
|
581
|
-
check();
|
|
582
|
-
} catch (err) {
|
|
583
|
-
return stop(err);
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
|
-
const r = await it.next();
|
|
587
|
-
if (r.done === true) return r;
|
|
549
|
+
if (key !== void 0) last = key;
|
|
550
|
+
}
|
|
551
|
+
return r;
|
|
552
|
+
};
|
|
553
|
+
return {
|
|
554
|
+
[Symbol.asyncIterator]() {
|
|
555
|
+
return this;
|
|
556
|
+
},
|
|
557
|
+
async next() {
|
|
558
|
+
if (over) return done();
|
|
559
|
+
if (first) {
|
|
560
|
+
first = false;
|
|
588
561
|
try {
|
|
589
562
|
check();
|
|
590
563
|
} catch (err) {
|
|
591
564
|
return stop(err);
|
|
592
565
|
}
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
566
|
+
}
|
|
567
|
+
return deliver(await it.next());
|
|
568
|
+
},
|
|
569
|
+
async return(value) {
|
|
570
|
+
return done(await end(value));
|
|
571
|
+
},
|
|
572
|
+
async throw(err) {
|
|
573
|
+
first = false;
|
|
574
|
+
if (it.throw === void 0) return stop(err);
|
|
575
|
+
let r;
|
|
576
|
+
try {
|
|
577
|
+
r = await it.throw(err);
|
|
578
|
+
} catch (thrown) {
|
|
579
|
+
over = true;
|
|
580
|
+
throw thrown;
|
|
581
|
+
}
|
|
582
|
+
return deliver(r);
|
|
583
|
+
}
|
|
584
|
+
};
|
|
585
|
+
};
|
|
586
|
+
var guardIdIterable = (inner, check) => guard(inner, check, (id) => id >>> 16);
|
|
587
|
+
var guardIds = (inner, check) => Object.assign(guardIdIterable(inner, check), {
|
|
588
|
+
batches: () => guardItems(inner.batches(), check)
|
|
589
|
+
});
|
|
590
|
+
var guardItems = (inner, check) => guard(inner, check);
|
|
591
|
+
|
|
592
|
+
// src/wire-store.ts
|
|
593
|
+
import {
|
|
594
|
+
CrbmStorageChunkSource,
|
|
595
|
+
DEFAULT_RETRY_POLICY,
|
|
596
|
+
NOOP_METRICS,
|
|
597
|
+
RetryingStorageChunkSource,
|
|
598
|
+
ValidationError as ValidationError2,
|
|
599
|
+
isStorageBackend
|
|
600
|
+
} from "@cloudbitmaps/core";
|
|
601
|
+
function resolveRetryPolicy(partial) {
|
|
602
|
+
const baseDelayMs = partial.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs;
|
|
603
|
+
return {
|
|
604
|
+
maxAttempts: partial.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
605
|
+
baseDelayMs,
|
|
606
|
+
maxDelayMs: partial.maxDelayMs ?? (typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs),
|
|
607
|
+
backoffFactor: partial.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
608
|
+
jitter: partial.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
function wireStore(options, clock, rng, metrics) {
|
|
612
|
+
const resolved = resolveStorageSource(options, clock);
|
|
613
|
+
if (metrics !== NOOP_METRICS && isStorageBackend(options.storage)) {
|
|
614
|
+
options.storage.attachMetrics?.(metrics);
|
|
615
|
+
}
|
|
616
|
+
let storage = resolved.source;
|
|
617
|
+
let retryOptions;
|
|
618
|
+
if (options.retry !== false) {
|
|
619
|
+
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
620
|
+
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
621
|
+
throw new ValidationError2("retry.onRetry must be a function");
|
|
622
|
+
}
|
|
623
|
+
retryOptions = {
|
|
624
|
+
clock,
|
|
625
|
+
rng,
|
|
626
|
+
policy: resolveRetryPolicy(ov),
|
|
627
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
628
|
+
onRetry: (info) => {
|
|
629
|
+
metrics.onEvent({
|
|
630
|
+
kind: "retry",
|
|
631
|
+
reason: "transient",
|
|
632
|
+
attempt: info.attempt,
|
|
633
|
+
delayMs: info.delayMs
|
|
634
|
+
});
|
|
635
|
+
userOnRetry?.(info);
|
|
598
636
|
}
|
|
599
637
|
};
|
|
638
|
+
storage = new RetryingStorageChunkSource(storage, retryOptions);
|
|
600
639
|
}
|
|
601
|
-
|
|
602
|
-
|
|
640
|
+
return {
|
|
641
|
+
storage,
|
|
642
|
+
source: resolved.source,
|
|
643
|
+
driver: resolved.driver,
|
|
644
|
+
registry: resolved.registry,
|
|
645
|
+
retryOptions
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
function resolveStorageSource(options, clock) {
|
|
649
|
+
const storage = options.storage;
|
|
650
|
+
if (storage === null || typeof storage !== "object") {
|
|
651
|
+
throw new ValidationError2(
|
|
652
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
const asBackend = storage;
|
|
656
|
+
const isObject = (v) => v !== null && typeof v === "object";
|
|
657
|
+
const hasGetChunk = typeof storage.getChunk === "function";
|
|
658
|
+
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
659
|
+
const isBackend = isStorageBackend(storage);
|
|
660
|
+
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
661
|
+
throw new ValidationError2(
|
|
662
|
+
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
const hasStorageHalf = isObject(asBackend.storage);
|
|
666
|
+
const hasRegistryHalf = isObject(asBackend.registry);
|
|
667
|
+
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
668
|
+
throw new ValidationError2(
|
|
669
|
+
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
673
|
+
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
674
|
+
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
675
|
+
if (!storageOk || !registryOk) {
|
|
676
|
+
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
677
|
+
throw new ValidationError2(
|
|
678
|
+
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
throw new ValidationError2(
|
|
682
|
+
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
if (isBackend) {
|
|
686
|
+
const backend = storage;
|
|
687
|
+
return {
|
|
688
|
+
source: new CrbmStorageChunkSource(backend.storage, {
|
|
689
|
+
registry: backend.registry,
|
|
690
|
+
keystore: options.encryption?.keystore,
|
|
691
|
+
requireEncryption: options.encryption?.required,
|
|
692
|
+
clock,
|
|
693
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
694
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
695
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
696
|
+
}),
|
|
697
|
+
driver: backend.storage,
|
|
698
|
+
registry: backend.registry
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
702
|
+
throw new ValidationError2(
|
|
703
|
+
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
704
|
+
);
|
|
705
|
+
}
|
|
706
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
707
|
+
throw new ValidationError2(
|
|
708
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
if (hasGetChunk) {
|
|
712
|
+
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
713
|
+
throw new ValidationError2(
|
|
714
|
+
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
return { source: storage, driver: void 0, registry: void 0 };
|
|
718
|
+
}
|
|
719
|
+
const driver = storage;
|
|
720
|
+
return {
|
|
721
|
+
source: new CrbmStorageChunkSource(driver, {
|
|
722
|
+
keystore: options.encryption?.keystore,
|
|
723
|
+
requireEncryption: options.encryption?.required,
|
|
724
|
+
clock,
|
|
725
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
726
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
727
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
728
|
+
}),
|
|
729
|
+
driver,
|
|
730
|
+
registry: void 0
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// src/options.ts
|
|
735
|
+
import { ValidationError as ValidationError3 } from "@cloudbitmaps/core";
|
|
736
|
+
var keysOf = () => (keys) => keys;
|
|
737
|
+
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
738
|
+
function optionsError(options, op, known, booleans = []) {
|
|
739
|
+
if (options === void 0 || options === null) return void 0;
|
|
740
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
741
|
+
return new ValidationError3(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
742
|
+
}
|
|
743
|
+
const bag = options;
|
|
744
|
+
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
745
|
+
if (unknown.length > 0) {
|
|
746
|
+
return new ValidationError3(
|
|
747
|
+
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
const audit = bag.audit;
|
|
751
|
+
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
752
|
+
return new ValidationError3(
|
|
753
|
+
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
754
|
+
);
|
|
755
|
+
}
|
|
756
|
+
for (const name of booleans) {
|
|
757
|
+
const v = bag[name];
|
|
758
|
+
if (v !== void 0 && typeof v !== "boolean") {
|
|
759
|
+
return new ValidationError3(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
return void 0;
|
|
763
|
+
}
|
|
764
|
+
function picked(options, keys) {
|
|
765
|
+
const out = {};
|
|
766
|
+
if (options === void 0 || options === null) return out;
|
|
767
|
+
for (const key of keys) {
|
|
768
|
+
const value = options[key];
|
|
769
|
+
if (value !== void 0) out[key] = value;
|
|
770
|
+
}
|
|
771
|
+
return out;
|
|
772
|
+
}
|
|
773
|
+
function checkOptions(options, op, known, booleans) {
|
|
774
|
+
const err = optionsError(options, op, known, booleans);
|
|
775
|
+
if (err !== void 0) throw err;
|
|
776
|
+
}
|
|
603
777
|
|
|
604
778
|
// src/index.ts
|
|
605
779
|
import {
|
|
606
780
|
MemoryStorage,
|
|
607
781
|
LocalFsStorage,
|
|
608
|
-
CrbmStorageChunkSource as
|
|
782
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource3,
|
|
609
783
|
destroySegment,
|
|
610
784
|
eraseNamespace,
|
|
611
785
|
excludingReservedRows as excludingReservedRows2,
|
|
@@ -618,7 +792,7 @@ import {
|
|
|
618
792
|
NodeAead,
|
|
619
793
|
aadFor,
|
|
620
794
|
CloudRoaringError,
|
|
621
|
-
ValidationError as
|
|
795
|
+
ValidationError as ValidationError5,
|
|
622
796
|
WriteConflictError as WriteConflictError2,
|
|
623
797
|
IntegrityError as IntegrityError3,
|
|
624
798
|
NotFoundError as NotFoundError2,
|
|
@@ -647,7 +821,7 @@ import {
|
|
|
647
821
|
BufferReader,
|
|
648
822
|
CountingMetricsSink,
|
|
649
823
|
RecordingAuditSink,
|
|
650
|
-
DEFAULT_RETRY_POLICY as
|
|
824
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY3
|
|
651
825
|
} from "@cloudbitmaps/core";
|
|
652
826
|
var SystemRng = class {
|
|
653
827
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -663,14 +837,12 @@ function rowVersionOfLive(version) {
|
|
|
663
837
|
return cut < 0 ? version : version.slice(0, cut);
|
|
664
838
|
}
|
|
665
839
|
function validateConcurrency(concurrency) {
|
|
666
|
-
|
|
667
|
-
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
668
|
-
}
|
|
840
|
+
requirePositiveInt("concurrency", concurrency);
|
|
669
841
|
}
|
|
670
842
|
function requireScope(options, op) {
|
|
671
843
|
if (options.namespace !== void 0) {
|
|
672
844
|
if (typeof options.namespace !== "string") {
|
|
673
|
-
throw new
|
|
845
|
+
throw new ValidationError4(
|
|
674
846
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
675
847
|
);
|
|
676
848
|
}
|
|
@@ -678,12 +850,11 @@ function requireScope(options, op) {
|
|
|
678
850
|
}
|
|
679
851
|
refuseReservedNamespace(options.namespace);
|
|
680
852
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
681
|
-
throw new
|
|
853
|
+
throw new ValidationError4(
|
|
682
854
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
683
855
|
);
|
|
684
856
|
}
|
|
685
857
|
}
|
|
686
|
-
var keysOf = () => (keys) => keys;
|
|
687
858
|
var CALL_KEYS = {
|
|
688
859
|
range: keysOf()(["after", "through"]),
|
|
689
860
|
andNot: keysOf()([
|
|
@@ -743,7 +914,7 @@ var CALL_KEYS = {
|
|
|
743
914
|
]),
|
|
744
915
|
rollback: keysOf()([
|
|
745
916
|
"audit",
|
|
746
|
-
"
|
|
917
|
+
"allowUnknown"
|
|
747
918
|
]),
|
|
748
919
|
dropSegment: keysOf()([
|
|
749
920
|
"confirmSegment",
|
|
@@ -767,50 +938,12 @@ var CALL_KEYS = {
|
|
|
767
938
|
checkConsistency: keysOf()([
|
|
768
939
|
"namespace",
|
|
769
940
|
"concurrency",
|
|
770
|
-
"summaries"
|
|
941
|
+
"summaries",
|
|
942
|
+
"generations"
|
|
771
943
|
]),
|
|
772
944
|
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
773
945
|
};
|
|
774
|
-
var
|
|
775
|
-
function optionsError(options, op, known, booleans = []) {
|
|
776
|
-
if (options === void 0 || options === null) return void 0;
|
|
777
|
-
if (typeof options !== "object" || Array.isArray(options)) {
|
|
778
|
-
return new ValidationError2(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
779
|
-
}
|
|
780
|
-
const bag = options;
|
|
781
|
-
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
782
|
-
if (unknown.length > 0) {
|
|
783
|
-
return new ValidationError2(
|
|
784
|
-
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
785
|
-
);
|
|
786
|
-
}
|
|
787
|
-
const audit = bag.audit;
|
|
788
|
-
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
789
|
-
return new ValidationError2(
|
|
790
|
-
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
791
|
-
);
|
|
792
|
-
}
|
|
793
|
-
for (const name of booleans) {
|
|
794
|
-
const v = bag[name];
|
|
795
|
-
if (v !== void 0 && typeof v !== "boolean") {
|
|
796
|
-
return new ValidationError2(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
return void 0;
|
|
800
|
-
}
|
|
801
|
-
function picked(options, keys) {
|
|
802
|
-
const out = {};
|
|
803
|
-
if (options === void 0 || options === null) return out;
|
|
804
|
-
for (const key of keys) {
|
|
805
|
-
const value = options[key];
|
|
806
|
-
if (value !== void 0) out[key] = value;
|
|
807
|
-
}
|
|
808
|
-
return out;
|
|
809
|
-
}
|
|
810
|
-
function checkOptions(options, op, known, booleans) {
|
|
811
|
-
const err = optionsError(options, op, known, booleans);
|
|
812
|
-
if (err !== void 0) throw err;
|
|
813
|
-
}
|
|
946
|
+
var SEGMENT_BRAND = /* @__PURE__ */ Symbol.for("cloudbitmaps.roaring.segment/1");
|
|
814
947
|
var viewOf;
|
|
815
948
|
var MATERIALIZE_MANY_KEYS = [
|
|
816
949
|
"operands",
|
|
@@ -842,91 +975,6 @@ var DEFAULT_MAX_BUFFERED_BYTES = 256 * 1024 * 1024;
|
|
|
842
975
|
var DEFAULT_PUBLISH_CONCURRENCY = 8;
|
|
843
976
|
var DEFAULT_BATCH_CONCURRENCY = 1;
|
|
844
977
|
var PIN_PARALLELISM = 16;
|
|
845
|
-
function resolveStorageSource(options, clock) {
|
|
846
|
-
const storage = options.storage;
|
|
847
|
-
if (storage === null || typeof storage !== "object") {
|
|
848
|
-
throw new ValidationError2(
|
|
849
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
850
|
-
);
|
|
851
|
-
}
|
|
852
|
-
const asBackend = storage;
|
|
853
|
-
const isObject = (v) => v !== null && typeof v === "object";
|
|
854
|
-
const hasGetChunk = typeof storage.getChunk === "function";
|
|
855
|
-
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
856
|
-
const isBackend = isStorageBackend(storage);
|
|
857
|
-
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
858
|
-
throw new ValidationError2(
|
|
859
|
-
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
860
|
-
);
|
|
861
|
-
}
|
|
862
|
-
const hasStorageHalf = isObject(asBackend.storage);
|
|
863
|
-
const hasRegistryHalf = isObject(asBackend.registry);
|
|
864
|
-
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
865
|
-
throw new ValidationError2(
|
|
866
|
-
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
867
|
-
);
|
|
868
|
-
}
|
|
869
|
-
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
870
|
-
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
871
|
-
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
872
|
-
if (!storageOk || !registryOk) {
|
|
873
|
-
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
874
|
-
throw new ValidationError2(
|
|
875
|
-
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
876
|
-
);
|
|
877
|
-
}
|
|
878
|
-
throw new ValidationError2(
|
|
879
|
-
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
880
|
-
);
|
|
881
|
-
}
|
|
882
|
-
if (isBackend) {
|
|
883
|
-
const backend = storage;
|
|
884
|
-
return {
|
|
885
|
-
source: new CrbmStorageChunkSource(backend.storage, {
|
|
886
|
-
registry: backend.registry,
|
|
887
|
-
keystore: options.encryption?.keystore,
|
|
888
|
-
requireEncryption: options.encryption?.required,
|
|
889
|
-
clock,
|
|
890
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
891
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
892
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
893
|
-
}),
|
|
894
|
-
driver: backend.storage,
|
|
895
|
-
registry: backend.registry
|
|
896
|
-
};
|
|
897
|
-
}
|
|
898
|
-
if (hasGetChunk && hasPutImmutable) {
|
|
899
|
-
throw new ValidationError2(
|
|
900
|
-
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
901
|
-
);
|
|
902
|
-
}
|
|
903
|
-
if (!hasGetChunk && !hasPutImmutable) {
|
|
904
|
-
throw new ValidationError2(
|
|
905
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
906
|
-
);
|
|
907
|
-
}
|
|
908
|
-
if (hasGetChunk) {
|
|
909
|
-
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
910
|
-
throw new ValidationError2(
|
|
911
|
-
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
912
|
-
);
|
|
913
|
-
}
|
|
914
|
-
return { source: storage, driver: void 0, registry: void 0 };
|
|
915
|
-
}
|
|
916
|
-
const driver = storage;
|
|
917
|
-
return {
|
|
918
|
-
source: new CrbmStorageChunkSource(driver, {
|
|
919
|
-
keystore: options.encryption?.keystore,
|
|
920
|
-
requireEncryption: options.encryption?.required,
|
|
921
|
-
clock,
|
|
922
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
923
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
924
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
925
|
-
}),
|
|
926
|
-
driver,
|
|
927
|
-
registry: void 0
|
|
928
|
-
};
|
|
929
|
-
}
|
|
930
978
|
var NOT_A_GROUP = {
|
|
931
979
|
Null: "null",
|
|
932
980
|
Array: "an array",
|
|
@@ -969,7 +1017,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
969
1017
|
*/
|
|
970
1018
|
static rejectUnknownOptions(options) {
|
|
971
1019
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
972
|
-
throw new
|
|
1020
|
+
throw new ValidationError4(
|
|
973
1021
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
974
1022
|
);
|
|
975
1023
|
}
|
|
@@ -987,7 +1035,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
987
1035
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
988
1036
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
989
1037
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
990
|
-
throw new
|
|
1038
|
+
throw new ValidationError4(
|
|
991
1039
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
992
1040
|
);
|
|
993
1041
|
}
|
|
@@ -997,7 +1045,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
997
1045
|
if (unknown.length === 0) return;
|
|
998
1046
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
999
1047
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1000
|
-
throw new
|
|
1048
|
+
throw new ValidationError4(
|
|
1001
1049
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1002
1050
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1003
1051
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1017,7 +1065,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1017
1065
|
if (seam === void 0 || seam === null) continue;
|
|
1018
1066
|
const has = (m) => typeof seam[m] === "function";
|
|
1019
1067
|
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1020
|
-
throw new
|
|
1068
|
+
throw new ValidationError4(
|
|
1021
1069
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1022
1070
|
);
|
|
1023
1071
|
}
|
|
@@ -1028,50 +1076,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1028
1076
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1029
1077
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1030
1078
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1031
|
-
const metrics = safeMetrics(options.metrics ??
|
|
1079
|
+
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS2);
|
|
1032
1080
|
const cache = new BoundedLru({
|
|
1033
1081
|
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
1034
1082
|
ttlMs: options.cache?.ttlMs,
|
|
1035
1083
|
clock
|
|
1036
1084
|
});
|
|
1037
|
-
const
|
|
1038
|
-
|
|
1039
|
-
options.storage.attachMetrics?.(metrics);
|
|
1040
|
-
}
|
|
1041
|
-
let storage = resolved.source;
|
|
1042
|
-
if (options.retry !== false) {
|
|
1043
|
-
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
1044
|
-
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
1045
|
-
throw new ValidationError2("retry.onRetry must be a function");
|
|
1046
|
-
}
|
|
1047
|
-
const policy = {
|
|
1048
|
-
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
1049
|
-
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
1050
|
-
maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
1051
|
-
backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
1052
|
-
jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
1053
|
-
};
|
|
1054
|
-
const retryOpts = {
|
|
1055
|
-
clock,
|
|
1056
|
-
rng,
|
|
1057
|
-
policy,
|
|
1058
|
-
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
1059
|
-
onRetry: (info) => {
|
|
1060
|
-
metrics.onEvent({
|
|
1061
|
-
kind: "retry",
|
|
1062
|
-
reason: "transient",
|
|
1063
|
-
attempt: info.attempt,
|
|
1064
|
-
delayMs: info.delayMs
|
|
1065
|
-
});
|
|
1066
|
-
userOnRetry?.(info);
|
|
1067
|
-
}
|
|
1068
|
-
};
|
|
1069
|
-
storage = new RetryingStorageChunkSource(storage, retryOpts);
|
|
1070
|
-
this.retryOptions = retryOpts;
|
|
1071
|
-
}
|
|
1085
|
+
const wired = wireStore(options, clock, rng, metrics);
|
|
1086
|
+
this.retryOptions = wired.retryOptions;
|
|
1072
1087
|
this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
|
|
1073
1088
|
const deps = {
|
|
1074
|
-
storage,
|
|
1089
|
+
storage: wired.storage,
|
|
1075
1090
|
cache,
|
|
1076
1091
|
codec: roaringCodec,
|
|
1077
1092
|
// the facade injects the flagship codec; core stays codec-agnostic
|
|
@@ -1081,12 +1096,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1081
1096
|
};
|
|
1082
1097
|
this.engine = new SegmentEngine(deps);
|
|
1083
1098
|
this.cache = cache;
|
|
1084
|
-
this.crbmSource =
|
|
1099
|
+
this.crbmSource = wired.source instanceof CrbmStorageChunkSource2 ? wired.source : void 0;
|
|
1085
1100
|
this.clock = clock;
|
|
1086
1101
|
this.rng = rng;
|
|
1087
1102
|
this.metrics = metrics;
|
|
1088
|
-
this.storageDriver =
|
|
1089
|
-
this.registry =
|
|
1103
|
+
this.storageDriver = wired.driver;
|
|
1104
|
+
this.registry = wired.registry;
|
|
1090
1105
|
this.keystore = options.encryption?.keystore;
|
|
1091
1106
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1092
1107
|
}
|
|
@@ -1130,11 +1145,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1130
1145
|
segment(name, options) {
|
|
1131
1146
|
if (options !== void 0 && options !== null) {
|
|
1132
1147
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1133
|
-
throw new
|
|
1148
|
+
throw new ValidationError4("segment: options must be an object such as { namespace }");
|
|
1134
1149
|
}
|
|
1135
1150
|
const proto = Object.getPrototypeOf(options);
|
|
1136
1151
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1137
|
-
throw new
|
|
1152
|
+
throw new ValidationError4(
|
|
1138
1153
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1139
1154
|
);
|
|
1140
1155
|
}
|
|
@@ -1142,7 +1157,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1142
1157
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1143
1158
|
);
|
|
1144
1159
|
if (unknown.length > 0) {
|
|
1145
|
-
throw new
|
|
1160
|
+
throw new ValidationError4(
|
|
1146
1161
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1147
1162
|
);
|
|
1148
1163
|
}
|
|
@@ -1177,7 +1192,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1177
1192
|
* row it is kept (hard invariant 1: deleting it after a purge-and-recreate would put a live row over a missing
|
|
1178
1193
|
* generation).
|
|
1179
1194
|
*
|
|
1180
|
-
* Where the row is gone, or keyed over a cleartext write, the object
|
|
1195
|
+
* Where the row is gone, or keyed over a cleartext write, the object's footer must match what the load wrote, and
|
|
1181
1196
|
* the delete passes the version that read reported, so on a storage driver that reports `conditionalDelete` an
|
|
1182
1197
|
* object put under the number since is kept. Where the row is unchanged or `destroyed`, the check narrows the
|
|
1183
1198
|
* window rather than closing it: the row read and the delete are two round trips, and the delete is by number,
|
|
@@ -1232,9 +1247,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1232
1247
|
}
|
|
1233
1248
|
const reason = result.reason;
|
|
1234
1249
|
if (reason === "superseded") {
|
|
1235
|
-
const wrote = result.size > 0 ? `generation ${result.generation} was written and
|
|
1250
|
+
const wrote = result.size > 0 ? `generation ${result.generation} was written and may not be current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
1236
1251
|
throw new WriteConflictError(
|
|
1237
|
-
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result
|
|
1252
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result may not be the destination's current generation: ${wrote}. The pointer may have moved, the row may have been created or rewritten (a retention policy does either), dropped or purged. Re-read the destination and re-run.`
|
|
1238
1253
|
);
|
|
1239
1254
|
}
|
|
1240
1255
|
return {
|
|
@@ -1251,7 +1266,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1251
1266
|
/** The one body of the forms above. */
|
|
1252
1267
|
async materializeMany(options) {
|
|
1253
1268
|
if (typeof options !== "object" || options === null) {
|
|
1254
|
-
throw new
|
|
1269
|
+
throw new ValidationError4(
|
|
1255
1270
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1256
1271
|
);
|
|
1257
1272
|
}
|
|
@@ -1260,13 +1275,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1260
1275
|
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1261
1276
|
);
|
|
1262
1277
|
if (unknown.length > 0) {
|
|
1263
|
-
throw new
|
|
1278
|
+
throw new ValidationError4(
|
|
1264
1279
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1265
1280
|
);
|
|
1266
1281
|
}
|
|
1267
1282
|
const dryRunIn = options.dryRun;
|
|
1268
1283
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1269
|
-
throw new
|
|
1284
|
+
throw new ValidationError4("materializeMany: dryRun must be a boolean");
|
|
1270
1285
|
}
|
|
1271
1286
|
const dryRun = dryRunIn === true;
|
|
1272
1287
|
const operandsIn = options.operands;
|
|
@@ -1278,7 +1293,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1278
1293
|
const feedIn = options.feed;
|
|
1279
1294
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1280
1295
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1281
|
-
throw new
|
|
1296
|
+
throw new ValidationError4(
|
|
1282
1297
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1283
1298
|
);
|
|
1284
1299
|
}
|
|
@@ -1288,22 +1303,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1288
1303
|
const budgetOption = options.budget;
|
|
1289
1304
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1290
1305
|
if (typeof pinOption !== "boolean")
|
|
1291
|
-
throw new
|
|
1306
|
+
throw new ValidationError4("materializeMany: pin must be a boolean");
|
|
1292
1307
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1293
|
-
throw new
|
|
1308
|
+
throw new ValidationError4("materializeMany: allowAbsentOperands must be a boolean");
|
|
1294
1309
|
}
|
|
1295
1310
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1296
|
-
throw new
|
|
1311
|
+
throw new ValidationError4("materializeMany: operands must be an object of name to segment");
|
|
1297
1312
|
}
|
|
1298
1313
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1299
|
-
throw new
|
|
1314
|
+
throw new ValidationError4("materializeMany: outputs must be a non-empty array");
|
|
1300
1315
|
}
|
|
1301
1316
|
const names = Object.keys(operandsIn);
|
|
1302
1317
|
const handles = /* @__PURE__ */ new Map();
|
|
1303
1318
|
for (const name of names) {
|
|
1304
1319
|
const handle = operandsIn[name];
|
|
1305
1320
|
if (!(handle instanceof Segment)) {
|
|
1306
|
-
throw new
|
|
1321
|
+
throw new ValidationError4(
|
|
1307
1322
|
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1308
1323
|
);
|
|
1309
1324
|
}
|
|
@@ -1311,14 +1326,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1311
1326
|
}
|
|
1312
1327
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1313
1328
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1314
|
-
throw new
|
|
1329
|
+
throw new ValidationError4(
|
|
1315
1330
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1316
1331
|
);
|
|
1317
1332
|
}
|
|
1318
1333
|
if (mayBeEmptyIn.length > 0) {
|
|
1319
1334
|
const first = mayBeEmptyIn[0];
|
|
1320
1335
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1321
|
-
throw new
|
|
1336
|
+
throw new ValidationError4(
|
|
1322
1337
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1323
1338
|
);
|
|
1324
1339
|
}
|
|
@@ -1328,28 +1343,28 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1328
1343
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1329
1344
|
outputsIn.forEach((raw, i) => {
|
|
1330
1345
|
if (typeof raw !== "object" || raw === null) {
|
|
1331
|
-
throw new
|
|
1346
|
+
throw new ValidationError4(`outputs[${i}] must be an object`);
|
|
1332
1347
|
}
|
|
1333
1348
|
const bad = Object.keys(raw).filter(
|
|
1334
1349
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1335
1350
|
);
|
|
1336
1351
|
if (bad.length > 0) {
|
|
1337
|
-
throw new
|
|
1352
|
+
throw new ValidationError4(
|
|
1338
1353
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1339
1354
|
);
|
|
1340
1355
|
}
|
|
1341
1356
|
const spec = raw;
|
|
1342
1357
|
const dest = spec.dest;
|
|
1343
1358
|
if (!(dest instanceof Segment)) {
|
|
1344
|
-
throw new
|
|
1359
|
+
throw new ValidationError4(`outputs[${i}].dest must be a segment from store.segment()`);
|
|
1345
1360
|
}
|
|
1346
1361
|
if (operandKeys.has(dest.key())) {
|
|
1347
|
-
throw new
|
|
1362
|
+
throw new ValidationError4(
|
|
1348
1363
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1349
1364
|
);
|
|
1350
1365
|
}
|
|
1351
1366
|
if (destKeys.has(dest.key())) {
|
|
1352
|
-
throw new
|
|
1367
|
+
throw new ValidationError4(
|
|
1353
1368
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1354
1369
|
);
|
|
1355
1370
|
}
|
|
@@ -1369,7 +1384,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1369
1384
|
...pinnedAt === void 0 ? {} : {
|
|
1370
1385
|
pinnedGeneration: pinnedAt.generation,
|
|
1371
1386
|
pinnedVersion: pinnedAt.version,
|
|
1372
|
-
pinnedFingerprint: pinnedAt.fingerprint
|
|
1387
|
+
pinnedFingerprint: pinnedAt.fingerprint
|
|
1373
1388
|
},
|
|
1374
1389
|
current: (read) => this.currentRowOf(seg.ref, read?.fingerprint === true)
|
|
1375
1390
|
};
|
|
@@ -1422,13 +1437,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1422
1437
|
held.set(key, { at, handle });
|
|
1423
1438
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1424
1439
|
if (!samePin(prior.at, at)) {
|
|
1425
|
-
throw new
|
|
1440
|
+
throw new ValidationError4(
|
|
1426
1441
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1427
1442
|
);
|
|
1428
1443
|
}
|
|
1429
1444
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1430
1445
|
if (!pinOption) {
|
|
1431
|
-
throw new
|
|
1446
|
+
throw new ValidationError4(
|
|
1432
1447
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1433
1448
|
);
|
|
1434
1449
|
}
|
|
@@ -1442,7 +1457,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1442
1457
|
else if (pinOption) toPin.push([key, handle]);
|
|
1443
1458
|
else pins.set(key, void 0);
|
|
1444
1459
|
}
|
|
1445
|
-
const metricsOn = this.metrics !==
|
|
1460
|
+
const metricsOn = this.metrics !== NOOP_METRICS2;
|
|
1446
1461
|
const work = async () => {
|
|
1447
1462
|
const taken = await mapWithConcurrency(
|
|
1448
1463
|
toPin,
|
|
@@ -1452,7 +1467,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1452
1467
|
for (const [key, at] of taken) pins.set(key, at);
|
|
1453
1468
|
const live = new Map([...pins].filter((e) => e[1] !== void 0));
|
|
1454
1469
|
const pinnedSource = new PinnedStorageChunkSource(crbm, live);
|
|
1455
|
-
const source = this.retryOptions === void 0 ? pinnedSource : new
|
|
1470
|
+
const source = this.retryOptions === void 0 ? pinnedSource : new RetryingStorageChunkSource2(pinnedSource, this.retryOptions);
|
|
1456
1471
|
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1457
1472
|
return runCombineMany(final, {
|
|
1458
1473
|
source,
|
|
@@ -1516,7 +1531,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1516
1531
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1517
1532
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1518
1533
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1519
|
-
throw new
|
|
1534
|
+
throw new ValidationError4(
|
|
1520
1535
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1521
1536
|
);
|
|
1522
1537
|
}
|
|
@@ -1754,6 +1769,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1754
1769
|
* What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
|
|
1755
1770
|
* the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
|
|
1756
1771
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1772
|
+
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1773
|
+
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1757
1774
|
*
|
|
1758
1775
|
* Needs a backend.
|
|
1759
1776
|
*/
|
|
@@ -1838,36 +1855,58 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1838
1855
|
* (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
|
|
1839
1856
|
* one cannot.
|
|
1840
1857
|
*
|
|
1841
|
-
* It refuses rather than guesses
|
|
1842
|
-
*
|
|
1843
|
-
*
|
|
1844
|
-
*
|
|
1845
|
-
*
|
|
1858
|
+
* It refuses rather than guesses. The row records, in `knownGens`, the generations its loads and rollbacks made
|
|
1859
|
+
* current, each with the footer checksum of its object, and a rollback moves only onto one of those: a generation the
|
|
1860
|
+
* row does not know (a load that wrote its object and died before the publish, a refused load's object, an object
|
|
1861
|
+
* sealed under a key no row holds, a load published after the point a registry was restored to) throws
|
|
1862
|
+
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1863
|
+
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1864
|
+
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1865
|
+
* crypto-shredded segment throws {@link ValidationError} because every generation of it is unreadable, and a target
|
|
1866
|
+
* that is cleartext under an encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1867
|
+
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1868
|
+
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1869
|
+
* gone it throws `NotFoundError` naming what is.
|
|
1846
1870
|
*
|
|
1847
1871
|
* The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
|
|
1848
1872
|
* metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
|
|
1849
1873
|
* what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
|
|
1850
|
-
* key
|
|
1874
|
+
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1875
|
+
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1876
|
+
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1877
|
+
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one that landed, the row being the
|
|
1878
|
+
* one the swap leaves, is checked as any is, and otherwise its `TransientError` is thrown. After the swap the target's
|
|
1879
|
+
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1851
1880
|
*
|
|
1852
1881
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
1853
|
-
* and are then *above* `currentGen`, where collection never looks
|
|
1854
|
-
*
|
|
1855
|
-
* below its pointer
|
|
1882
|
+
* and are then *above* `currentGen`, where collection never looks, and still known to the row, so rolling forward onto
|
|
1883
|
+
* one needs no flag. They remain until loads pass them (the first load whose
|
|
1884
|
+
* number one of them holds numbers above them all, and collection then keeps `keep` of what is below its pointer, the
|
|
1885
|
+
* generation that load superseded first), {@link CloudRoaring.dropSegment}
|
|
1856
1886
|
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and, when the
|
|
1857
1887
|
* current generation does not hold the id, only those holding it and any no read of the segment can open, whatever
|
|
1858
1888
|
* the id. An operator who has just undone a bad load should not have the
|
|
1859
1889
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
1860
1890
|
* erasure undoable.
|
|
1861
1891
|
*
|
|
1862
|
-
*
|
|
1863
|
-
*
|
|
1864
|
-
*
|
|
1892
|
+
* `{ allowUnknown: true }` moves onto a present object the row does not vouch for, or one whose checksum differs from
|
|
1893
|
+
* the recorded one, as long as it is the segment's kind of object, opens, and names the generation, and records it as
|
|
1894
|
+
* known (over a usable record; a row with none stays without one). On an encrypted segment, outside the two cases the pointer alone decides (below), it also needs the key: with no keystore, or one that holds none of the key's KEKs or
|
|
1895
|
+
* one that does not unwrap it, it is refused with `KeyUnavailableError` before the target is read, since an object
|
|
1896
|
+
* sealed under a key no row holds passes every check that needs no key, and a key service that did not answer in
|
|
1897
|
+
* time is thrown as `TransientError`, to retry. Every move the row did not vouch for is audited with `unknown: true`.
|
|
1898
|
+
* Two cases are decided by the pointer alone: a target below it needs no flag, no key and no checksum check, and the
|
|
1899
|
+
* move is audited the same way. They are a row with no usable `knownGens` (a registry that drops the field, or a pointer
|
|
1900
|
+
* moved by hand), and a target numbered below the bound of a full record (128 entries or more, leased ones included),
|
|
1901
|
+
* which is its least recently current entry that no lease names (an `*Into` destination keeps every generation in the
|
|
1902
|
+
* bucket and names only the newest 128). A generation a publish with a smaller `keep` no longer names but that stays in
|
|
1903
|
+
* the bucket (a lease spared it, or its delete failed) needs `allowUnknown` until a later listing collects it.
|
|
1865
1904
|
*
|
|
1866
1905
|
* Needs a backend.
|
|
1867
1906
|
*/
|
|
1868
1907
|
async rollback(ref, toGeneration, options = {}) {
|
|
1869
1908
|
validateSegmentRef(ref);
|
|
1870
|
-
checkOptions(options, "rollback", CALL_KEYS.rollback, ["
|
|
1909
|
+
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1871
1910
|
options ??= {};
|
|
1872
1911
|
const deps = this.lifecycleDeps("rollback");
|
|
1873
1912
|
try {
|
|
@@ -1924,12 +1963,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1924
1963
|
* published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
|
|
1925
1964
|
*
|
|
1926
1965
|
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
1927
|
-
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only
|
|
1928
|
-
*
|
|
1929
|
-
* encrypted segment's bucket. Under a row that is gone or keyed,
|
|
1930
|
-
* load
|
|
1931
|
-
*
|
|
1932
|
-
*
|
|
1966
|
+
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only in
|
|
1967
|
+
* its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1968
|
+
* encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
|
|
1969
|
+
* load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
|
|
1970
|
+
* writer's object, and a footer names bytes, not the load that wrote them, so it leaves the orphan rather than risk
|
|
1971
|
+
* deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
|
|
1972
|
+
* generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
|
|
1973
|
+
* with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
|
|
1974
|
+
* after carry it until the window pushes it out. Until it goes, a rollback onto it is refused unless `allowUnknown` is passed, since the list never names it (a list seeded
|
|
1975
|
+
* over a row with none can); with the override it publishes what no load published.
|
|
1933
1976
|
*
|
|
1934
1977
|
* **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
|
|
1935
1978
|
* that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
|
|
@@ -2020,7 +2063,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2020
2063
|
async dropSegment(ref, options) {
|
|
2021
2064
|
validateSegmentRef(ref);
|
|
2022
2065
|
if (options === void 0 || options === null) {
|
|
2023
|
-
throw new
|
|
2066
|
+
throw new ValidationError4("dropSegment: options { confirmSegment } are required");
|
|
2024
2067
|
}
|
|
2025
2068
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2026
2069
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2188,7 +2231,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2188
2231
|
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
2189
2232
|
*
|
|
2190
2233
|
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
2191
|
-
* object back, the pin reads it again.
|
|
2234
|
+
* object back, the pin reads it again. A read of the segment already running on this store, a pin's and a combine's
|
|
2235
|
+
* with a pinned operand included, caches nothing more of what it began reading before the call, and a pinned one
|
|
2236
|
+
* opens its object again before its next chunk, failing with `NotFoundError` if the object is gone or replaced, or
|
|
2237
|
+
* its row is gone or destroyed.
|
|
2192
2238
|
*
|
|
2193
2239
|
* Synchronous, best-effort, and safe to call for a segment this store has never read.
|
|
2194
2240
|
*
|
|
@@ -2208,7 +2254,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2208
2254
|
*
|
|
2209
2255
|
* The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
|
|
2210
2256
|
* pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
|
|
2211
|
-
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
2257
|
+
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}). It
|
|
2258
|
+
* shares the store engine's invalidations too, so the store's writes reach a pinned read already running.
|
|
2212
2259
|
*/
|
|
2213
2260
|
async pinSegment(ref, leaseUntil, named) {
|
|
2214
2261
|
const crbm = this.crbmSource;
|
|
@@ -2287,7 +2334,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2287
2334
|
* An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
|
|
2288
2335
|
* retries, so a transient fault on a pinned read is retried as it would be on a live one.
|
|
2289
2336
|
*
|
|
2290
|
-
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment
|
|
2337
|
+
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment, and the
|
|
2338
|
+
* store engine's invalidations, so an invalidation of a pinned segment reaches a read of the pin already running.
|
|
2291
2339
|
*/
|
|
2292
2340
|
engineWithPins(pins) {
|
|
2293
2341
|
const crbm = this.crbmSource;
|
|
@@ -2298,12 +2346,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2298
2346
|
}
|
|
2299
2347
|
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
2300
2348
|
return new SegmentEngine({
|
|
2301
|
-
storage: this.retryOptions === void 0 ? pinned : new
|
|
2349
|
+
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource2(pinned, this.retryOptions),
|
|
2302
2350
|
cache: this.cache,
|
|
2303
2351
|
codec: roaringCodec,
|
|
2304
2352
|
clock: this.clock,
|
|
2305
2353
|
metrics: this.metrics,
|
|
2306
|
-
budget: this.budget
|
|
2354
|
+
budget: this.budget,
|
|
2355
|
+
sharesInvalidationsWith: this.engine
|
|
2307
2356
|
});
|
|
2308
2357
|
}
|
|
2309
2358
|
/**
|
|
@@ -2329,7 +2378,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2329
2378
|
if (held.has(key)) {
|
|
2330
2379
|
const before = held.get(key);
|
|
2331
2380
|
if (!samePin(before, at)) {
|
|
2332
|
-
throw new
|
|
2381
|
+
throw new ValidationError4(
|
|
2333
2382
|
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`intersectInto(dest, [])\``
|
|
2334
2383
|
);
|
|
2335
2384
|
}
|
|
@@ -2343,7 +2392,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2343
2392
|
/** `op` under the store's retries, as the store's own reads run; bare when the store has retries off. */
|
|
2344
2393
|
withRetries(op) {
|
|
2345
2394
|
const r = this.retryOptions;
|
|
2346
|
-
return r === void 0 ? op() : withRetry(op, r.policy ??
|
|
2395
|
+
return r === void 0 ? op() : withRetry(op, r.policy ?? DEFAULT_RETRY_POLICY2, r);
|
|
2347
2396
|
}
|
|
2348
2397
|
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
2349
2398
|
requireRegistry(op) {
|
|
@@ -2372,13 +2421,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2372
2421
|
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
2373
2422
|
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
2374
2423
|
*
|
|
2424
|
+
* With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
|
|
2425
|
+
* included, against its object: one footer read each (104 bytes, no key, not retried), reporting
|
|
2426
|
+
* `known-generation-mismatch` with the `generation` where the object under that number is not the one the row
|
|
2427
|
+
* recorded. A named generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot
|
|
2428
|
+
* be used is counted in `generationsUnchecked`, and none of its objects is read.
|
|
2429
|
+
*
|
|
2375
2430
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
2376
2431
|
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
2377
2432
|
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
2378
2433
|
* `storage` and `registry` with a higher `maxScanSegments`.
|
|
2379
2434
|
*/
|
|
2380
2435
|
async checkConsistency(options = {}) {
|
|
2381
|
-
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2436
|
+
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2437
|
+
"summaries",
|
|
2438
|
+
"generations"
|
|
2439
|
+
]);
|
|
2382
2440
|
options ??= {};
|
|
2383
2441
|
const deps = this.lifecycleDeps("checkConsistency");
|
|
2384
2442
|
return runConsistencyCheck(
|
|
@@ -2406,13 +2464,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2406
2464
|
}
|
|
2407
2465
|
};
|
|
2408
2466
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
2409
|
-
var
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2467
|
+
var refusing = (err) => (
|
|
2468
|
+
// eslint-disable-next-line require-yield
|
|
2469
|
+
async function* () {
|
|
2470
|
+
throw err;
|
|
2471
|
+
}
|
|
2472
|
+
);
|
|
2473
|
+
var failing = (err) => withBatches(refusing(err)(), refusing(err));
|
|
2416
2474
|
var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
|
|
2417
2475
|
var twoPins = (a, b) => {
|
|
2418
2476
|
const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
|
|
@@ -2430,9 +2488,12 @@ var readOptions = (options) => ({
|
|
|
2430
2488
|
});
|
|
2431
2489
|
var NO_SEGMENTS = [];
|
|
2432
2490
|
function segmentList(list, what) {
|
|
2433
|
-
if (!Array.isArray(list)) throw new
|
|
2491
|
+
if (!Array.isArray(list)) throw new ValidationError4(`${what} must be an array of segments`);
|
|
2434
2492
|
return list;
|
|
2435
2493
|
}
|
|
2494
|
+
function operandsLabel(verb) {
|
|
2495
|
+
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2496
|
+
}
|
|
2436
2497
|
var AS_IDS = {
|
|
2437
2498
|
failing,
|
|
2438
2499
|
guard: guardIds,
|
|
@@ -2447,10 +2508,8 @@ var CombineChunks = class {
|
|
|
2447
2508
|
chunks;
|
|
2448
2509
|
};
|
|
2449
2510
|
var AS_CHUNKS = {
|
|
2450
|
-
guard:
|
|
2451
|
-
failing: (err) => (
|
|
2452
|
-
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
|
|
2453
|
-
}),
|
|
2511
|
+
guard: guardItems,
|
|
2512
|
+
failing: (err) => refusing(err)(),
|
|
2454
2513
|
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2455
2514
|
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2456
2515
|
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
@@ -2488,6 +2547,7 @@ var Segment = class _Segment {
|
|
|
2488
2547
|
lease;
|
|
2489
2548
|
leaseHold;
|
|
2490
2549
|
static {
|
|
2550
|
+
Object.defineProperty(_Segment.prototype, SEGMENT_BRAND, { value: true });
|
|
2491
2551
|
makeSegment = (parts) => {
|
|
2492
2552
|
minting = true;
|
|
2493
2553
|
try {
|
|
@@ -2508,7 +2568,7 @@ var Segment = class _Segment {
|
|
|
2508
2568
|
*/
|
|
2509
2569
|
constructor(parts) {
|
|
2510
2570
|
if (!minting) {
|
|
2511
|
-
throw new
|
|
2571
|
+
throw new ValidationError4(
|
|
2512
2572
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2513
2573
|
);
|
|
2514
2574
|
}
|
|
@@ -2524,7 +2584,7 @@ var Segment = class _Segment {
|
|
|
2524
2584
|
if (parts.lease !== void 0) {
|
|
2525
2585
|
this.lease = Object.freeze({ holder: parts.lease.holder, until: parts.lease.until });
|
|
2526
2586
|
}
|
|
2527
|
-
this.metricsOn = parts.metrics !==
|
|
2587
|
+
this.metricsOn = parts.metrics !== NOOP_METRICS2;
|
|
2528
2588
|
}
|
|
2529
2589
|
/**
|
|
2530
2590
|
* **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
|
|
@@ -2579,11 +2639,16 @@ var Segment = class _Segment {
|
|
|
2579
2639
|
* - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
|
|
2580
2640
|
*
|
|
2581
2641
|
* An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
|
|
2582
|
-
* destroyed.
|
|
2642
|
+
* destroyed. A read of the pin already running when the invalidation comes, a combine on this store with the pin as
|
|
2643
|
+
* an operand included, looks the pin up again before it hands out its next chunk: it carries on with what it holds
|
|
2644
|
+
* where the object is still there, and fails with `NotFoundError` where it is gone. It caches nothing more of what it
|
|
2645
|
+
* began reading before the invalidation, so the pin's next read opens the object again too.
|
|
2583
2646
|
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
2584
2647
|
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
2585
2648
|
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
2586
|
-
* {@link CloudRoaring.invalidate} is called on this store.
|
|
2649
|
+
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store that takes the pin as an
|
|
2650
|
+
* operand reads it through that store's reader and chunk cache, which only that store's own writes and its
|
|
2651
|
+
* `invalidate` reach. Where the
|
|
2587
2652
|
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
2588
2653
|
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
2589
2654
|
*
|
|
@@ -2606,7 +2671,7 @@ var Segment = class _Segment {
|
|
|
2606
2671
|
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
2607
2672
|
*/
|
|
2608
2673
|
async pin(options) {
|
|
2609
|
-
this.
|
|
2674
|
+
this.assertLease();
|
|
2610
2675
|
const leaseUntil = this.leaseUntilOf("pin", options);
|
|
2611
2676
|
return this.pinned(this.ref, leaseUntil);
|
|
2612
2677
|
}
|
|
@@ -2618,11 +2683,11 @@ var Segment = class _Segment {
|
|
|
2618
2683
|
leaseUntilOf(op, options) {
|
|
2619
2684
|
if (options === void 0 || options === null) return void 0;
|
|
2620
2685
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2621
|
-
throw new
|
|
2686
|
+
throw new ValidationError4(`${op}: options must be an object such as { leaseUntil }`);
|
|
2622
2687
|
}
|
|
2623
2688
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2624
2689
|
if (unknown.length > 0) {
|
|
2625
|
-
throw new
|
|
2690
|
+
throw new ValidationError4(
|
|
2626
2691
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2627
2692
|
);
|
|
2628
2693
|
}
|
|
@@ -2630,17 +2695,17 @@ var Segment = class _Segment {
|
|
|
2630
2695
|
if (until === void 0) return void 0;
|
|
2631
2696
|
const now = this.clock.now();
|
|
2632
2697
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2633
|
-
throw new
|
|
2698
|
+
throw new ValidationError4(
|
|
2634
2699
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2635
2700
|
);
|
|
2636
2701
|
}
|
|
2637
2702
|
if (until <= now) {
|
|
2638
|
-
throw new
|
|
2703
|
+
throw new ValidationError4(
|
|
2639
2704
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2640
2705
|
);
|
|
2641
2706
|
}
|
|
2642
2707
|
if (until > now + MAX_LEASE_MS) {
|
|
2643
|
-
throw new
|
|
2708
|
+
throw new ValidationError4(
|
|
2644
2709
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2645
2710
|
);
|
|
2646
2711
|
}
|
|
@@ -2696,10 +2761,12 @@ var Segment = class _Segment {
|
|
|
2696
2761
|
*/
|
|
2697
2762
|
assertSegments(handles) {
|
|
2698
2763
|
for (const h of handles) {
|
|
2699
|
-
if (h instanceof _Segment || typeof h === "object" && h !== null &&
|
|
2764
|
+
if (h instanceof _Segment || typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
2700
2765
|
continue;
|
|
2701
2766
|
}
|
|
2702
|
-
throw new
|
|
2767
|
+
throw new ValidationError4(
|
|
2768
|
+
"an operand must be a segment from `store.segment()` of a compatible copy of @cloudbitmaps/roaring; install one version of it"
|
|
2769
|
+
);
|
|
2703
2770
|
}
|
|
2704
2771
|
}
|
|
2705
2772
|
/**
|
|
@@ -2712,6 +2779,11 @@ var Segment = class _Segment {
|
|
|
2712
2779
|
if (err !== void 0) throw err;
|
|
2713
2780
|
}
|
|
2714
2781
|
}
|
|
2782
|
+
/** Throw this handle's lease error, if it has one. The calls that return a value instead of throwing use {@link leaseError}. */
|
|
2783
|
+
assertLease() {
|
|
2784
|
+
const err = this.leaseError();
|
|
2785
|
+
if (err !== void 0) throw err;
|
|
2786
|
+
}
|
|
2715
2787
|
/**
|
|
2716
2788
|
* Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
|
|
2717
2789
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
@@ -2735,16 +2807,16 @@ var Segment = class _Segment {
|
|
|
2735
2807
|
* is in `options`, the second.
|
|
2736
2808
|
*/
|
|
2737
2809
|
async pinAt(at, options) {
|
|
2738
|
-
this.
|
|
2810
|
+
this.assertLease();
|
|
2739
2811
|
const { generation, fingerprint } = at ?? {};
|
|
2740
2812
|
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => k !== "generation" && k !== "fingerprint") : [];
|
|
2741
2813
|
if (unknown.length > 0) {
|
|
2742
|
-
throw new
|
|
2814
|
+
throw new ValidationError4(
|
|
2743
2815
|
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { generation, fingerprint } only`
|
|
2744
2816
|
);
|
|
2745
2817
|
}
|
|
2746
2818
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2747
|
-
throw new
|
|
2819
|
+
throw new ValidationError4(
|
|
2748
2820
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2749
2821
|
);
|
|
2750
2822
|
}
|
|
@@ -2778,8 +2850,9 @@ var Segment = class _Segment {
|
|
|
2778
2850
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
2779
2851
|
}
|
|
2780
2852
|
/**
|
|
2781
|
-
* Time an op with the injected clock and emit an `op` metric on completion (success or throw).
|
|
2782
|
-
*
|
|
2853
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). With no sink wired it
|
|
2854
|
+
* reads no clock and emits nothing, but a call still passes through this async frame; `has()` and `count()` return the
|
|
2855
|
+
* engine's promise directly instead.
|
|
2783
2856
|
*/
|
|
2784
2857
|
async timed(name, fn) {
|
|
2785
2858
|
if (!this.metricsOn) return fn();
|
|
@@ -2790,10 +2863,15 @@ var Segment = class _Segment {
|
|
|
2790
2863
|
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
2791
2864
|
}
|
|
2792
2865
|
}
|
|
2793
|
-
/**
|
|
2866
|
+
/**
|
|
2867
|
+
* Membership: one chunk. A cold resolution of the segment is one registry read and a warm one none; then a cached
|
|
2868
|
+
* chunk answers with no request, and an uncached one is one range read, after a tail read when no reader of the
|
|
2869
|
+
* generation is open, unless that tail carried the chunk. Throws {@link ValidationError} on a bad id.
|
|
2870
|
+
*/
|
|
2794
2871
|
has(id) {
|
|
2795
2872
|
const lease = this.leaseError();
|
|
2796
2873
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2874
|
+
if (!this.metricsOn) return this.engine.has(this.ref, id);
|
|
2797
2875
|
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
2798
2876
|
}
|
|
2799
2877
|
/**
|
|
@@ -2817,6 +2895,7 @@ var Segment = class _Segment {
|
|
|
2817
2895
|
count() {
|
|
2818
2896
|
const lease = this.leaseError();
|
|
2819
2897
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2898
|
+
if (!this.metricsOn) return this.engine.count(this.ref);
|
|
2820
2899
|
return this.timed("count", () => this.engine.count(this.ref));
|
|
2821
2900
|
}
|
|
2822
2901
|
/**
|
|
@@ -2847,7 +2926,7 @@ var Segment = class _Segment {
|
|
|
2847
2926
|
* ```
|
|
2848
2927
|
*/
|
|
2849
2928
|
async stat() {
|
|
2850
|
-
this.
|
|
2929
|
+
this.assertLease();
|
|
2851
2930
|
return this.engine.stat(this.ref);
|
|
2852
2931
|
}
|
|
2853
2932
|
/**
|
|
@@ -2877,7 +2956,7 @@ var Segment = class _Segment {
|
|
|
2877
2956
|
engine.iterate(ref, bounds),
|
|
2878
2957
|
() => engine.iterateBatches(ref, bounds)
|
|
2879
2958
|
);
|
|
2880
|
-
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.
|
|
2959
|
+
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease());
|
|
2881
2960
|
}
|
|
2882
2961
|
/**
|
|
2883
2962
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -2911,22 +2990,22 @@ var Segment = class _Segment {
|
|
|
2911
2990
|
*/
|
|
2912
2991
|
everyNth(n, options) {
|
|
2913
2992
|
const lease = this.leaseError();
|
|
2914
|
-
if (lease !== void 0) return
|
|
2993
|
+
if (lease !== void 0) return refusing(lease)();
|
|
2915
2994
|
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
2916
|
-
if (bad !== void 0) return
|
|
2995
|
+
if (bad !== void 0) return refusing(bad)();
|
|
2917
2996
|
if (this.pinnedAt === void 0) {
|
|
2918
|
-
return
|
|
2997
|
+
return refusing(
|
|
2919
2998
|
new UnsupportedError(
|
|
2920
2999
|
"everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
|
|
2921
3000
|
)
|
|
2922
|
-
);
|
|
3001
|
+
)();
|
|
2923
3002
|
}
|
|
2924
3003
|
const stream = this.engine.everyNth(
|
|
2925
3004
|
this.ref,
|
|
2926
3005
|
n,
|
|
2927
3006
|
options == null ? void 0 : rangeOf(options)
|
|
2928
3007
|
);
|
|
2929
|
-
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.
|
|
3008
|
+
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease());
|
|
2930
3009
|
}
|
|
2931
3010
|
/**
|
|
2932
3011
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -2966,28 +3045,7 @@ var Segment = class _Segment {
|
|
|
2966
3045
|
intersect(others, options) {
|
|
2967
3046
|
const bad = optionsError(options, "intersect", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
2968
3047
|
if (bad !== void 0) return failing(bad);
|
|
2969
|
-
return this.
|
|
2970
|
-
}
|
|
2971
|
-
intersectAs(out, others, options) {
|
|
2972
|
-
let exclude;
|
|
2973
|
-
let handles;
|
|
2974
|
-
try {
|
|
2975
|
-
exclude = this.excludesOf(options);
|
|
2976
|
-
handles = [this, ...segmentList(others, "intersect: `others`"), ...exclude];
|
|
2977
|
-
this.assertSegments(handles);
|
|
2978
|
-
this.assertLeases(handles);
|
|
2979
|
-
} catch (err) {
|
|
2980
|
-
return out.failing(err);
|
|
2981
|
-
}
|
|
2982
|
-
let engine;
|
|
2983
|
-
try {
|
|
2984
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
2985
|
-
} catch (err) {
|
|
2986
|
-
return out.failing(err);
|
|
2987
|
-
}
|
|
2988
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
2989
|
-
const opts = this.refsIn(options, exclude);
|
|
2990
|
-
return this.guarded(out, handles, out.intersect(engine, refs, opts));
|
|
3048
|
+
return this.combineAs(AS_IDS, "intersect", others, options ?? void 0);
|
|
2991
3049
|
}
|
|
2992
3050
|
/**
|
|
2993
3051
|
* Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
|
|
@@ -3011,21 +3069,7 @@ var Segment = class _Segment {
|
|
|
3011
3069
|
*/
|
|
3012
3070
|
async intersectInto(dest, others, options) {
|
|
3013
3071
|
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3014
|
-
|
|
3015
|
-
...segmentList(others, "intersectInto: `others`"),
|
|
3016
|
-
...this.excludesOf(options)
|
|
3017
|
-
];
|
|
3018
|
-
this.assertIntoHandles(dest, operands);
|
|
3019
|
-
this.combineEngine([this, ...operands]);
|
|
3020
|
-
return this.timed(
|
|
3021
|
-
"intersectInto",
|
|
3022
|
-
() => this.materialize(
|
|
3023
|
-
dest.ref,
|
|
3024
|
-
new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
|
|
3025
|
-
"intersectInto",
|
|
3026
|
-
options
|
|
3027
|
-
)
|
|
3028
|
-
);
|
|
3072
|
+
return this.combineInto("intersectInto", dest, others, options);
|
|
3029
3073
|
}
|
|
3030
3074
|
/**
|
|
3031
3075
|
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
@@ -3040,44 +3084,12 @@ var Segment = class _Segment {
|
|
|
3040
3084
|
union(others, options) {
|
|
3041
3085
|
const bad = optionsError(options, "union", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
3042
3086
|
if (bad !== void 0) return failing(bad);
|
|
3043
|
-
return this.
|
|
3044
|
-
}
|
|
3045
|
-
unionAs(out, others, options) {
|
|
3046
|
-
let exclude;
|
|
3047
|
-
let handles;
|
|
3048
|
-
try {
|
|
3049
|
-
exclude = this.excludesOf(options);
|
|
3050
|
-
handles = [this, ...segmentList(others, "union: `others`"), ...exclude];
|
|
3051
|
-
this.assertSegments(handles);
|
|
3052
|
-
this.assertLeases(handles);
|
|
3053
|
-
} catch (err) {
|
|
3054
|
-
return out.failing(err);
|
|
3055
|
-
}
|
|
3056
|
-
let engine;
|
|
3057
|
-
try {
|
|
3058
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
3059
|
-
} catch (err) {
|
|
3060
|
-
return out.failing(err);
|
|
3061
|
-
}
|
|
3062
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
3063
|
-
const opts = this.refsIn(options, exclude);
|
|
3064
|
-
return this.guarded(out, handles, out.union(engine, refs, opts));
|
|
3087
|
+
return this.combineAs(AS_IDS, "union", others, options ?? void 0);
|
|
3065
3088
|
}
|
|
3066
3089
|
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3067
3090
|
async unionInto(dest, others, options) {
|
|
3068
3091
|
checkOptions(options, "unionInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3069
|
-
|
|
3070
|
-
this.assertIntoHandles(dest, operands);
|
|
3071
|
-
this.combineEngine([this, ...operands]);
|
|
3072
|
-
return this.timed(
|
|
3073
|
-
"unionInto",
|
|
3074
|
-
() => this.materialize(
|
|
3075
|
-
dest.ref,
|
|
3076
|
-
new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
|
|
3077
|
-
"unionInto",
|
|
3078
|
-
options
|
|
3079
|
-
)
|
|
3080
|
-
);
|
|
3092
|
+
return this.combineInto("unionInto", dest, others, options);
|
|
3081
3093
|
}
|
|
3082
3094
|
/**
|
|
3083
3095
|
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
@@ -3093,12 +3105,26 @@ var Segment = class _Segment {
|
|
|
3093
3105
|
andNot(excludes, options) {
|
|
3094
3106
|
const bad = optionsError(options, "andNot", CALL_KEYS.andNot, ["allowAbsentOperands"]);
|
|
3095
3107
|
if (bad !== void 0) return failing(bad);
|
|
3096
|
-
return this.
|
|
3108
|
+
return this.combineAs(AS_IDS, "andNot", excludes, options ?? void 0);
|
|
3097
3109
|
}
|
|
3098
|
-
|
|
3110
|
+
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3111
|
+
async andNotInto(dest, excludes, options) {
|
|
3112
|
+
checkOptions(options, "andNotInto", CALL_KEYS.andNotInto, [
|
|
3113
|
+
"allowEmpty",
|
|
3114
|
+
"allowAbsentOperands"
|
|
3115
|
+
]);
|
|
3116
|
+
return this.combineInto("andNotInto", dest, excludes, options);
|
|
3117
|
+
}
|
|
3118
|
+
/**
|
|
3119
|
+
* The one body of the three combines, read as ids or as chunks. `operands` is `others` for `intersect` and
|
|
3120
|
+
* `union`, and `excludes` for `andNot`, which takes no `exclude` option.
|
|
3121
|
+
*/
|
|
3122
|
+
combineAs(out, verb, operands, options) {
|
|
3123
|
+
let exclude;
|
|
3099
3124
|
let handles;
|
|
3100
3125
|
try {
|
|
3101
|
-
|
|
3126
|
+
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3127
|
+
handles = [this, ...segmentList(operands, operandsLabel(verb)), ...exclude];
|
|
3102
3128
|
this.assertSegments(handles);
|
|
3103
3129
|
this.assertLeases(handles);
|
|
3104
3130
|
} catch (err) {
|
|
@@ -3106,29 +3132,35 @@ var Segment = class _Segment {
|
|
|
3106
3132
|
}
|
|
3107
3133
|
let engine;
|
|
3108
3134
|
try {
|
|
3109
|
-
engine = this.combineEngine([this, ...
|
|
3135
|
+
engine = this.combineEngine([this, ...operands, ...exclude]) ?? this.engine;
|
|
3110
3136
|
} catch (err) {
|
|
3111
3137
|
return out.failing(err);
|
|
3112
3138
|
}
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3139
|
+
if (verb === "andNot") {
|
|
3140
|
+
const refs2 = operands.map((o) => o.ref);
|
|
3141
|
+
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3142
|
+
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3143
|
+
}
|
|
3144
|
+
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3145
|
+
const opts = this.refsIn(options, exclude);
|
|
3146
|
+
return this.guarded(
|
|
3147
|
+
out,
|
|
3148
|
+
handles,
|
|
3149
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3150
|
+
);
|
|
3117
3151
|
}
|
|
3118
|
-
/**
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
]);
|
|
3124
|
-
this.assertIntoHandles(dest, segmentList(excludes, "andNotInto: `excludes`"));
|
|
3125
|
-
this.combineEngine([this, ...excludes]);
|
|
3152
|
+
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
3153
|
+
combineInto(name, dest, operands, options) {
|
|
3154
|
+
const verb = name.slice(0, -"Into".length);
|
|
3155
|
+
const all = verb === "andNot" ? segmentList(operands, `${name}: \`excludes\``) : [...segmentList(operands, `${name}: \`others\``), ...this.excludesOf(options)];
|
|
3156
|
+
this.assertIntoHandles(dest, all);
|
|
3157
|
+
this.combineEngine([this, ...all]);
|
|
3126
3158
|
return this.timed(
|
|
3127
|
-
|
|
3159
|
+
name,
|
|
3128
3160
|
() => this.materialize(
|
|
3129
3161
|
dest.ref,
|
|
3130
|
-
new CombineChunks(this.
|
|
3131
|
-
|
|
3162
|
+
new CombineChunks(this.combineAs(AS_CHUNKS, verb, operands, options)),
|
|
3163
|
+
name,
|
|
3132
3164
|
options
|
|
3133
3165
|
)
|
|
3134
3166
|
);
|
|
@@ -3142,8 +3174,8 @@ export {
|
|
|
3142
3174
|
CloudRoaringError,
|
|
3143
3175
|
CountingMetricsSink,
|
|
3144
3176
|
CrbmReader,
|
|
3145
|
-
|
|
3146
|
-
|
|
3177
|
+
CrbmStorageChunkSource3 as CrbmStorageChunkSource,
|
|
3178
|
+
DEFAULT_RETRY_POLICY3 as DEFAULT_RETRY_POLICY,
|
|
3147
3179
|
InProcessKeystore,
|
|
3148
3180
|
IntegrityError3 as IntegrityError,
|
|
3149
3181
|
KeyUnavailableError,
|
|
@@ -3162,7 +3194,7 @@ export {
|
|
|
3162
3194
|
StaleOperandError,
|
|
3163
3195
|
TransientError,
|
|
3164
3196
|
UnsupportedError2 as UnsupportedError,
|
|
3165
|
-
|
|
3197
|
+
ValidationError5 as ValidationError,
|
|
3166
3198
|
WriteConflictError2 as WriteConflictError,
|
|
3167
3199
|
aadFor,
|
|
3168
3200
|
deserializePortable,
|