@cloudbitmaps/roaring 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +27 -11
- package/README.md +1 -1
- package/dist/bin/export-segments.d.ts +5 -1
- package/dist/bin/export-segments.js +497 -448
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +119 -47
- package/dist/index.js +474 -442
- package/dist/index.js.map +4 -4
- package/dist/lease-guards.d.ts +25 -13
- package/dist/open-charging-storage.d.ts +4 -2
- package/dist/options.d.ts +30 -0
- package/dist/portable/decode.d.ts +2 -1
- package/dist/portable/layout.d.ts +3 -2
- package/dist/testing/conformance.d.ts +2 -2
- package/dist/wire-store.d.ts +38 -0
- package/package.json +2 -2
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// src/bin/export-segments.ts
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { realpathSync } from "node:fs";
|
|
6
|
-
import { lstat, mkdir, open, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
6
|
+
import { lstat, mkdir, open, readdir, rename, rm, rmdir, writeFile } from "node:fs/promises";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import { pathToFileURL } from "node:url";
|
|
9
9
|
import { encodeNameForPath, namespacePathPart } from "@cloudbitmaps/core/driver-kit";
|
|
@@ -13,20 +13,19 @@ import {
|
|
|
13
13
|
BoundedLru,
|
|
14
14
|
PinnedStorageChunkSource,
|
|
15
15
|
segmentKey,
|
|
16
|
-
CrbmStorageChunkSource,
|
|
16
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource2,
|
|
17
17
|
DEFAULT_BUDGET,
|
|
18
|
-
DEFAULT_RETRY_POLICY,
|
|
18
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2,
|
|
19
19
|
withRetry,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
RetryingStorageChunkSource,
|
|
20
|
+
NOOP_METRICS as NOOP_METRICS2,
|
|
21
|
+
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
23
22
|
SegmentEngine,
|
|
24
23
|
LEASE_SKEW_MS,
|
|
25
24
|
LeaseExpiredError,
|
|
26
25
|
MAX_LEASE_MS,
|
|
27
26
|
NotFoundError,
|
|
28
27
|
UnsupportedError,
|
|
29
|
-
ValidationError as
|
|
28
|
+
ValidationError as ValidationError4,
|
|
30
29
|
WriteConflictError,
|
|
31
30
|
checkBudget,
|
|
32
31
|
collectWithinBudget,
|
|
@@ -53,7 +52,7 @@ import {
|
|
|
53
52
|
splitId,
|
|
54
53
|
takeLease
|
|
55
54
|
} from "@cloudbitmaps/core";
|
|
56
|
-
import { pointerIdOf, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
55
|
+
import { pointerIdOf, requirePositiveInt, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
57
56
|
|
|
58
57
|
// src/open-charging-storage.ts
|
|
59
58
|
var OpenChargingStorage = class {
|
|
@@ -529,90 +528,265 @@ var OPTION_KEYS = {
|
|
|
529
528
|
};
|
|
530
529
|
|
|
531
530
|
// src/lease-guards.ts
|
|
532
|
-
var
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
return
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
check();
|
|
557
|
-
} catch (err) {
|
|
558
|
-
return stop(err);
|
|
559
|
-
}
|
|
560
|
-
chunk = key;
|
|
561
|
-
}
|
|
562
|
-
return r;
|
|
563
|
-
},
|
|
564
|
-
async return() {
|
|
565
|
-
await it.return?.();
|
|
566
|
-
return { done: true, value: void 0 };
|
|
531
|
+
var done = (value) => ({ done: true, value });
|
|
532
|
+
var guard = (inner, check, keyOf) => {
|
|
533
|
+
const it = inner[Symbol.asyncIterator]();
|
|
534
|
+
let first = true;
|
|
535
|
+
let over = false;
|
|
536
|
+
let last = -1;
|
|
537
|
+
const end = async (value) => {
|
|
538
|
+
over = true;
|
|
539
|
+
await it.return?.(value);
|
|
540
|
+
return value;
|
|
541
|
+
};
|
|
542
|
+
const stop = async (err) => {
|
|
543
|
+
await end();
|
|
544
|
+
throw err;
|
|
545
|
+
};
|
|
546
|
+
const deliver = (r) => {
|
|
547
|
+
if (over) return done();
|
|
548
|
+
if (r.done === true) return r;
|
|
549
|
+
const key = keyOf?.(r.value);
|
|
550
|
+
if (key === void 0 || key !== last) {
|
|
551
|
+
try {
|
|
552
|
+
check();
|
|
553
|
+
} catch (err) {
|
|
554
|
+
return stop(err);
|
|
567
555
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
await it.return?.();
|
|
581
|
-
throw err;
|
|
582
|
-
};
|
|
583
|
-
return {
|
|
584
|
-
async next() {
|
|
585
|
-
if (first) {
|
|
586
|
-
first = false;
|
|
587
|
-
try {
|
|
588
|
-
check();
|
|
589
|
-
} catch (err) {
|
|
590
|
-
return stop(err);
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
const r = await it.next();
|
|
594
|
-
if (r.done === true) return r;
|
|
556
|
+
if (key !== void 0) last = key;
|
|
557
|
+
}
|
|
558
|
+
return r;
|
|
559
|
+
};
|
|
560
|
+
return {
|
|
561
|
+
[Symbol.asyncIterator]() {
|
|
562
|
+
return this;
|
|
563
|
+
},
|
|
564
|
+
async next() {
|
|
565
|
+
if (over) return done();
|
|
566
|
+
if (first) {
|
|
567
|
+
first = false;
|
|
595
568
|
try {
|
|
596
569
|
check();
|
|
597
570
|
} catch (err) {
|
|
598
571
|
return stop(err);
|
|
599
572
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
573
|
+
}
|
|
574
|
+
return deliver(await it.next());
|
|
575
|
+
},
|
|
576
|
+
async return(value) {
|
|
577
|
+
return done(await end(value));
|
|
578
|
+
},
|
|
579
|
+
async throw(err) {
|
|
580
|
+
first = false;
|
|
581
|
+
if (it.throw === void 0) return stop(err);
|
|
582
|
+
let r;
|
|
583
|
+
try {
|
|
584
|
+
r = await it.throw(err);
|
|
585
|
+
} catch (thrown) {
|
|
586
|
+
over = true;
|
|
587
|
+
throw thrown;
|
|
588
|
+
}
|
|
589
|
+
return deliver(r);
|
|
590
|
+
}
|
|
591
|
+
};
|
|
592
|
+
};
|
|
593
|
+
var guardIdIterable = (inner, check) => guard(inner, check, (id) => id >>> 16);
|
|
594
|
+
var guardIds = (inner, check) => Object.assign(guardIdIterable(inner, check), {
|
|
595
|
+
batches: () => guardItems(inner.batches(), check)
|
|
596
|
+
});
|
|
597
|
+
var guardItems = (inner, check) => guard(inner, check);
|
|
598
|
+
|
|
599
|
+
// src/wire-store.ts
|
|
600
|
+
import {
|
|
601
|
+
CrbmStorageChunkSource,
|
|
602
|
+
DEFAULT_RETRY_POLICY,
|
|
603
|
+
NOOP_METRICS,
|
|
604
|
+
RetryingStorageChunkSource,
|
|
605
|
+
ValidationError as ValidationError2,
|
|
606
|
+
isStorageBackend
|
|
607
|
+
} from "@cloudbitmaps/core";
|
|
608
|
+
function resolveRetryPolicy(partial) {
|
|
609
|
+
const baseDelayMs = partial.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs;
|
|
610
|
+
return {
|
|
611
|
+
maxAttempts: partial.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
612
|
+
baseDelayMs,
|
|
613
|
+
maxDelayMs: partial.maxDelayMs ?? (typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs),
|
|
614
|
+
backoffFactor: partial.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
615
|
+
jitter: partial.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
function wireStore(options, clock, rng, metrics) {
|
|
619
|
+
const resolved = resolveStorageSource(options, clock);
|
|
620
|
+
if (metrics !== NOOP_METRICS && isStorageBackend(options.storage)) {
|
|
621
|
+
options.storage.attachMetrics?.(metrics);
|
|
622
|
+
}
|
|
623
|
+
let storage = resolved.source;
|
|
624
|
+
let retryOptions;
|
|
625
|
+
if (options.retry !== false) {
|
|
626
|
+
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
627
|
+
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
628
|
+
throw new ValidationError2("retry.onRetry must be a function");
|
|
629
|
+
}
|
|
630
|
+
retryOptions = {
|
|
631
|
+
clock,
|
|
632
|
+
rng,
|
|
633
|
+
policy: resolveRetryPolicy(ov),
|
|
634
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
635
|
+
onRetry: (info) => {
|
|
636
|
+
metrics.onEvent({
|
|
637
|
+
kind: "retry",
|
|
638
|
+
reason: "transient",
|
|
639
|
+
attempt: info.attempt,
|
|
640
|
+
delayMs: info.delayMs
|
|
641
|
+
});
|
|
642
|
+
userOnRetry?.(info);
|
|
605
643
|
}
|
|
606
644
|
};
|
|
645
|
+
storage = new RetryingStorageChunkSource(storage, retryOptions);
|
|
607
646
|
}
|
|
608
|
-
|
|
609
|
-
|
|
647
|
+
return {
|
|
648
|
+
storage,
|
|
649
|
+
source: resolved.source,
|
|
650
|
+
driver: resolved.driver,
|
|
651
|
+
registry: resolved.registry,
|
|
652
|
+
retryOptions
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
function resolveStorageSource(options, clock) {
|
|
656
|
+
const storage = options.storage;
|
|
657
|
+
if (storage === null || typeof storage !== "object") {
|
|
658
|
+
throw new ValidationError2(
|
|
659
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
660
|
+
);
|
|
661
|
+
}
|
|
662
|
+
const asBackend = storage;
|
|
663
|
+
const isObject = (v) => v !== null && typeof v === "object";
|
|
664
|
+
const hasGetChunk = typeof storage.getChunk === "function";
|
|
665
|
+
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
666
|
+
const isBackend = isStorageBackend(storage);
|
|
667
|
+
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
668
|
+
throw new ValidationError2(
|
|
669
|
+
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
const hasStorageHalf = isObject(asBackend.storage);
|
|
673
|
+
const hasRegistryHalf = isObject(asBackend.registry);
|
|
674
|
+
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
675
|
+
throw new ValidationError2(
|
|
676
|
+
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
680
|
+
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
681
|
+
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
682
|
+
if (!storageOk || !registryOk) {
|
|
683
|
+
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
684
|
+
throw new ValidationError2(
|
|
685
|
+
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
throw new ValidationError2(
|
|
689
|
+
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
if (isBackend) {
|
|
693
|
+
const backend = storage;
|
|
694
|
+
return {
|
|
695
|
+
source: new CrbmStorageChunkSource(backend.storage, {
|
|
696
|
+
registry: backend.registry,
|
|
697
|
+
keystore: options.encryption?.keystore,
|
|
698
|
+
requireEncryption: options.encryption?.required,
|
|
699
|
+
clock,
|
|
700
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
701
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
702
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
703
|
+
}),
|
|
704
|
+
driver: backend.storage,
|
|
705
|
+
registry: backend.registry
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
709
|
+
throw new ValidationError2(
|
|
710
|
+
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
714
|
+
throw new ValidationError2(
|
|
715
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
if (hasGetChunk) {
|
|
719
|
+
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
720
|
+
throw new ValidationError2(
|
|
721
|
+
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
return { source: storage, driver: void 0, registry: void 0 };
|
|
725
|
+
}
|
|
726
|
+
const driver = storage;
|
|
727
|
+
return {
|
|
728
|
+
source: new CrbmStorageChunkSource(driver, {
|
|
729
|
+
keystore: options.encryption?.keystore,
|
|
730
|
+
requireEncryption: options.encryption?.required,
|
|
731
|
+
clock,
|
|
732
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
733
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
734
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
735
|
+
}),
|
|
736
|
+
driver,
|
|
737
|
+
registry: void 0
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/options.ts
|
|
742
|
+
import { ValidationError as ValidationError3 } from "@cloudbitmaps/core";
|
|
743
|
+
var keysOf = () => (keys) => keys;
|
|
744
|
+
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
745
|
+
function optionsError(options, op, known, booleans = []) {
|
|
746
|
+
if (options === void 0 || options === null) return void 0;
|
|
747
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
748
|
+
return new ValidationError3(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
749
|
+
}
|
|
750
|
+
const bag = options;
|
|
751
|
+
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
752
|
+
if (unknown.length > 0) {
|
|
753
|
+
return new ValidationError3(
|
|
754
|
+
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
const audit = bag.audit;
|
|
758
|
+
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
759
|
+
return new ValidationError3(
|
|
760
|
+
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
for (const name of booleans) {
|
|
764
|
+
const v = bag[name];
|
|
765
|
+
if (v !== void 0 && typeof v !== "boolean") {
|
|
766
|
+
return new ValidationError3(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return void 0;
|
|
770
|
+
}
|
|
771
|
+
function picked(options, keys) {
|
|
772
|
+
const out = {};
|
|
773
|
+
if (options === void 0 || options === null) return out;
|
|
774
|
+
for (const key of keys) {
|
|
775
|
+
const value = options[key];
|
|
776
|
+
if (value !== void 0) out[key] = value;
|
|
777
|
+
}
|
|
778
|
+
return out;
|
|
779
|
+
}
|
|
780
|
+
function checkOptions(options, op, known, booleans) {
|
|
781
|
+
const err = optionsError(options, op, known, booleans);
|
|
782
|
+
if (err !== void 0) throw err;
|
|
783
|
+
}
|
|
610
784
|
|
|
611
785
|
// src/index.ts
|
|
612
786
|
import {
|
|
613
787
|
MemoryStorage,
|
|
614
788
|
LocalFsStorage,
|
|
615
|
-
CrbmStorageChunkSource as
|
|
789
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource3,
|
|
616
790
|
destroySegment,
|
|
617
791
|
eraseNamespace,
|
|
618
792
|
excludingReservedRows as excludingReservedRows2,
|
|
@@ -625,7 +799,7 @@ import {
|
|
|
625
799
|
NodeAead,
|
|
626
800
|
aadFor,
|
|
627
801
|
CloudRoaringError,
|
|
628
|
-
ValidationError as
|
|
802
|
+
ValidationError as ValidationError5,
|
|
629
803
|
WriteConflictError as WriteConflictError2,
|
|
630
804
|
IntegrityError as IntegrityError3,
|
|
631
805
|
NotFoundError as NotFoundError2,
|
|
@@ -654,7 +828,7 @@ import {
|
|
|
654
828
|
BufferReader,
|
|
655
829
|
CountingMetricsSink,
|
|
656
830
|
RecordingAuditSink,
|
|
657
|
-
DEFAULT_RETRY_POLICY as
|
|
831
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY3
|
|
658
832
|
} from "@cloudbitmaps/core";
|
|
659
833
|
var SystemRng = class {
|
|
660
834
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -670,14 +844,12 @@ function rowVersionOfLive(version) {
|
|
|
670
844
|
return cut < 0 ? version : version.slice(0, cut);
|
|
671
845
|
}
|
|
672
846
|
function validateConcurrency(concurrency) {
|
|
673
|
-
|
|
674
|
-
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
675
|
-
}
|
|
847
|
+
requirePositiveInt("concurrency", concurrency);
|
|
676
848
|
}
|
|
677
849
|
function requireScope(options, op) {
|
|
678
850
|
if (options.namespace !== void 0) {
|
|
679
851
|
if (typeof options.namespace !== "string") {
|
|
680
|
-
throw new
|
|
852
|
+
throw new ValidationError4(
|
|
681
853
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
682
854
|
);
|
|
683
855
|
}
|
|
@@ -685,12 +857,11 @@ function requireScope(options, op) {
|
|
|
685
857
|
}
|
|
686
858
|
refuseReservedNamespace(options.namespace);
|
|
687
859
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
688
|
-
throw new
|
|
860
|
+
throw new ValidationError4(
|
|
689
861
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
690
862
|
);
|
|
691
863
|
}
|
|
692
864
|
}
|
|
693
|
-
var keysOf = () => (keys) => keys;
|
|
694
865
|
var CALL_KEYS = {
|
|
695
866
|
range: keysOf()(["after", "through"]),
|
|
696
867
|
andNot: keysOf()([
|
|
@@ -750,7 +921,7 @@ var CALL_KEYS = {
|
|
|
750
921
|
]),
|
|
751
922
|
rollback: keysOf()([
|
|
752
923
|
"audit",
|
|
753
|
-
"
|
|
924
|
+
"allowUnknown"
|
|
754
925
|
]),
|
|
755
926
|
dropSegment: keysOf()([
|
|
756
927
|
"confirmSegment",
|
|
@@ -774,50 +945,12 @@ var CALL_KEYS = {
|
|
|
774
945
|
checkConsistency: keysOf()([
|
|
775
946
|
"namespace",
|
|
776
947
|
"concurrency",
|
|
777
|
-
"summaries"
|
|
948
|
+
"summaries",
|
|
949
|
+
"generations"
|
|
778
950
|
]),
|
|
779
951
|
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
780
952
|
};
|
|
781
|
-
var
|
|
782
|
-
function optionsError(options, op, known, booleans = []) {
|
|
783
|
-
if (options === void 0 || options === null) return void 0;
|
|
784
|
-
if (typeof options !== "object" || Array.isArray(options)) {
|
|
785
|
-
return new ValidationError2(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
786
|
-
}
|
|
787
|
-
const bag = options;
|
|
788
|
-
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
789
|
-
if (unknown.length > 0) {
|
|
790
|
-
return new ValidationError2(
|
|
791
|
-
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
792
|
-
);
|
|
793
|
-
}
|
|
794
|
-
const audit = bag.audit;
|
|
795
|
-
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
796
|
-
return new ValidationError2(
|
|
797
|
-
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
798
|
-
);
|
|
799
|
-
}
|
|
800
|
-
for (const name of booleans) {
|
|
801
|
-
const v = bag[name];
|
|
802
|
-
if (v !== void 0 && typeof v !== "boolean") {
|
|
803
|
-
return new ValidationError2(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
804
|
-
}
|
|
805
|
-
}
|
|
806
|
-
return void 0;
|
|
807
|
-
}
|
|
808
|
-
function picked(options, keys) {
|
|
809
|
-
const out = {};
|
|
810
|
-
if (options === void 0 || options === null) return out;
|
|
811
|
-
for (const key of keys) {
|
|
812
|
-
const value = options[key];
|
|
813
|
-
if (value !== void 0) out[key] = value;
|
|
814
|
-
}
|
|
815
|
-
return out;
|
|
816
|
-
}
|
|
817
|
-
function checkOptions(options, op, known, booleans) {
|
|
818
|
-
const err = optionsError(options, op, known, booleans);
|
|
819
|
-
if (err !== void 0) throw err;
|
|
820
|
-
}
|
|
953
|
+
var SEGMENT_BRAND = /* @__PURE__ */ Symbol.for("cloudbitmaps.roaring.segment/1");
|
|
821
954
|
var viewOf;
|
|
822
955
|
var MATERIALIZE_MANY_KEYS = [
|
|
823
956
|
"operands",
|
|
@@ -849,91 +982,6 @@ var DEFAULT_MAX_BUFFERED_BYTES = 256 * 1024 * 1024;
|
|
|
849
982
|
var DEFAULT_PUBLISH_CONCURRENCY = 8;
|
|
850
983
|
var DEFAULT_BATCH_CONCURRENCY = 1;
|
|
851
984
|
var PIN_PARALLELISM = 16;
|
|
852
|
-
function resolveStorageSource(options, clock) {
|
|
853
|
-
const storage = options.storage;
|
|
854
|
-
if (storage === null || typeof storage !== "object") {
|
|
855
|
-
throw new ValidationError2(
|
|
856
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
857
|
-
);
|
|
858
|
-
}
|
|
859
|
-
const asBackend = storage;
|
|
860
|
-
const isObject = (v) => v !== null && typeof v === "object";
|
|
861
|
-
const hasGetChunk = typeof storage.getChunk === "function";
|
|
862
|
-
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
863
|
-
const isBackend = isStorageBackend(storage);
|
|
864
|
-
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
865
|
-
throw new ValidationError2(
|
|
866
|
-
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
867
|
-
);
|
|
868
|
-
}
|
|
869
|
-
const hasStorageHalf = isObject(asBackend.storage);
|
|
870
|
-
const hasRegistryHalf = isObject(asBackend.registry);
|
|
871
|
-
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
872
|
-
throw new ValidationError2(
|
|
873
|
-
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
874
|
-
);
|
|
875
|
-
}
|
|
876
|
-
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
877
|
-
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
878
|
-
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
879
|
-
if (!storageOk || !registryOk) {
|
|
880
|
-
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
881
|
-
throw new ValidationError2(
|
|
882
|
-
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
883
|
-
);
|
|
884
|
-
}
|
|
885
|
-
throw new ValidationError2(
|
|
886
|
-
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
887
|
-
);
|
|
888
|
-
}
|
|
889
|
-
if (isBackend) {
|
|
890
|
-
const backend = storage;
|
|
891
|
-
return {
|
|
892
|
-
source: new CrbmStorageChunkSource(backend.storage, {
|
|
893
|
-
registry: backend.registry,
|
|
894
|
-
keystore: options.encryption?.keystore,
|
|
895
|
-
requireEncryption: options.encryption?.required,
|
|
896
|
-
clock,
|
|
897
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
898
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
899
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
900
|
-
}),
|
|
901
|
-
driver: backend.storage,
|
|
902
|
-
registry: backend.registry
|
|
903
|
-
};
|
|
904
|
-
}
|
|
905
|
-
if (hasGetChunk && hasPutImmutable) {
|
|
906
|
-
throw new ValidationError2(
|
|
907
|
-
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
908
|
-
);
|
|
909
|
-
}
|
|
910
|
-
if (!hasGetChunk && !hasPutImmutable) {
|
|
911
|
-
throw new ValidationError2(
|
|
912
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
913
|
-
);
|
|
914
|
-
}
|
|
915
|
-
if (hasGetChunk) {
|
|
916
|
-
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
917
|
-
throw new ValidationError2(
|
|
918
|
-
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
919
|
-
);
|
|
920
|
-
}
|
|
921
|
-
return { source: storage, driver: void 0, registry: void 0 };
|
|
922
|
-
}
|
|
923
|
-
const driver = storage;
|
|
924
|
-
return {
|
|
925
|
-
source: new CrbmStorageChunkSource(driver, {
|
|
926
|
-
keystore: options.encryption?.keystore,
|
|
927
|
-
requireEncryption: options.encryption?.required,
|
|
928
|
-
clock,
|
|
929
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
930
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
931
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
932
|
-
}),
|
|
933
|
-
driver,
|
|
934
|
-
registry: void 0
|
|
935
|
-
};
|
|
936
|
-
}
|
|
937
985
|
var NOT_A_GROUP = {
|
|
938
986
|
Null: "null",
|
|
939
987
|
Array: "an array",
|
|
@@ -976,7 +1024,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
976
1024
|
*/
|
|
977
1025
|
static rejectUnknownOptions(options) {
|
|
978
1026
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
979
|
-
throw new
|
|
1027
|
+
throw new ValidationError4(
|
|
980
1028
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
981
1029
|
);
|
|
982
1030
|
}
|
|
@@ -994,7 +1042,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
994
1042
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
995
1043
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
996
1044
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
997
|
-
throw new
|
|
1045
|
+
throw new ValidationError4(
|
|
998
1046
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
999
1047
|
);
|
|
1000
1048
|
}
|
|
@@ -1004,7 +1052,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1004
1052
|
if (unknown.length === 0) return;
|
|
1005
1053
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1006
1054
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1007
|
-
throw new
|
|
1055
|
+
throw new ValidationError4(
|
|
1008
1056
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1009
1057
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1010
1058
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1024,7 +1072,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1024
1072
|
if (seam === void 0 || seam === null) continue;
|
|
1025
1073
|
const has = (m) => typeof seam[m] === "function";
|
|
1026
1074
|
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1027
|
-
throw new
|
|
1075
|
+
throw new ValidationError4(
|
|
1028
1076
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1029
1077
|
);
|
|
1030
1078
|
}
|
|
@@ -1035,50 +1083,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1035
1083
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1036
1084
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1037
1085
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1038
|
-
const metrics = safeMetrics(options.metrics ??
|
|
1086
|
+
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS2);
|
|
1039
1087
|
const cache = new BoundedLru({
|
|
1040
1088
|
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
1041
1089
|
ttlMs: options.cache?.ttlMs,
|
|
1042
1090
|
clock
|
|
1043
1091
|
});
|
|
1044
|
-
const
|
|
1045
|
-
|
|
1046
|
-
options.storage.attachMetrics?.(metrics);
|
|
1047
|
-
}
|
|
1048
|
-
let storage = resolved.source;
|
|
1049
|
-
if (options.retry !== false) {
|
|
1050
|
-
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
1051
|
-
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
1052
|
-
throw new ValidationError2("retry.onRetry must be a function");
|
|
1053
|
-
}
|
|
1054
|
-
const policy = {
|
|
1055
|
-
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
1056
|
-
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
1057
|
-
maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
1058
|
-
backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
1059
|
-
jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
1060
|
-
};
|
|
1061
|
-
const retryOpts = {
|
|
1062
|
-
clock,
|
|
1063
|
-
rng,
|
|
1064
|
-
policy,
|
|
1065
|
-
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
1066
|
-
onRetry: (info) => {
|
|
1067
|
-
metrics.onEvent({
|
|
1068
|
-
kind: "retry",
|
|
1069
|
-
reason: "transient",
|
|
1070
|
-
attempt: info.attempt,
|
|
1071
|
-
delayMs: info.delayMs
|
|
1072
|
-
});
|
|
1073
|
-
userOnRetry?.(info);
|
|
1074
|
-
}
|
|
1075
|
-
};
|
|
1076
|
-
storage = new RetryingStorageChunkSource(storage, retryOpts);
|
|
1077
|
-
this.retryOptions = retryOpts;
|
|
1078
|
-
}
|
|
1092
|
+
const wired = wireStore(options, clock, rng, metrics);
|
|
1093
|
+
this.retryOptions = wired.retryOptions;
|
|
1079
1094
|
this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
|
|
1080
1095
|
const deps = {
|
|
1081
|
-
storage,
|
|
1096
|
+
storage: wired.storage,
|
|
1082
1097
|
cache,
|
|
1083
1098
|
codec: roaringCodec,
|
|
1084
1099
|
// the facade injects the flagship codec; core stays codec-agnostic
|
|
@@ -1088,12 +1103,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1088
1103
|
};
|
|
1089
1104
|
this.engine = new SegmentEngine(deps);
|
|
1090
1105
|
this.cache = cache;
|
|
1091
|
-
this.crbmSource =
|
|
1106
|
+
this.crbmSource = wired.source instanceof CrbmStorageChunkSource2 ? wired.source : void 0;
|
|
1092
1107
|
this.clock = clock;
|
|
1093
1108
|
this.rng = rng;
|
|
1094
1109
|
this.metrics = metrics;
|
|
1095
|
-
this.storageDriver =
|
|
1096
|
-
this.registry =
|
|
1110
|
+
this.storageDriver = wired.driver;
|
|
1111
|
+
this.registry = wired.registry;
|
|
1097
1112
|
this.keystore = options.encryption?.keystore;
|
|
1098
1113
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1099
1114
|
}
|
|
@@ -1137,11 +1152,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1137
1152
|
segment(name, options) {
|
|
1138
1153
|
if (options !== void 0 && options !== null) {
|
|
1139
1154
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1140
|
-
throw new
|
|
1155
|
+
throw new ValidationError4("segment: options must be an object such as { namespace }");
|
|
1141
1156
|
}
|
|
1142
1157
|
const proto = Object.getPrototypeOf(options);
|
|
1143
1158
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1144
|
-
throw new
|
|
1159
|
+
throw new ValidationError4(
|
|
1145
1160
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1146
1161
|
);
|
|
1147
1162
|
}
|
|
@@ -1149,7 +1164,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1149
1164
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1150
1165
|
);
|
|
1151
1166
|
if (unknown.length > 0) {
|
|
1152
|
-
throw new
|
|
1167
|
+
throw new ValidationError4(
|
|
1153
1168
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1154
1169
|
);
|
|
1155
1170
|
}
|
|
@@ -1184,7 +1199,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1184
1199
|
* row it is kept (hard invariant 1: deleting it after a purge-and-recreate would put a live row over a missing
|
|
1185
1200
|
* generation).
|
|
1186
1201
|
*
|
|
1187
|
-
* Where the row is gone, or keyed over a cleartext write, the object
|
|
1202
|
+
* Where the row is gone, or keyed over a cleartext write, the object's footer must match what the load wrote, and
|
|
1188
1203
|
* the delete passes the version that read reported, so on a storage driver that reports `conditionalDelete` an
|
|
1189
1204
|
* object put under the number since is kept. Where the row is unchanged or `destroyed`, the check narrows the
|
|
1190
1205
|
* window rather than closing it: the row read and the delete are two round trips, and the delete is by number,
|
|
@@ -1239,9 +1254,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1239
1254
|
}
|
|
1240
1255
|
const reason = result.reason;
|
|
1241
1256
|
if (reason === "superseded") {
|
|
1242
|
-
const wrote = result.size > 0 ? `generation ${result.generation} was written and
|
|
1257
|
+
const wrote = result.size > 0 ? `generation ${result.generation} was written and may not be current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
1243
1258
|
throw new WriteConflictError(
|
|
1244
|
-
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result
|
|
1259
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result may not be the destination's current generation: ${wrote}. The pointer may have moved, the row may have been created or rewritten (a retention policy does either), dropped or purged. Re-read the destination and re-run.`
|
|
1245
1260
|
);
|
|
1246
1261
|
}
|
|
1247
1262
|
return {
|
|
@@ -1258,7 +1273,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1258
1273
|
/** The one body of the forms above. */
|
|
1259
1274
|
async materializeMany(options) {
|
|
1260
1275
|
if (typeof options !== "object" || options === null) {
|
|
1261
|
-
throw new
|
|
1276
|
+
throw new ValidationError4(
|
|
1262
1277
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1263
1278
|
);
|
|
1264
1279
|
}
|
|
@@ -1267,13 +1282,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1267
1282
|
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1268
1283
|
);
|
|
1269
1284
|
if (unknown.length > 0) {
|
|
1270
|
-
throw new
|
|
1285
|
+
throw new ValidationError4(
|
|
1271
1286
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1272
1287
|
);
|
|
1273
1288
|
}
|
|
1274
1289
|
const dryRunIn = options.dryRun;
|
|
1275
1290
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1276
|
-
throw new
|
|
1291
|
+
throw new ValidationError4("materializeMany: dryRun must be a boolean");
|
|
1277
1292
|
}
|
|
1278
1293
|
const dryRun = dryRunIn === true;
|
|
1279
1294
|
const operandsIn = options.operands;
|
|
@@ -1285,7 +1300,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1285
1300
|
const feedIn = options.feed;
|
|
1286
1301
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1287
1302
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1288
|
-
throw new
|
|
1303
|
+
throw new ValidationError4(
|
|
1289
1304
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1290
1305
|
);
|
|
1291
1306
|
}
|
|
@@ -1295,22 +1310,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1295
1310
|
const budgetOption = options.budget;
|
|
1296
1311
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1297
1312
|
if (typeof pinOption !== "boolean")
|
|
1298
|
-
throw new
|
|
1313
|
+
throw new ValidationError4("materializeMany: pin must be a boolean");
|
|
1299
1314
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1300
|
-
throw new
|
|
1315
|
+
throw new ValidationError4("materializeMany: allowAbsentOperands must be a boolean");
|
|
1301
1316
|
}
|
|
1302
1317
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1303
|
-
throw new
|
|
1318
|
+
throw new ValidationError4("materializeMany: operands must be an object of name to segment");
|
|
1304
1319
|
}
|
|
1305
1320
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1306
|
-
throw new
|
|
1321
|
+
throw new ValidationError4("materializeMany: outputs must be a non-empty array");
|
|
1307
1322
|
}
|
|
1308
1323
|
const names = Object.keys(operandsIn);
|
|
1309
1324
|
const handles = /* @__PURE__ */ new Map();
|
|
1310
1325
|
for (const name of names) {
|
|
1311
1326
|
const handle = operandsIn[name];
|
|
1312
1327
|
if (!(handle instanceof Segment)) {
|
|
1313
|
-
throw new
|
|
1328
|
+
throw new ValidationError4(
|
|
1314
1329
|
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1315
1330
|
);
|
|
1316
1331
|
}
|
|
@@ -1318,14 +1333,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1318
1333
|
}
|
|
1319
1334
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1320
1335
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1321
|
-
throw new
|
|
1336
|
+
throw new ValidationError4(
|
|
1322
1337
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1323
1338
|
);
|
|
1324
1339
|
}
|
|
1325
1340
|
if (mayBeEmptyIn.length > 0) {
|
|
1326
1341
|
const first = mayBeEmptyIn[0];
|
|
1327
1342
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1328
|
-
throw new
|
|
1343
|
+
throw new ValidationError4(
|
|
1329
1344
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1330
1345
|
);
|
|
1331
1346
|
}
|
|
@@ -1335,28 +1350,28 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1335
1350
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1336
1351
|
outputsIn.forEach((raw, i) => {
|
|
1337
1352
|
if (typeof raw !== "object" || raw === null) {
|
|
1338
|
-
throw new
|
|
1353
|
+
throw new ValidationError4(`outputs[${i}] must be an object`);
|
|
1339
1354
|
}
|
|
1340
1355
|
const bad = Object.keys(raw).filter(
|
|
1341
1356
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1342
1357
|
);
|
|
1343
1358
|
if (bad.length > 0) {
|
|
1344
|
-
throw new
|
|
1359
|
+
throw new ValidationError4(
|
|
1345
1360
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1346
1361
|
);
|
|
1347
1362
|
}
|
|
1348
1363
|
const spec = raw;
|
|
1349
1364
|
const dest = spec.dest;
|
|
1350
1365
|
if (!(dest instanceof Segment)) {
|
|
1351
|
-
throw new
|
|
1366
|
+
throw new ValidationError4(`outputs[${i}].dest must be a segment from store.segment()`);
|
|
1352
1367
|
}
|
|
1353
1368
|
if (operandKeys.has(dest.key())) {
|
|
1354
|
-
throw new
|
|
1369
|
+
throw new ValidationError4(
|
|
1355
1370
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1356
1371
|
);
|
|
1357
1372
|
}
|
|
1358
1373
|
if (destKeys.has(dest.key())) {
|
|
1359
|
-
throw new
|
|
1374
|
+
throw new ValidationError4(
|
|
1360
1375
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1361
1376
|
);
|
|
1362
1377
|
}
|
|
@@ -1376,7 +1391,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1376
1391
|
...pinnedAt === void 0 ? {} : {
|
|
1377
1392
|
pinnedGeneration: pinnedAt.generation,
|
|
1378
1393
|
pinnedVersion: pinnedAt.version,
|
|
1379
|
-
pinnedFingerprint: pinnedAt.fingerprint
|
|
1394
|
+
pinnedFingerprint: pinnedAt.fingerprint
|
|
1380
1395
|
},
|
|
1381
1396
|
current: (read) => this.currentRowOf(seg.ref, read?.fingerprint === true)
|
|
1382
1397
|
};
|
|
@@ -1429,13 +1444,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1429
1444
|
held.set(key, { at, handle });
|
|
1430
1445
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1431
1446
|
if (!samePin(prior.at, at)) {
|
|
1432
|
-
throw new
|
|
1447
|
+
throw new ValidationError4(
|
|
1433
1448
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1434
1449
|
);
|
|
1435
1450
|
}
|
|
1436
1451
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1437
1452
|
if (!pinOption) {
|
|
1438
|
-
throw new
|
|
1453
|
+
throw new ValidationError4(
|
|
1439
1454
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1440
1455
|
);
|
|
1441
1456
|
}
|
|
@@ -1449,7 +1464,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1449
1464
|
else if (pinOption) toPin.push([key, handle]);
|
|
1450
1465
|
else pins.set(key, void 0);
|
|
1451
1466
|
}
|
|
1452
|
-
const metricsOn = this.metrics !==
|
|
1467
|
+
const metricsOn = this.metrics !== NOOP_METRICS2;
|
|
1453
1468
|
const work = async () => {
|
|
1454
1469
|
const taken = await mapWithConcurrency(
|
|
1455
1470
|
toPin,
|
|
@@ -1459,7 +1474,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1459
1474
|
for (const [key, at] of taken) pins.set(key, at);
|
|
1460
1475
|
const live = new Map([...pins].filter((e) => e[1] !== void 0));
|
|
1461
1476
|
const pinnedSource = new PinnedStorageChunkSource(crbm, live);
|
|
1462
|
-
const source = this.retryOptions === void 0 ? pinnedSource : new
|
|
1477
|
+
const source = this.retryOptions === void 0 ? pinnedSource : new RetryingStorageChunkSource2(pinnedSource, this.retryOptions);
|
|
1463
1478
|
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1464
1479
|
return runCombineMany(final, {
|
|
1465
1480
|
source,
|
|
@@ -1523,7 +1538,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1523
1538
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1524
1539
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1525
1540
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1526
|
-
throw new
|
|
1541
|
+
throw new ValidationError4(
|
|
1527
1542
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1528
1543
|
);
|
|
1529
1544
|
}
|
|
@@ -1761,6 +1776,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1761
1776
|
* What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
|
|
1762
1777
|
* the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
|
|
1763
1778
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1779
|
+
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1780
|
+
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1764
1781
|
*
|
|
1765
1782
|
* Needs a backend.
|
|
1766
1783
|
*/
|
|
@@ -1845,36 +1862,58 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1845
1862
|
* (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
|
|
1846
1863
|
* one cannot.
|
|
1847
1864
|
*
|
|
1848
|
-
* It refuses rather than guesses
|
|
1849
|
-
*
|
|
1850
|
-
*
|
|
1851
|
-
*
|
|
1852
|
-
*
|
|
1865
|
+
* It refuses rather than guesses. The row records, in `knownGens`, the generations its loads and rollbacks made
|
|
1866
|
+
* current, each with the footer checksum of its object, and a rollback moves only onto one of those: a generation the
|
|
1867
|
+
* row does not know (a load that wrote its object and died before the publish, a refused load's object, an object
|
|
1868
|
+
* sealed under a key no row holds, a load published after the point a registry was restored to) throws
|
|
1869
|
+
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1870
|
+
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1871
|
+
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1872
|
+
* crypto-shredded segment throws {@link ValidationError} because every generation of it is unreadable, and a target
|
|
1873
|
+
* that is cleartext under an encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1874
|
+
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1875
|
+
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1876
|
+
* gone it throws `NotFoundError` naming what is.
|
|
1853
1877
|
*
|
|
1854
1878
|
* The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
|
|
1855
1879
|
* metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
|
|
1856
1880
|
* what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
|
|
1857
|
-
* key
|
|
1881
|
+
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1882
|
+
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1883
|
+
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1884
|
+
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one that landed, the row being the
|
|
1885
|
+
* one the swap leaves, is checked as any is, and otherwise its `TransientError` is thrown. After the swap the target's
|
|
1886
|
+
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1858
1887
|
*
|
|
1859
1888
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
1860
|
-
* and are then *above* `currentGen`, where collection never looks
|
|
1861
|
-
*
|
|
1862
|
-
* below its pointer
|
|
1889
|
+
* and are then *above* `currentGen`, where collection never looks, and still known to the row, so rolling forward onto
|
|
1890
|
+
* one needs no flag. They remain until loads pass them (the first load whose
|
|
1891
|
+
* number one of them holds numbers above them all, and collection then keeps `keep` of what is below its pointer, the
|
|
1892
|
+
* generation that load superseded first), {@link CloudRoaring.dropSegment}
|
|
1863
1893
|
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and, when the
|
|
1864
1894
|
* current generation does not hold the id, only those holding it and any no read of the segment can open, whatever
|
|
1865
1895
|
* the id. An operator who has just undone a bad load should not have the
|
|
1866
1896
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
1867
1897
|
* erasure undoable.
|
|
1868
1898
|
*
|
|
1869
|
-
*
|
|
1870
|
-
*
|
|
1871
|
-
*
|
|
1899
|
+
* `{ allowUnknown: true }` moves onto a present object the row does not vouch for, or one whose checksum differs from
|
|
1900
|
+
* the recorded one, as long as it is the segment's kind of object, opens, and names the generation, and records it as
|
|
1901
|
+
* known (over a usable record; a row with none stays without one). On an encrypted segment, outside the two cases the pointer alone decides (below), it also needs the key: with no keystore, or one that holds none of the key's KEKs or
|
|
1902
|
+
* one that does not unwrap it, it is refused with `KeyUnavailableError` before the target is read, since an object
|
|
1903
|
+
* sealed under a key no row holds passes every check that needs no key, and a key service that did not answer in
|
|
1904
|
+
* time is thrown as `TransientError`, to retry. Every move the row did not vouch for is audited with `unknown: true`.
|
|
1905
|
+
* Two cases are decided by the pointer alone: a target below it needs no flag, no key and no checksum check, and the
|
|
1906
|
+
* move is audited the same way. They are a row with no usable `knownGens` (a registry that drops the field, or a pointer
|
|
1907
|
+
* moved by hand), and a target numbered below the bound of a full record (128 entries or more, leased ones included),
|
|
1908
|
+
* which is its least recently current entry that no lease names (an `*Into` destination keeps every generation in the
|
|
1909
|
+
* bucket and names only the newest 128). A generation a publish with a smaller `keep` no longer names but that stays in
|
|
1910
|
+
* the bucket (a lease spared it, or its delete failed) needs `allowUnknown` until a later listing collects it.
|
|
1872
1911
|
*
|
|
1873
1912
|
* Needs a backend.
|
|
1874
1913
|
*/
|
|
1875
1914
|
async rollback(ref, toGeneration, options = {}) {
|
|
1876
1915
|
validateSegmentRef(ref);
|
|
1877
|
-
checkOptions(options, "rollback", CALL_KEYS.rollback, ["
|
|
1916
|
+
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1878
1917
|
options ??= {};
|
|
1879
1918
|
const deps = this.lifecycleDeps("rollback");
|
|
1880
1919
|
try {
|
|
@@ -1931,12 +1970,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1931
1970
|
* published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
|
|
1932
1971
|
*
|
|
1933
1972
|
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
1934
|
-
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only
|
|
1935
|
-
*
|
|
1936
|
-
* encrypted segment's bucket. Under a row that is gone or keyed,
|
|
1937
|
-
* load
|
|
1938
|
-
*
|
|
1939
|
-
*
|
|
1973
|
+
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only in
|
|
1974
|
+
* its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1975
|
+
* encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
|
|
1976
|
+
* load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
|
|
1977
|
+
* writer's object, and a footer names bytes, not the load that wrote them, so it leaves the orphan rather than risk
|
|
1978
|
+
* deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
|
|
1979
|
+
* generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
|
|
1980
|
+
* with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
|
|
1981
|
+
* after carry it until the window pushes it out. Until it goes, a rollback onto it is refused unless `allowUnknown` is passed, since the list never names it (a list seeded
|
|
1982
|
+
* over a row with none can); with the override it publishes what no load published.
|
|
1940
1983
|
*
|
|
1941
1984
|
* **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
|
|
1942
1985
|
* that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
|
|
@@ -2027,7 +2070,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2027
2070
|
async dropSegment(ref, options) {
|
|
2028
2071
|
validateSegmentRef(ref);
|
|
2029
2072
|
if (options === void 0 || options === null) {
|
|
2030
|
-
throw new
|
|
2073
|
+
throw new ValidationError4("dropSegment: options { confirmSegment } are required");
|
|
2031
2074
|
}
|
|
2032
2075
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2033
2076
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2195,7 +2238,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2195
2238
|
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
2196
2239
|
*
|
|
2197
2240
|
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
2198
|
-
* object back, the pin reads it again.
|
|
2241
|
+
* object back, the pin reads it again. A read of the segment already running on this store, a pin's and a combine's
|
|
2242
|
+
* with a pinned operand included, caches nothing more of what it began reading before the call, and a pinned one
|
|
2243
|
+
* opens its object again before its next chunk, failing with `NotFoundError` if the object is gone or replaced, or
|
|
2244
|
+
* its row is gone or destroyed.
|
|
2199
2245
|
*
|
|
2200
2246
|
* Synchronous, best-effort, and safe to call for a segment this store has never read.
|
|
2201
2247
|
*
|
|
@@ -2215,7 +2261,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2215
2261
|
*
|
|
2216
2262
|
* The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
|
|
2217
2263
|
* pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
|
|
2218
|
-
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
2264
|
+
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}). It
|
|
2265
|
+
* shares the store engine's invalidations too, so the store's writes reach a pinned read already running.
|
|
2219
2266
|
*/
|
|
2220
2267
|
async pinSegment(ref, leaseUntil, named) {
|
|
2221
2268
|
const crbm = this.crbmSource;
|
|
@@ -2294,7 +2341,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2294
2341
|
* An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
|
|
2295
2342
|
* retries, so a transient fault on a pinned read is retried as it would be on a live one.
|
|
2296
2343
|
*
|
|
2297
|
-
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment
|
|
2344
|
+
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment, and the
|
|
2345
|
+
* store engine's invalidations, so an invalidation of a pinned segment reaches a read of the pin already running.
|
|
2298
2346
|
*/
|
|
2299
2347
|
engineWithPins(pins) {
|
|
2300
2348
|
const crbm = this.crbmSource;
|
|
@@ -2305,12 +2353,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2305
2353
|
}
|
|
2306
2354
|
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
2307
2355
|
return new SegmentEngine({
|
|
2308
|
-
storage: this.retryOptions === void 0 ? pinned : new
|
|
2356
|
+
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource2(pinned, this.retryOptions),
|
|
2309
2357
|
cache: this.cache,
|
|
2310
2358
|
codec: roaringCodec,
|
|
2311
2359
|
clock: this.clock,
|
|
2312
2360
|
metrics: this.metrics,
|
|
2313
|
-
budget: this.budget
|
|
2361
|
+
budget: this.budget,
|
|
2362
|
+
sharesInvalidationsWith: this.engine
|
|
2314
2363
|
});
|
|
2315
2364
|
}
|
|
2316
2365
|
/**
|
|
@@ -2336,7 +2385,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2336
2385
|
if (held.has(key)) {
|
|
2337
2386
|
const before = held.get(key);
|
|
2338
2387
|
if (!samePin(before, at)) {
|
|
2339
|
-
throw new
|
|
2388
|
+
throw new ValidationError4(
|
|
2340
2389
|
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`intersectInto(dest, [])\``
|
|
2341
2390
|
);
|
|
2342
2391
|
}
|
|
@@ -2350,7 +2399,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2350
2399
|
/** `op` under the store's retries, as the store's own reads run; bare when the store has retries off. */
|
|
2351
2400
|
withRetries(op) {
|
|
2352
2401
|
const r = this.retryOptions;
|
|
2353
|
-
return r === void 0 ? op() : withRetry(op, r.policy ??
|
|
2402
|
+
return r === void 0 ? op() : withRetry(op, r.policy ?? DEFAULT_RETRY_POLICY2, r);
|
|
2354
2403
|
}
|
|
2355
2404
|
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
2356
2405
|
requireRegistry(op) {
|
|
@@ -2379,13 +2428,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2379
2428
|
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
2380
2429
|
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
2381
2430
|
*
|
|
2431
|
+
* With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
|
|
2432
|
+
* included, against its object: one footer read each (104 bytes, no key, not retried), reporting
|
|
2433
|
+
* `known-generation-mismatch` with the `generation` where the object under that number is not the one the row
|
|
2434
|
+
* recorded. A named generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot
|
|
2435
|
+
* be used is counted in `generationsUnchecked`, and none of its objects is read.
|
|
2436
|
+
*
|
|
2382
2437
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
2383
2438
|
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
2384
2439
|
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
2385
2440
|
* `storage` and `registry` with a higher `maxScanSegments`.
|
|
2386
2441
|
*/
|
|
2387
2442
|
async checkConsistency(options = {}) {
|
|
2388
|
-
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2443
|
+
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2444
|
+
"summaries",
|
|
2445
|
+
"generations"
|
|
2446
|
+
]);
|
|
2389
2447
|
options ??= {};
|
|
2390
2448
|
const deps = this.lifecycleDeps("checkConsistency");
|
|
2391
2449
|
return runConsistencyCheck(
|
|
@@ -2413,13 +2471,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2413
2471
|
}
|
|
2414
2472
|
};
|
|
2415
2473
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
2416
|
-
var
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2474
|
+
var refusing = (err) => (
|
|
2475
|
+
// eslint-disable-next-line require-yield
|
|
2476
|
+
async function* () {
|
|
2477
|
+
throw err;
|
|
2478
|
+
}
|
|
2479
|
+
);
|
|
2480
|
+
var failing = (err) => withBatches(refusing(err)(), refusing(err));
|
|
2423
2481
|
var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
|
|
2424
2482
|
var twoPins = (a, b) => {
|
|
2425
2483
|
const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
|
|
@@ -2437,9 +2495,12 @@ var readOptions = (options) => ({
|
|
|
2437
2495
|
});
|
|
2438
2496
|
var NO_SEGMENTS = [];
|
|
2439
2497
|
function segmentList(list, what) {
|
|
2440
|
-
if (!Array.isArray(list)) throw new
|
|
2498
|
+
if (!Array.isArray(list)) throw new ValidationError4(`${what} must be an array of segments`);
|
|
2441
2499
|
return list;
|
|
2442
2500
|
}
|
|
2501
|
+
function operandsLabel(verb) {
|
|
2502
|
+
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2503
|
+
}
|
|
2443
2504
|
var AS_IDS = {
|
|
2444
2505
|
failing,
|
|
2445
2506
|
guard: guardIds,
|
|
@@ -2454,10 +2515,8 @@ var CombineChunks = class {
|
|
|
2454
2515
|
chunks;
|
|
2455
2516
|
};
|
|
2456
2517
|
var AS_CHUNKS = {
|
|
2457
|
-
guard:
|
|
2458
|
-
failing: (err) => (
|
|
2459
|
-
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
|
|
2460
|
-
}),
|
|
2518
|
+
guard: guardItems,
|
|
2519
|
+
failing: (err) => refusing(err)(),
|
|
2461
2520
|
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2462
2521
|
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2463
2522
|
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
@@ -2495,6 +2554,7 @@ var Segment = class _Segment {
|
|
|
2495
2554
|
lease;
|
|
2496
2555
|
leaseHold;
|
|
2497
2556
|
static {
|
|
2557
|
+
Object.defineProperty(_Segment.prototype, SEGMENT_BRAND, { value: true });
|
|
2498
2558
|
makeSegment = (parts) => {
|
|
2499
2559
|
minting = true;
|
|
2500
2560
|
try {
|
|
@@ -2515,7 +2575,7 @@ var Segment = class _Segment {
|
|
|
2515
2575
|
*/
|
|
2516
2576
|
constructor(parts) {
|
|
2517
2577
|
if (!minting) {
|
|
2518
|
-
throw new
|
|
2578
|
+
throw new ValidationError4(
|
|
2519
2579
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2520
2580
|
);
|
|
2521
2581
|
}
|
|
@@ -2531,7 +2591,7 @@ var Segment = class _Segment {
|
|
|
2531
2591
|
if (parts.lease !== void 0) {
|
|
2532
2592
|
this.lease = Object.freeze({ holder: parts.lease.holder, until: parts.lease.until });
|
|
2533
2593
|
}
|
|
2534
|
-
this.metricsOn = parts.metrics !==
|
|
2594
|
+
this.metricsOn = parts.metrics !== NOOP_METRICS2;
|
|
2535
2595
|
}
|
|
2536
2596
|
/**
|
|
2537
2597
|
* **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
|
|
@@ -2586,11 +2646,16 @@ var Segment = class _Segment {
|
|
|
2586
2646
|
* - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
|
|
2587
2647
|
*
|
|
2588
2648
|
* An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
|
|
2589
|
-
* destroyed.
|
|
2649
|
+
* destroyed. A read of the pin already running when the invalidation comes, a combine on this store with the pin as
|
|
2650
|
+
* an operand included, looks the pin up again before it hands out its next chunk: it carries on with what it holds
|
|
2651
|
+
* where the object is still there, and fails with `NotFoundError` where it is gone. It caches nothing more of what it
|
|
2652
|
+
* began reading before the invalidation, so the pin's next read opens the object again too.
|
|
2590
2653
|
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
2591
2654
|
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
2592
2655
|
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
2593
|
-
* {@link CloudRoaring.invalidate} is called on this store.
|
|
2656
|
+
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store that takes the pin as an
|
|
2657
|
+
* operand reads it through that store's reader and chunk cache, which only that store's own writes and its
|
|
2658
|
+
* `invalidate` reach. Where the
|
|
2594
2659
|
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
2595
2660
|
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
2596
2661
|
*
|
|
@@ -2613,7 +2678,7 @@ var Segment = class _Segment {
|
|
|
2613
2678
|
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
2614
2679
|
*/
|
|
2615
2680
|
async pin(options) {
|
|
2616
|
-
this.
|
|
2681
|
+
this.assertLease();
|
|
2617
2682
|
const leaseUntil = this.leaseUntilOf("pin", options);
|
|
2618
2683
|
return this.pinned(this.ref, leaseUntil);
|
|
2619
2684
|
}
|
|
@@ -2625,11 +2690,11 @@ var Segment = class _Segment {
|
|
|
2625
2690
|
leaseUntilOf(op, options) {
|
|
2626
2691
|
if (options === void 0 || options === null) return void 0;
|
|
2627
2692
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2628
|
-
throw new
|
|
2693
|
+
throw new ValidationError4(`${op}: options must be an object such as { leaseUntil }`);
|
|
2629
2694
|
}
|
|
2630
2695
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2631
2696
|
if (unknown.length > 0) {
|
|
2632
|
-
throw new
|
|
2697
|
+
throw new ValidationError4(
|
|
2633
2698
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2634
2699
|
);
|
|
2635
2700
|
}
|
|
@@ -2637,17 +2702,17 @@ var Segment = class _Segment {
|
|
|
2637
2702
|
if (until === void 0) return void 0;
|
|
2638
2703
|
const now = this.clock.now();
|
|
2639
2704
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2640
|
-
throw new
|
|
2705
|
+
throw new ValidationError4(
|
|
2641
2706
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2642
2707
|
);
|
|
2643
2708
|
}
|
|
2644
2709
|
if (until <= now) {
|
|
2645
|
-
throw new
|
|
2710
|
+
throw new ValidationError4(
|
|
2646
2711
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2647
2712
|
);
|
|
2648
2713
|
}
|
|
2649
2714
|
if (until > now + MAX_LEASE_MS) {
|
|
2650
|
-
throw new
|
|
2715
|
+
throw new ValidationError4(
|
|
2651
2716
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2652
2717
|
);
|
|
2653
2718
|
}
|
|
@@ -2703,10 +2768,12 @@ var Segment = class _Segment {
|
|
|
2703
2768
|
*/
|
|
2704
2769
|
assertSegments(handles) {
|
|
2705
2770
|
for (const h of handles) {
|
|
2706
|
-
if (h instanceof _Segment || typeof h === "object" && h !== null &&
|
|
2771
|
+
if (h instanceof _Segment || typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
2707
2772
|
continue;
|
|
2708
2773
|
}
|
|
2709
|
-
throw new
|
|
2774
|
+
throw new ValidationError4(
|
|
2775
|
+
"an operand must be a segment from `store.segment()` of a compatible copy of @cloudbitmaps/roaring; install one version of it"
|
|
2776
|
+
);
|
|
2710
2777
|
}
|
|
2711
2778
|
}
|
|
2712
2779
|
/**
|
|
@@ -2719,6 +2786,11 @@ var Segment = class _Segment {
|
|
|
2719
2786
|
if (err !== void 0) throw err;
|
|
2720
2787
|
}
|
|
2721
2788
|
}
|
|
2789
|
+
/** Throw this handle's lease error, if it has one. The calls that return a value instead of throwing use {@link leaseError}. */
|
|
2790
|
+
assertLease() {
|
|
2791
|
+
const err = this.leaseError();
|
|
2792
|
+
if (err !== void 0) throw err;
|
|
2793
|
+
}
|
|
2722
2794
|
/**
|
|
2723
2795
|
* Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
|
|
2724
2796
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
@@ -2742,16 +2814,16 @@ var Segment = class _Segment {
|
|
|
2742
2814
|
* is in `options`, the second.
|
|
2743
2815
|
*/
|
|
2744
2816
|
async pinAt(at, options) {
|
|
2745
|
-
this.
|
|
2817
|
+
this.assertLease();
|
|
2746
2818
|
const { generation, fingerprint } = at ?? {};
|
|
2747
2819
|
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => k !== "generation" && k !== "fingerprint") : [];
|
|
2748
2820
|
if (unknown.length > 0) {
|
|
2749
|
-
throw new
|
|
2821
|
+
throw new ValidationError4(
|
|
2750
2822
|
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { generation, fingerprint } only`
|
|
2751
2823
|
);
|
|
2752
2824
|
}
|
|
2753
2825
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2754
|
-
throw new
|
|
2826
|
+
throw new ValidationError4(
|
|
2755
2827
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2756
2828
|
);
|
|
2757
2829
|
}
|
|
@@ -2785,8 +2857,9 @@ var Segment = class _Segment {
|
|
|
2785
2857
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
2786
2858
|
}
|
|
2787
2859
|
/**
|
|
2788
|
-
* Time an op with the injected clock and emit an `op` metric on completion (success or throw).
|
|
2789
|
-
*
|
|
2860
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). With no sink wired it
|
|
2861
|
+
* reads no clock and emits nothing, but a call still passes through this async frame; `has()` and `count()` return the
|
|
2862
|
+
* engine's promise directly instead.
|
|
2790
2863
|
*/
|
|
2791
2864
|
async timed(name, fn) {
|
|
2792
2865
|
if (!this.metricsOn) return fn();
|
|
@@ -2797,10 +2870,15 @@ var Segment = class _Segment {
|
|
|
2797
2870
|
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
2798
2871
|
}
|
|
2799
2872
|
}
|
|
2800
|
-
/**
|
|
2873
|
+
/**
|
|
2874
|
+
* Membership: one chunk. A cold resolution of the segment is one registry read and a warm one none; then a cached
|
|
2875
|
+
* chunk answers with no request, and an uncached one is one range read, after a tail read when no reader of the
|
|
2876
|
+
* generation is open, unless that tail carried the chunk. Throws {@link ValidationError} on a bad id.
|
|
2877
|
+
*/
|
|
2801
2878
|
has(id) {
|
|
2802
2879
|
const lease = this.leaseError();
|
|
2803
2880
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2881
|
+
if (!this.metricsOn) return this.engine.has(this.ref, id);
|
|
2804
2882
|
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
2805
2883
|
}
|
|
2806
2884
|
/**
|
|
@@ -2824,6 +2902,7 @@ var Segment = class _Segment {
|
|
|
2824
2902
|
count() {
|
|
2825
2903
|
const lease = this.leaseError();
|
|
2826
2904
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2905
|
+
if (!this.metricsOn) return this.engine.count(this.ref);
|
|
2827
2906
|
return this.timed("count", () => this.engine.count(this.ref));
|
|
2828
2907
|
}
|
|
2829
2908
|
/**
|
|
@@ -2854,7 +2933,7 @@ var Segment = class _Segment {
|
|
|
2854
2933
|
* ```
|
|
2855
2934
|
*/
|
|
2856
2935
|
async stat() {
|
|
2857
|
-
this.
|
|
2936
|
+
this.assertLease();
|
|
2858
2937
|
return this.engine.stat(this.ref);
|
|
2859
2938
|
}
|
|
2860
2939
|
/**
|
|
@@ -2884,7 +2963,7 @@ var Segment = class _Segment {
|
|
|
2884
2963
|
engine.iterate(ref, bounds),
|
|
2885
2964
|
() => engine.iterateBatches(ref, bounds)
|
|
2886
2965
|
);
|
|
2887
|
-
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.
|
|
2966
|
+
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease());
|
|
2888
2967
|
}
|
|
2889
2968
|
/**
|
|
2890
2969
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -2918,22 +2997,22 @@ var Segment = class _Segment {
|
|
|
2918
2997
|
*/
|
|
2919
2998
|
everyNth(n, options) {
|
|
2920
2999
|
const lease = this.leaseError();
|
|
2921
|
-
if (lease !== void 0) return
|
|
3000
|
+
if (lease !== void 0) return refusing(lease)();
|
|
2922
3001
|
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
2923
|
-
if (bad !== void 0) return
|
|
3002
|
+
if (bad !== void 0) return refusing(bad)();
|
|
2924
3003
|
if (this.pinnedAt === void 0) {
|
|
2925
|
-
return
|
|
3004
|
+
return refusing(
|
|
2926
3005
|
new UnsupportedError(
|
|
2927
3006
|
"everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
|
|
2928
3007
|
)
|
|
2929
|
-
);
|
|
3008
|
+
)();
|
|
2930
3009
|
}
|
|
2931
3010
|
const stream = this.engine.everyNth(
|
|
2932
3011
|
this.ref,
|
|
2933
3012
|
n,
|
|
2934
3013
|
options == null ? void 0 : rangeOf(options)
|
|
2935
3014
|
);
|
|
2936
|
-
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.
|
|
3015
|
+
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease());
|
|
2937
3016
|
}
|
|
2938
3017
|
/**
|
|
2939
3018
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -2973,28 +3052,7 @@ var Segment = class _Segment {
|
|
|
2973
3052
|
intersect(others, options) {
|
|
2974
3053
|
const bad = optionsError(options, "intersect", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
2975
3054
|
if (bad !== void 0) return failing(bad);
|
|
2976
|
-
return this.
|
|
2977
|
-
}
|
|
2978
|
-
intersectAs(out, others, options) {
|
|
2979
|
-
let exclude;
|
|
2980
|
-
let handles;
|
|
2981
|
-
try {
|
|
2982
|
-
exclude = this.excludesOf(options);
|
|
2983
|
-
handles = [this, ...segmentList(others, "intersect: `others`"), ...exclude];
|
|
2984
|
-
this.assertSegments(handles);
|
|
2985
|
-
this.assertLeases(handles);
|
|
2986
|
-
} catch (err) {
|
|
2987
|
-
return out.failing(err);
|
|
2988
|
-
}
|
|
2989
|
-
let engine;
|
|
2990
|
-
try {
|
|
2991
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
2992
|
-
} catch (err) {
|
|
2993
|
-
return out.failing(err);
|
|
2994
|
-
}
|
|
2995
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
2996
|
-
const opts = this.refsIn(options, exclude);
|
|
2997
|
-
return this.guarded(out, handles, out.intersect(engine, refs, opts));
|
|
3055
|
+
return this.combineAs(AS_IDS, "intersect", others, options ?? void 0);
|
|
2998
3056
|
}
|
|
2999
3057
|
/**
|
|
3000
3058
|
* Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
|
|
@@ -3018,21 +3076,7 @@ var Segment = class _Segment {
|
|
|
3018
3076
|
*/
|
|
3019
3077
|
async intersectInto(dest, others, options) {
|
|
3020
3078
|
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3021
|
-
|
|
3022
|
-
...segmentList(others, "intersectInto: `others`"),
|
|
3023
|
-
...this.excludesOf(options)
|
|
3024
|
-
];
|
|
3025
|
-
this.assertIntoHandles(dest, operands);
|
|
3026
|
-
this.combineEngine([this, ...operands]);
|
|
3027
|
-
return this.timed(
|
|
3028
|
-
"intersectInto",
|
|
3029
|
-
() => this.materialize(
|
|
3030
|
-
dest.ref,
|
|
3031
|
-
new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
|
|
3032
|
-
"intersectInto",
|
|
3033
|
-
options
|
|
3034
|
-
)
|
|
3035
|
-
);
|
|
3079
|
+
return this.combineInto("intersectInto", dest, others, options);
|
|
3036
3080
|
}
|
|
3037
3081
|
/**
|
|
3038
3082
|
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
@@ -3047,44 +3091,12 @@ var Segment = class _Segment {
|
|
|
3047
3091
|
union(others, options) {
|
|
3048
3092
|
const bad = optionsError(options, "union", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
3049
3093
|
if (bad !== void 0) return failing(bad);
|
|
3050
|
-
return this.
|
|
3051
|
-
}
|
|
3052
|
-
unionAs(out, others, options) {
|
|
3053
|
-
let exclude;
|
|
3054
|
-
let handles;
|
|
3055
|
-
try {
|
|
3056
|
-
exclude = this.excludesOf(options);
|
|
3057
|
-
handles = [this, ...segmentList(others, "union: `others`"), ...exclude];
|
|
3058
|
-
this.assertSegments(handles);
|
|
3059
|
-
this.assertLeases(handles);
|
|
3060
|
-
} catch (err) {
|
|
3061
|
-
return out.failing(err);
|
|
3062
|
-
}
|
|
3063
|
-
let engine;
|
|
3064
|
-
try {
|
|
3065
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
3066
|
-
} catch (err) {
|
|
3067
|
-
return out.failing(err);
|
|
3068
|
-
}
|
|
3069
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
3070
|
-
const opts = this.refsIn(options, exclude);
|
|
3071
|
-
return this.guarded(out, handles, out.union(engine, refs, opts));
|
|
3094
|
+
return this.combineAs(AS_IDS, "union", others, options ?? void 0);
|
|
3072
3095
|
}
|
|
3073
3096
|
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3074
3097
|
async unionInto(dest, others, options) {
|
|
3075
3098
|
checkOptions(options, "unionInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3076
|
-
|
|
3077
|
-
this.assertIntoHandles(dest, operands);
|
|
3078
|
-
this.combineEngine([this, ...operands]);
|
|
3079
|
-
return this.timed(
|
|
3080
|
-
"unionInto",
|
|
3081
|
-
() => this.materialize(
|
|
3082
|
-
dest.ref,
|
|
3083
|
-
new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
|
|
3084
|
-
"unionInto",
|
|
3085
|
-
options
|
|
3086
|
-
)
|
|
3087
|
-
);
|
|
3099
|
+
return this.combineInto("unionInto", dest, others, options);
|
|
3088
3100
|
}
|
|
3089
3101
|
/**
|
|
3090
3102
|
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
@@ -3100,12 +3112,26 @@ var Segment = class _Segment {
|
|
|
3100
3112
|
andNot(excludes, options) {
|
|
3101
3113
|
const bad = optionsError(options, "andNot", CALL_KEYS.andNot, ["allowAbsentOperands"]);
|
|
3102
3114
|
if (bad !== void 0) return failing(bad);
|
|
3103
|
-
return this.
|
|
3115
|
+
return this.combineAs(AS_IDS, "andNot", excludes, options ?? void 0);
|
|
3104
3116
|
}
|
|
3105
|
-
|
|
3117
|
+
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3118
|
+
async andNotInto(dest, excludes, options) {
|
|
3119
|
+
checkOptions(options, "andNotInto", CALL_KEYS.andNotInto, [
|
|
3120
|
+
"allowEmpty",
|
|
3121
|
+
"allowAbsentOperands"
|
|
3122
|
+
]);
|
|
3123
|
+
return this.combineInto("andNotInto", dest, excludes, options);
|
|
3124
|
+
}
|
|
3125
|
+
/**
|
|
3126
|
+
* The one body of the three combines, read as ids or as chunks. `operands` is `others` for `intersect` and
|
|
3127
|
+
* `union`, and `excludes` for `andNot`, which takes no `exclude` option.
|
|
3128
|
+
*/
|
|
3129
|
+
combineAs(out, verb, operands, options) {
|
|
3130
|
+
let exclude;
|
|
3106
3131
|
let handles;
|
|
3107
3132
|
try {
|
|
3108
|
-
|
|
3133
|
+
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3134
|
+
handles = [this, ...segmentList(operands, operandsLabel(verb)), ...exclude];
|
|
3109
3135
|
this.assertSegments(handles);
|
|
3110
3136
|
this.assertLeases(handles);
|
|
3111
3137
|
} catch (err) {
|
|
@@ -3113,29 +3139,35 @@ var Segment = class _Segment {
|
|
|
3113
3139
|
}
|
|
3114
3140
|
let engine;
|
|
3115
3141
|
try {
|
|
3116
|
-
engine = this.combineEngine([this, ...
|
|
3142
|
+
engine = this.combineEngine([this, ...operands, ...exclude]) ?? this.engine;
|
|
3117
3143
|
} catch (err) {
|
|
3118
3144
|
return out.failing(err);
|
|
3119
3145
|
}
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3146
|
+
if (verb === "andNot") {
|
|
3147
|
+
const refs2 = operands.map((o) => o.ref);
|
|
3148
|
+
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3149
|
+
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3150
|
+
}
|
|
3151
|
+
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3152
|
+
const opts = this.refsIn(options, exclude);
|
|
3153
|
+
return this.guarded(
|
|
3154
|
+
out,
|
|
3155
|
+
handles,
|
|
3156
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3157
|
+
);
|
|
3124
3158
|
}
|
|
3125
|
-
/**
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
]);
|
|
3131
|
-
this.assertIntoHandles(dest, segmentList(excludes, "andNotInto: `excludes`"));
|
|
3132
|
-
this.combineEngine([this, ...excludes]);
|
|
3159
|
+
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
3160
|
+
combineInto(name, dest, operands, options) {
|
|
3161
|
+
const verb = name.slice(0, -"Into".length);
|
|
3162
|
+
const all = verb === "andNot" ? segmentList(operands, `${name}: \`excludes\``) : [...segmentList(operands, `${name}: \`others\``), ...this.excludesOf(options)];
|
|
3163
|
+
this.assertIntoHandles(dest, all);
|
|
3164
|
+
this.combineEngine([this, ...all]);
|
|
3133
3165
|
return this.timed(
|
|
3134
|
-
|
|
3166
|
+
name,
|
|
3135
3167
|
() => this.materialize(
|
|
3136
3168
|
dest.ref,
|
|
3137
|
-
new CombineChunks(this.
|
|
3138
|
-
|
|
3169
|
+
new CombineChunks(this.combineAs(AS_CHUNKS, verb, operands, options)),
|
|
3170
|
+
name,
|
|
3139
3171
|
options
|
|
3140
3172
|
)
|
|
3141
3173
|
);
|
|
@@ -3146,15 +3178,15 @@ var Segment = class _Segment {
|
|
|
3146
3178
|
function parseConfig(env) {
|
|
3147
3179
|
const root = env.CR_EXPORT_ROOT;
|
|
3148
3180
|
if (root === void 0 || root === "") {
|
|
3149
|
-
throw new
|
|
3181
|
+
throw new ValidationError5("CR_EXPORT_ROOT is required (the local-filesystem storage root)");
|
|
3150
3182
|
}
|
|
3151
3183
|
const out = env.CR_EXPORT_OUT;
|
|
3152
3184
|
if (out === void 0 || out === "") {
|
|
3153
|
-
throw new
|
|
3185
|
+
throw new ValidationError5("CR_EXPORT_OUT is required (the output directory for the dump)");
|
|
3154
3186
|
}
|
|
3155
3187
|
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3156
3188
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3157
|
-
throw new
|
|
3189
|
+
throw new ValidationError5(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3158
3190
|
}
|
|
3159
3191
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
3160
3192
|
return { root, out, format, namespace };
|
|
@@ -3169,7 +3201,7 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3169
3201
|
}
|
|
3170
3202
|
const found = await lstat(dir);
|
|
3171
3203
|
const refuse = (why) => {
|
|
3172
|
-
throw new
|
|
3204
|
+
throw new ValidationError5(`the export directory is not one this tool made: ${why}`);
|
|
3173
3205
|
};
|
|
3174
3206
|
if (found.isSymbolicLink()) refuse("a namespace directory is a symbolic link");
|
|
3175
3207
|
if (!found.isDirectory()) refuse("a namespace path is not a directory");
|
|
@@ -3178,10 +3210,17 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3178
3210
|
if ((found.mode & 63) !== 0) refuse("a namespace directory is open to group or others");
|
|
3179
3211
|
}
|
|
3180
3212
|
}
|
|
3213
|
+
var MANIFEST_NAME = "manifest.json";
|
|
3181
3214
|
function fsSink(out) {
|
|
3182
3215
|
return {
|
|
3183
3216
|
async open(ref, ext) {
|
|
3184
|
-
const
|
|
3217
|
+
const part = namespacePathPart(ref.namespace);
|
|
3218
|
+
if (part.toLowerCase() === MANIFEST_NAME) {
|
|
3219
|
+
throw new ValidationError5(
|
|
3220
|
+
`a namespace named ${MANIFEST_NAME} (in any letter case) cannot be exported to a directory: its folder would be the manifest's path`
|
|
3221
|
+
);
|
|
3222
|
+
}
|
|
3223
|
+
const dir = join(out, part);
|
|
3185
3224
|
await ensureOwnDirectory(out, dir);
|
|
3186
3225
|
const finalPath = join(dir, `${encodeNameForPath(ref.segment)}${ext}`);
|
|
3187
3226
|
const tmpPath = `${finalPath}.${randomUUID()}.part`;
|
|
@@ -3208,6 +3247,10 @@ function fsSink(out) {
|
|
|
3208
3247
|
await handle.close().catch(() => {
|
|
3209
3248
|
});
|
|
3210
3249
|
await rm(tmpPath, { force: true });
|
|
3250
|
+
await rmdir(dir).catch((err) => {
|
|
3251
|
+
if (err.code !== "ENOTEMPTY" && err.code !== "EEXIST" && err.code !== "ENOENT")
|
|
3252
|
+
throw err;
|
|
3253
|
+
});
|
|
3211
3254
|
}
|
|
3212
3255
|
};
|
|
3213
3256
|
}
|
|
@@ -3218,10 +3261,10 @@ async function requireStoreRoot(root) {
|
|
|
3218
3261
|
try {
|
|
3219
3262
|
entries = await readdir(root);
|
|
3220
3263
|
} catch {
|
|
3221
|
-
throw new
|
|
3264
|
+
throw new ValidationError5("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3222
3265
|
}
|
|
3223
3266
|
if (entries.length === 0) {
|
|
3224
|
-
throw new
|
|
3267
|
+
throw new ValidationError5(
|
|
3225
3268
|
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3226
3269
|
);
|
|
3227
3270
|
}
|
|
@@ -3235,7 +3278,13 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3235
3278
|
await requireStoreRoot(config.root);
|
|
3236
3279
|
const storage = new LocalFsStorage(config.root);
|
|
3237
3280
|
const store = new CloudRoaring({ storage });
|
|
3238
|
-
await
|
|
3281
|
+
const priorManifest = await lstat(join(config.out, MANIFEST_NAME)).catch(() => null);
|
|
3282
|
+
if (priorManifest?.isDirectory() === true) {
|
|
3283
|
+
throw new ValidationError5(
|
|
3284
|
+
`${join(config.out, MANIFEST_NAME)} is a directory, so the manifest cannot be written there: remove or rename it, or export to a fresh directory`
|
|
3285
|
+
);
|
|
3286
|
+
}
|
|
3287
|
+
await rm(join(config.out, MANIFEST_NAME), { force: true });
|
|
3239
3288
|
const manifest = await store.exportSegments(fsSink(config.out), {
|
|
3240
3289
|
format: config.format,
|
|
3241
3290
|
namespace: config.namespace
|