@cloudbitmaps/roaring 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +84 -43
- package/README.md +4 -2
- package/dist/bin/export-segments.d.ts +5 -1
- package/dist/bin/export-segments.js +579 -549
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +220 -111
- package/dist/index.js +554 -545
- package/dist/index.js.map +4 -4
- package/dist/lease-guards.d.ts +25 -13
- package/dist/open-charging-storage.d.ts +7 -4
- package/dist/options.d.ts +30 -0
- package/dist/portable/decode.d.ts +2 -1
- package/dist/portable/layout.d.ts +3 -2
- package/dist/testing/conformance.d.ts +15 -10
- package/dist/wire-store.d.ts +38 -0
- package/package.json +2 -2
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// src/bin/export-segments.ts
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { realpathSync } from "node:fs";
|
|
6
|
-
import { lstat, mkdir, open, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
6
|
+
import { lstat, mkdir, open, readdir, rename, rm, rmdir, writeFile } from "node:fs/promises";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import { pathToFileURL } from "node:url";
|
|
9
9
|
import { encodeNameForPath, namespacePathPart } from "@cloudbitmaps/core/driver-kit";
|
|
@@ -13,28 +13,25 @@ import {
|
|
|
13
13
|
BoundedLru,
|
|
14
14
|
PinnedStorageChunkSource,
|
|
15
15
|
segmentKey,
|
|
16
|
-
CrbmStorageChunkSource,
|
|
16
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource2,
|
|
17
17
|
DEFAULT_BUDGET,
|
|
18
|
-
DEFAULT_RETRY_POLICY,
|
|
18
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2,
|
|
19
19
|
withRetry,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
RetryingStorageChunkSource,
|
|
20
|
+
NOOP_METRICS as NOOP_METRICS2,
|
|
21
|
+
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
23
22
|
SegmentEngine,
|
|
24
23
|
LEASE_SKEW_MS,
|
|
25
24
|
LeaseExpiredError,
|
|
26
25
|
MAX_LEASE_MS,
|
|
27
26
|
NotFoundError,
|
|
28
27
|
UnsupportedError,
|
|
29
|
-
ValidationError as
|
|
28
|
+
ValidationError as ValidationError4,
|
|
30
29
|
WriteConflictError,
|
|
31
30
|
checkBudget,
|
|
32
31
|
collectWithinBudget,
|
|
33
32
|
excludingReservedRows,
|
|
34
33
|
dropSegment,
|
|
35
34
|
eraseIdFromSegment,
|
|
36
|
-
estimateCost,
|
|
37
|
-
groundedReport,
|
|
38
35
|
judgeLoad,
|
|
39
36
|
loadSegment,
|
|
40
37
|
loadSegmentChunks,
|
|
@@ -55,7 +52,7 @@ import {
|
|
|
55
52
|
splitId,
|
|
56
53
|
takeLease
|
|
57
54
|
} from "@cloudbitmaps/core";
|
|
58
|
-
import { validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
55
|
+
import { pointerIdOf, requirePositiveInt, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
59
56
|
|
|
60
57
|
// src/open-charging-storage.ts
|
|
61
58
|
var OpenChargingStorage = class {
|
|
@@ -88,8 +85,9 @@ var OpenChargingStorage = class {
|
|
|
88
85
|
if (!this.free.has(key.generation)) this.charge();
|
|
89
86
|
return this.inner.getTail(key, maxBytes);
|
|
90
87
|
}
|
|
91
|
-
delete
|
|
92
|
-
|
|
88
|
+
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
89
|
+
delete(key, options) {
|
|
90
|
+
return this.inner.delete(key, options);
|
|
93
91
|
}
|
|
94
92
|
list(ref) {
|
|
95
93
|
return this.inner.list(ref);
|
|
@@ -530,90 +528,265 @@ var OPTION_KEYS = {
|
|
|
530
528
|
};
|
|
531
529
|
|
|
532
530
|
// src/lease-guards.ts
|
|
533
|
-
var
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
return
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
check();
|
|
558
|
-
} catch (err) {
|
|
559
|
-
return stop(err);
|
|
560
|
-
}
|
|
561
|
-
chunk = key;
|
|
562
|
-
}
|
|
563
|
-
return r;
|
|
564
|
-
},
|
|
565
|
-
async return() {
|
|
566
|
-
await it.return?.();
|
|
567
|
-
return { done: true, value: void 0 };
|
|
531
|
+
var done = (value) => ({ done: true, value });
|
|
532
|
+
var guard = (inner, check, keyOf) => {
|
|
533
|
+
const it = inner[Symbol.asyncIterator]();
|
|
534
|
+
let first = true;
|
|
535
|
+
let over = false;
|
|
536
|
+
let last = -1;
|
|
537
|
+
const end = async (value) => {
|
|
538
|
+
over = true;
|
|
539
|
+
await it.return?.(value);
|
|
540
|
+
return value;
|
|
541
|
+
};
|
|
542
|
+
const stop = async (err) => {
|
|
543
|
+
await end();
|
|
544
|
+
throw err;
|
|
545
|
+
};
|
|
546
|
+
const deliver = (r) => {
|
|
547
|
+
if (over) return done();
|
|
548
|
+
if (r.done === true) return r;
|
|
549
|
+
const key = keyOf?.(r.value);
|
|
550
|
+
if (key === void 0 || key !== last) {
|
|
551
|
+
try {
|
|
552
|
+
check();
|
|
553
|
+
} catch (err) {
|
|
554
|
+
return stop(err);
|
|
568
555
|
}
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
await it.return?.();
|
|
582
|
-
throw err;
|
|
583
|
-
};
|
|
584
|
-
return {
|
|
585
|
-
async next() {
|
|
586
|
-
if (first) {
|
|
587
|
-
first = false;
|
|
588
|
-
try {
|
|
589
|
-
check();
|
|
590
|
-
} catch (err) {
|
|
591
|
-
return stop(err);
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
const r = await it.next();
|
|
595
|
-
if (r.done === true) return r;
|
|
556
|
+
if (key !== void 0) last = key;
|
|
557
|
+
}
|
|
558
|
+
return r;
|
|
559
|
+
};
|
|
560
|
+
return {
|
|
561
|
+
[Symbol.asyncIterator]() {
|
|
562
|
+
return this;
|
|
563
|
+
},
|
|
564
|
+
async next() {
|
|
565
|
+
if (over) return done();
|
|
566
|
+
if (first) {
|
|
567
|
+
first = false;
|
|
596
568
|
try {
|
|
597
569
|
check();
|
|
598
570
|
} catch (err) {
|
|
599
571
|
return stop(err);
|
|
600
572
|
}
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
573
|
+
}
|
|
574
|
+
return deliver(await it.next());
|
|
575
|
+
},
|
|
576
|
+
async return(value) {
|
|
577
|
+
return done(await end(value));
|
|
578
|
+
},
|
|
579
|
+
async throw(err) {
|
|
580
|
+
first = false;
|
|
581
|
+
if (it.throw === void 0) return stop(err);
|
|
582
|
+
let r;
|
|
583
|
+
try {
|
|
584
|
+
r = await it.throw(err);
|
|
585
|
+
} catch (thrown) {
|
|
586
|
+
over = true;
|
|
587
|
+
throw thrown;
|
|
588
|
+
}
|
|
589
|
+
return deliver(r);
|
|
590
|
+
}
|
|
591
|
+
};
|
|
592
|
+
};
|
|
593
|
+
var guardIdIterable = (inner, check) => guard(inner, check, (id) => id >>> 16);
|
|
594
|
+
var guardIds = (inner, check) => Object.assign(guardIdIterable(inner, check), {
|
|
595
|
+
batches: () => guardItems(inner.batches(), check)
|
|
596
|
+
});
|
|
597
|
+
var guardItems = (inner, check) => guard(inner, check);
|
|
598
|
+
|
|
599
|
+
// src/wire-store.ts
|
|
600
|
+
import {
|
|
601
|
+
CrbmStorageChunkSource,
|
|
602
|
+
DEFAULT_RETRY_POLICY,
|
|
603
|
+
NOOP_METRICS,
|
|
604
|
+
RetryingStorageChunkSource,
|
|
605
|
+
ValidationError as ValidationError2,
|
|
606
|
+
isStorageBackend
|
|
607
|
+
} from "@cloudbitmaps/core";
|
|
608
|
+
function resolveRetryPolicy(partial) {
|
|
609
|
+
const baseDelayMs = partial.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs;
|
|
610
|
+
return {
|
|
611
|
+
maxAttempts: partial.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
612
|
+
baseDelayMs,
|
|
613
|
+
maxDelayMs: partial.maxDelayMs ?? (typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs),
|
|
614
|
+
backoffFactor: partial.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
615
|
+
jitter: partial.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
function wireStore(options, clock, rng, metrics) {
|
|
619
|
+
const resolved = resolveStorageSource(options, clock);
|
|
620
|
+
if (metrics !== NOOP_METRICS && isStorageBackend(options.storage)) {
|
|
621
|
+
options.storage.attachMetrics?.(metrics);
|
|
622
|
+
}
|
|
623
|
+
let storage = resolved.source;
|
|
624
|
+
let retryOptions;
|
|
625
|
+
if (options.retry !== false) {
|
|
626
|
+
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
627
|
+
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
628
|
+
throw new ValidationError2("retry.onRetry must be a function");
|
|
629
|
+
}
|
|
630
|
+
retryOptions = {
|
|
631
|
+
clock,
|
|
632
|
+
rng,
|
|
633
|
+
policy: resolveRetryPolicy(ov),
|
|
634
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
635
|
+
onRetry: (info) => {
|
|
636
|
+
metrics.onEvent({
|
|
637
|
+
kind: "retry",
|
|
638
|
+
reason: "transient",
|
|
639
|
+
attempt: info.attempt,
|
|
640
|
+
delayMs: info.delayMs
|
|
641
|
+
});
|
|
642
|
+
userOnRetry?.(info);
|
|
606
643
|
}
|
|
607
644
|
};
|
|
645
|
+
storage = new RetryingStorageChunkSource(storage, retryOptions);
|
|
608
646
|
}
|
|
609
|
-
|
|
610
|
-
|
|
647
|
+
return {
|
|
648
|
+
storage,
|
|
649
|
+
source: resolved.source,
|
|
650
|
+
driver: resolved.driver,
|
|
651
|
+
registry: resolved.registry,
|
|
652
|
+
retryOptions
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
function resolveStorageSource(options, clock) {
|
|
656
|
+
const storage = options.storage;
|
|
657
|
+
if (storage === null || typeof storage !== "object") {
|
|
658
|
+
throw new ValidationError2(
|
|
659
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
660
|
+
);
|
|
661
|
+
}
|
|
662
|
+
const asBackend = storage;
|
|
663
|
+
const isObject = (v) => v !== null && typeof v === "object";
|
|
664
|
+
const hasGetChunk = typeof storage.getChunk === "function";
|
|
665
|
+
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
666
|
+
const isBackend = isStorageBackend(storage);
|
|
667
|
+
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
668
|
+
throw new ValidationError2(
|
|
669
|
+
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
670
|
+
);
|
|
671
|
+
}
|
|
672
|
+
const hasStorageHalf = isObject(asBackend.storage);
|
|
673
|
+
const hasRegistryHalf = isObject(asBackend.registry);
|
|
674
|
+
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
675
|
+
throw new ValidationError2(
|
|
676
|
+
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
680
|
+
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
681
|
+
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
682
|
+
if (!storageOk || !registryOk) {
|
|
683
|
+
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
684
|
+
throw new ValidationError2(
|
|
685
|
+
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
throw new ValidationError2(
|
|
689
|
+
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
if (isBackend) {
|
|
693
|
+
const backend = storage;
|
|
694
|
+
return {
|
|
695
|
+
source: new CrbmStorageChunkSource(backend.storage, {
|
|
696
|
+
registry: backend.registry,
|
|
697
|
+
keystore: options.encryption?.keystore,
|
|
698
|
+
requireEncryption: options.encryption?.required,
|
|
699
|
+
clock,
|
|
700
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
701
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
702
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
703
|
+
}),
|
|
704
|
+
driver: backend.storage,
|
|
705
|
+
registry: backend.registry
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
709
|
+
throw new ValidationError2(
|
|
710
|
+
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
714
|
+
throw new ValidationError2(
|
|
715
|
+
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
if (hasGetChunk) {
|
|
719
|
+
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
720
|
+
throw new ValidationError2(
|
|
721
|
+
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
return { source: storage, driver: void 0, registry: void 0 };
|
|
725
|
+
}
|
|
726
|
+
const driver = storage;
|
|
727
|
+
return {
|
|
728
|
+
source: new CrbmStorageChunkSource(driver, {
|
|
729
|
+
keystore: options.encryption?.keystore,
|
|
730
|
+
requireEncryption: options.encryption?.required,
|
|
731
|
+
clock,
|
|
732
|
+
currentGenTtlMs: options.cache?.genTtlMs,
|
|
733
|
+
maxOpenSegments: options.cache?.readerMax,
|
|
734
|
+
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
735
|
+
}),
|
|
736
|
+
driver,
|
|
737
|
+
registry: void 0
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/options.ts
|
|
742
|
+
import { ValidationError as ValidationError3 } from "@cloudbitmaps/core";
|
|
743
|
+
var keysOf = () => (keys) => keys;
|
|
744
|
+
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
745
|
+
function optionsError(options, op, known, booleans = []) {
|
|
746
|
+
if (options === void 0 || options === null) return void 0;
|
|
747
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
748
|
+
return new ValidationError3(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
749
|
+
}
|
|
750
|
+
const bag = options;
|
|
751
|
+
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
752
|
+
if (unknown.length > 0) {
|
|
753
|
+
return new ValidationError3(
|
|
754
|
+
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
const audit = bag.audit;
|
|
758
|
+
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
759
|
+
return new ValidationError3(
|
|
760
|
+
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
for (const name of booleans) {
|
|
764
|
+
const v = bag[name];
|
|
765
|
+
if (v !== void 0 && typeof v !== "boolean") {
|
|
766
|
+
return new ValidationError3(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return void 0;
|
|
770
|
+
}
|
|
771
|
+
function picked(options, keys) {
|
|
772
|
+
const out = {};
|
|
773
|
+
if (options === void 0 || options === null) return out;
|
|
774
|
+
for (const key of keys) {
|
|
775
|
+
const value = options[key];
|
|
776
|
+
if (value !== void 0) out[key] = value;
|
|
777
|
+
}
|
|
778
|
+
return out;
|
|
779
|
+
}
|
|
780
|
+
function checkOptions(options, op, known, booleans) {
|
|
781
|
+
const err = optionsError(options, op, known, booleans);
|
|
782
|
+
if (err !== void 0) throw err;
|
|
783
|
+
}
|
|
611
784
|
|
|
612
785
|
// src/index.ts
|
|
613
786
|
import {
|
|
614
787
|
MemoryStorage,
|
|
615
788
|
LocalFsStorage,
|
|
616
|
-
CrbmStorageChunkSource as
|
|
789
|
+
CrbmStorageChunkSource as CrbmStorageChunkSource3,
|
|
617
790
|
destroySegment,
|
|
618
791
|
eraseNamespace,
|
|
619
792
|
excludingReservedRows as excludingReservedRows2,
|
|
@@ -626,7 +799,7 @@ import {
|
|
|
626
799
|
NodeAead,
|
|
627
800
|
aadFor,
|
|
628
801
|
CloudRoaringError,
|
|
629
|
-
ValidationError as
|
|
802
|
+
ValidationError as ValidationError5,
|
|
630
803
|
WriteConflictError as WriteConflictError2,
|
|
631
804
|
IntegrityError as IntegrityError3,
|
|
632
805
|
NotFoundError as NotFoundError2,
|
|
@@ -655,10 +828,7 @@ import {
|
|
|
655
828
|
BufferReader,
|
|
656
829
|
CountingMetricsSink,
|
|
657
830
|
RecordingAuditSink,
|
|
658
|
-
DEFAULT_RETRY_POLICY as
|
|
659
|
-
AWS_US_EAST_1_ONDEMAND,
|
|
660
|
-
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
661
|
-
ONE_REDIS_HA_CLUSTER
|
|
831
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY3
|
|
662
832
|
} from "@cloudbitmaps/core";
|
|
663
833
|
var SystemRng = class {
|
|
664
834
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -674,14 +844,12 @@ function rowVersionOfLive(version) {
|
|
|
674
844
|
return cut < 0 ? version : version.slice(0, cut);
|
|
675
845
|
}
|
|
676
846
|
function validateConcurrency(concurrency) {
|
|
677
|
-
|
|
678
|
-
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
679
|
-
}
|
|
847
|
+
requirePositiveInt("concurrency", concurrency);
|
|
680
848
|
}
|
|
681
849
|
function requireScope(options, op) {
|
|
682
850
|
if (options.namespace !== void 0) {
|
|
683
851
|
if (typeof options.namespace !== "string") {
|
|
684
|
-
throw new
|
|
852
|
+
throw new ValidationError4(
|
|
685
853
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
686
854
|
);
|
|
687
855
|
}
|
|
@@ -689,12 +857,11 @@ function requireScope(options, op) {
|
|
|
689
857
|
}
|
|
690
858
|
refuseReservedNamespace(options.namespace);
|
|
691
859
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
692
|
-
throw new
|
|
860
|
+
throw new ValidationError4(
|
|
693
861
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
694
862
|
);
|
|
695
863
|
}
|
|
696
864
|
}
|
|
697
|
-
var keysOf = () => (keys) => keys;
|
|
698
865
|
var CALL_KEYS = {
|
|
699
866
|
range: keysOf()(["after", "through"]),
|
|
700
867
|
andNot: keysOf()([
|
|
@@ -754,7 +921,7 @@ var CALL_KEYS = {
|
|
|
754
921
|
]),
|
|
755
922
|
rollback: keysOf()([
|
|
756
923
|
"audit",
|
|
757
|
-
"
|
|
924
|
+
"allowUnknown"
|
|
758
925
|
]),
|
|
759
926
|
dropSegment: keysOf()([
|
|
760
927
|
"confirmSegment",
|
|
@@ -778,66 +945,12 @@ var CALL_KEYS = {
|
|
|
778
945
|
checkConsistency: keysOf()([
|
|
779
946
|
"namespace",
|
|
780
947
|
"concurrency",
|
|
781
|
-
"summaries"
|
|
948
|
+
"summaries",
|
|
949
|
+
"generations"
|
|
782
950
|
]),
|
|
783
|
-
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
784
|
-
costReport: keysOf()(["pricing", "workload"]),
|
|
785
|
-
workload: keysOf()([
|
|
786
|
-
"readsPerSec",
|
|
787
|
-
"intersectsPerSec",
|
|
788
|
-
"cacheHitRate",
|
|
789
|
-
"chunksPerIntersect",
|
|
790
|
-
"operandsPerIntersect",
|
|
791
|
-
"loadsPerMonth",
|
|
792
|
-
"requestsPerLoad",
|
|
793
|
-
"hotSegments",
|
|
794
|
-
"readerProcesses",
|
|
795
|
-
"genTtlMs",
|
|
796
|
-
"retirementsPerMonth",
|
|
797
|
-
"purgesPerMonth",
|
|
798
|
-
"conditionalDelete"
|
|
799
|
-
])
|
|
951
|
+
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
800
952
|
};
|
|
801
|
-
var
|
|
802
|
-
function optionsError(options, op, known, booleans = []) {
|
|
803
|
-
if (options === void 0 || options === null) return void 0;
|
|
804
|
-
if (typeof options !== "object" || Array.isArray(options)) {
|
|
805
|
-
return new ValidationError2(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
806
|
-
}
|
|
807
|
-
const bag = options;
|
|
808
|
-
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
809
|
-
if (unknown.length > 0) {
|
|
810
|
-
return new ValidationError2(
|
|
811
|
-
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
812
|
-
);
|
|
813
|
-
}
|
|
814
|
-
const audit = bag.audit;
|
|
815
|
-
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
816
|
-
return new ValidationError2(
|
|
817
|
-
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
818
|
-
);
|
|
819
|
-
}
|
|
820
|
-
for (const name of booleans) {
|
|
821
|
-
const v = bag[name];
|
|
822
|
-
if (v !== void 0 && typeof v !== "boolean") {
|
|
823
|
-
return new ValidationError2(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
return void 0;
|
|
827
|
-
}
|
|
828
|
-
function picked(options, keys) {
|
|
829
|
-
const out = {};
|
|
830
|
-
if (options === void 0 || options === null) return out;
|
|
831
|
-
for (const key of keys) {
|
|
832
|
-
const value = options[key];
|
|
833
|
-
if (value !== void 0) out[key] = value;
|
|
834
|
-
}
|
|
835
|
-
return out;
|
|
836
|
-
}
|
|
837
|
-
function checkOptions(options, op, known, booleans) {
|
|
838
|
-
const err = optionsError(options, op, known, booleans);
|
|
839
|
-
if (err !== void 0) throw err;
|
|
840
|
-
}
|
|
953
|
+
var SEGMENT_BRAND = /* @__PURE__ */ Symbol.for("cloudbitmaps.roaring.segment/1");
|
|
841
954
|
var viewOf;
|
|
842
955
|
var MATERIALIZE_MANY_KEYS = [
|
|
843
956
|
"operands",
|
|
@@ -869,91 +982,6 @@ var DEFAULT_MAX_BUFFERED_BYTES = 256 * 1024 * 1024;
|
|
|
869
982
|
var DEFAULT_PUBLISH_CONCURRENCY = 8;
|
|
870
983
|
var DEFAULT_BATCH_CONCURRENCY = 1;
|
|
871
984
|
var PIN_PARALLELISM = 16;
|
|
872
|
-
function resolveStorageSource(options, clock) {
|
|
873
|
-
const storage = options.storage;
|
|
874
|
-
if (storage === null || typeof storage !== "object") {
|
|
875
|
-
throw new ValidationError2(
|
|
876
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
877
|
-
);
|
|
878
|
-
}
|
|
879
|
-
const asBackend = storage;
|
|
880
|
-
const isObject = (v) => v !== null && typeof v === "object";
|
|
881
|
-
const hasGetChunk = typeof storage.getChunk === "function";
|
|
882
|
-
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
883
|
-
const isBackend = isStorageBackend(storage);
|
|
884
|
-
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
885
|
-
throw new ValidationError2(
|
|
886
|
-
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
887
|
-
);
|
|
888
|
-
}
|
|
889
|
-
const hasStorageHalf = isObject(asBackend.storage);
|
|
890
|
-
const hasRegistryHalf = isObject(asBackend.registry);
|
|
891
|
-
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
892
|
-
throw new ValidationError2(
|
|
893
|
-
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
894
|
-
);
|
|
895
|
-
}
|
|
896
|
-
if (!isBackend && !hasGetChunk && !hasPutImmutable && hasStorageHalf && hasRegistryHalf) {
|
|
897
|
-
const storageOk = typeof asBackend.storage.putImmutable === "function";
|
|
898
|
-
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
899
|
-
if (!storageOk || !registryOk) {
|
|
900
|
-
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
901
|
-
throw new ValidationError2(
|
|
902
|
-
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
903
|
-
);
|
|
904
|
-
}
|
|
905
|
-
throw new ValidationError2(
|
|
906
|
-
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
907
|
-
);
|
|
908
|
-
}
|
|
909
|
-
if (isBackend) {
|
|
910
|
-
const backend = storage;
|
|
911
|
-
return {
|
|
912
|
-
source: new CrbmStorageChunkSource(backend.storage, {
|
|
913
|
-
registry: backend.registry,
|
|
914
|
-
keystore: options.encryption?.keystore,
|
|
915
|
-
requireEncryption: options.encryption?.required,
|
|
916
|
-
clock,
|
|
917
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
918
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
919
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
920
|
-
}),
|
|
921
|
-
driver: backend.storage,
|
|
922
|
-
registry: backend.registry
|
|
923
|
-
};
|
|
924
|
-
}
|
|
925
|
-
if (hasGetChunk && hasPutImmutable) {
|
|
926
|
-
throw new ValidationError2(
|
|
927
|
-
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
928
|
-
);
|
|
929
|
-
}
|
|
930
|
-
if (!hasGetChunk && !hasPutImmutable) {
|
|
931
|
-
throw new ValidationError2(
|
|
932
|
-
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
933
|
-
);
|
|
934
|
-
}
|
|
935
|
-
if (hasGetChunk) {
|
|
936
|
-
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
937
|
-
throw new ValidationError2(
|
|
938
|
-
"keystore/requireEncryption apply only when the store builds its own read path; configure them on the StorageChunkSource you passed instead"
|
|
939
|
-
);
|
|
940
|
-
}
|
|
941
|
-
return { source: storage, driver: void 0, registry: void 0 };
|
|
942
|
-
}
|
|
943
|
-
const driver = storage;
|
|
944
|
-
return {
|
|
945
|
-
source: new CrbmStorageChunkSource(driver, {
|
|
946
|
-
keystore: options.encryption?.keystore,
|
|
947
|
-
requireEncryption: options.encryption?.required,
|
|
948
|
-
clock,
|
|
949
|
-
currentGenTtlMs: options.cache?.genTtlMs,
|
|
950
|
-
maxOpenSegments: options.cache?.readerMax,
|
|
951
|
-
maxOpenIndexBytes: options.cache?.readerMaxBytes
|
|
952
|
-
}),
|
|
953
|
-
driver,
|
|
954
|
-
registry: void 0
|
|
955
|
-
};
|
|
956
|
-
}
|
|
957
985
|
var NOT_A_GROUP = {
|
|
958
986
|
Null: "null",
|
|
959
987
|
Array: "an array",
|
|
@@ -996,7 +1024,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
996
1024
|
*/
|
|
997
1025
|
static rejectUnknownOptions(options) {
|
|
998
1026
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
999
|
-
throw new
|
|
1027
|
+
throw new ValidationError4(
|
|
1000
1028
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1001
1029
|
);
|
|
1002
1030
|
}
|
|
@@ -1014,7 +1042,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1014
1042
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1015
1043
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1016
1044
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1017
|
-
throw new
|
|
1045
|
+
throw new ValidationError4(
|
|
1018
1046
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1019
1047
|
);
|
|
1020
1048
|
}
|
|
@@ -1024,7 +1052,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1024
1052
|
if (unknown.length === 0) return;
|
|
1025
1053
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1026
1054
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1027
|
-
throw new
|
|
1055
|
+
throw new ValidationError4(
|
|
1028
1056
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1029
1057
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1030
1058
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1044,7 +1072,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1044
1072
|
if (seam === void 0 || seam === null) continue;
|
|
1045
1073
|
const has = (m) => typeof seam[m] === "function";
|
|
1046
1074
|
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1047
|
-
throw new
|
|
1075
|
+
throw new ValidationError4(
|
|
1048
1076
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1049
1077
|
);
|
|
1050
1078
|
}
|
|
@@ -1055,50 +1083,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1055
1083
|
_CloudRoaring.rejectHollowSeams(options);
|
|
1056
1084
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1057
1085
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1058
|
-
const metrics = safeMetrics(options.metrics ??
|
|
1086
|
+
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS2);
|
|
1059
1087
|
const cache = new BoundedLru({
|
|
1060
1088
|
maxEntries: options.cache?.maxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
1061
1089
|
ttlMs: options.cache?.ttlMs,
|
|
1062
1090
|
clock
|
|
1063
1091
|
});
|
|
1064
|
-
const
|
|
1065
|
-
|
|
1066
|
-
options.storage.attachMetrics?.(metrics);
|
|
1067
|
-
}
|
|
1068
|
-
let storage = resolved.source;
|
|
1069
|
-
if (options.retry !== false) {
|
|
1070
|
-
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
1071
|
-
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
1072
|
-
throw new ValidationError2("retry.onRetry must be a function");
|
|
1073
|
-
}
|
|
1074
|
-
const policy = {
|
|
1075
|
-
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
1076
|
-
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
1077
|
-
maxDelayMs: ov.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
1078
|
-
backoffFactor: ov.backoffFactor ?? DEFAULT_RETRY_POLICY.backoffFactor,
|
|
1079
|
-
jitter: ov.jitter ?? DEFAULT_RETRY_POLICY.jitter
|
|
1080
|
-
};
|
|
1081
|
-
const retryOpts = {
|
|
1082
|
-
clock,
|
|
1083
|
-
rng,
|
|
1084
|
-
policy,
|
|
1085
|
-
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
1086
|
-
onRetry: (info) => {
|
|
1087
|
-
metrics.onEvent({
|
|
1088
|
-
kind: "retry",
|
|
1089
|
-
reason: "transient",
|
|
1090
|
-
attempt: info.attempt,
|
|
1091
|
-
delayMs: info.delayMs
|
|
1092
|
-
});
|
|
1093
|
-
userOnRetry?.(info);
|
|
1094
|
-
}
|
|
1095
|
-
};
|
|
1096
|
-
storage = new RetryingStorageChunkSource(storage, retryOpts);
|
|
1097
|
-
this.retryOptions = retryOpts;
|
|
1098
|
-
}
|
|
1092
|
+
const wired = wireStore(options, clock, rng, metrics);
|
|
1093
|
+
this.retryOptions = wired.retryOptions;
|
|
1099
1094
|
this.budget = resolveBudget(options.budget, DEFAULT_BUDGET);
|
|
1100
1095
|
const deps = {
|
|
1101
|
-
storage,
|
|
1096
|
+
storage: wired.storage,
|
|
1102
1097
|
cache,
|
|
1103
1098
|
codec: roaringCodec,
|
|
1104
1099
|
// the facade injects the flagship codec; core stays codec-agnostic
|
|
@@ -1108,12 +1103,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1108
1103
|
};
|
|
1109
1104
|
this.engine = new SegmentEngine(deps);
|
|
1110
1105
|
this.cache = cache;
|
|
1111
|
-
this.crbmSource =
|
|
1106
|
+
this.crbmSource = wired.source instanceof CrbmStorageChunkSource2 ? wired.source : void 0;
|
|
1112
1107
|
this.clock = clock;
|
|
1113
1108
|
this.rng = rng;
|
|
1114
1109
|
this.metrics = metrics;
|
|
1115
|
-
this.storageDriver =
|
|
1116
|
-
this.registry =
|
|
1110
|
+
this.storageDriver = wired.driver;
|
|
1111
|
+
this.registry = wired.registry;
|
|
1117
1112
|
this.keystore = options.encryption?.keystore;
|
|
1118
1113
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1119
1114
|
}
|
|
@@ -1148,26 +1143,28 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1148
1143
|
}
|
|
1149
1144
|
/**
|
|
1150
1145
|
* Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). The options
|
|
1151
|
-
* are `{ namespace }` and nothing else:
|
|
1152
|
-
* misspelt `namespace` would otherwise address the default namespace
|
|
1153
|
-
*
|
|
1154
|
-
* `undefined` is read as absent, so a
|
|
1146
|
+
* are a plain object holding `{ namespace }` and nothing else: any other own key, enumerable or not, is refused with
|
|
1147
|
+
* {@link ValidationError}, by name, since a misspelt `namespace` would otherwise address the default namespace, and a
|
|
1148
|
+
* deadline such as `expiresAt` would be left unread. A class instance, or an object built on another, is refused too,
|
|
1149
|
+
* since it can carry an option no scan of its own keys sees. A key whose value is `undefined` is read as absent, so a
|
|
1150
|
+
* spread of options keeps working.
|
|
1155
1151
|
*/
|
|
1156
1152
|
segment(name, options) {
|
|
1157
1153
|
if (options !== void 0 && options !== null) {
|
|
1158
1154
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1159
|
-
throw new
|
|
1155
|
+
throw new ValidationError4("segment: options must be an object such as { namespace }");
|
|
1160
1156
|
}
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1157
|
+
const proto = Object.getPrototypeOf(options);
|
|
1158
|
+
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1159
|
+
throw new ValidationError4(
|
|
1160
|
+
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1164
1161
|
);
|
|
1165
1162
|
}
|
|
1166
|
-
const unknown = Object.
|
|
1163
|
+
const unknown = Object.getOwnPropertyNames(options).filter(
|
|
1167
1164
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1168
1165
|
);
|
|
1169
1166
|
if (unknown.length > 0) {
|
|
1170
|
-
throw new
|
|
1167
|
+
throw new ValidationError4(
|
|
1171
1168
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1172
1169
|
);
|
|
1173
1170
|
}
|
|
@@ -1197,13 +1194,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1197
1194
|
* of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
|
|
1198
1195
|
* everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
|
|
1199
1196
|
* guard runs while the old generation is still authoritative, the publish is fenced as above, and a refused
|
|
1200
|
-
* object is reclaimed only when a re-read finds the row unchanged (the same token
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
* the
|
|
1206
|
-
*
|
|
1197
|
+
* object is reclaimed only when a re-read finds the row unchanged (the same token, or one that differs only in
|
|
1198
|
+
* its leases), `destroyed`, or gone, or finds key material on it when the load wrote cleartext; under any other
|
|
1199
|
+
* row it is kept (hard invariant 1: deleting it after a purge-and-recreate would put a live row over a missing
|
|
1200
|
+
* generation).
|
|
1201
|
+
*
|
|
1202
|
+
* Where the row is gone, or keyed over a cleartext write, the object's footer must match what the load wrote, and
|
|
1203
|
+
* the delete passes the version that read reported, so on a storage driver that reports `conditionalDelete` an
|
|
1204
|
+
* object put under the number since is kept. Where the row is unchanged or `destroyed`, the check narrows the
|
|
1205
|
+
* window rather than closing it: the row read and the delete are two round trips, and the delete is by number,
|
|
1206
|
+
* since the write reports no version of the object it made. The other deletes a load leads to are by number too: its
|
|
1207
|
+
* collection deletes what a listing, or the row's list of kept generations, names below the pointer, and a drop's
|
|
1208
|
+
* sweep and a tombstoned segment's collection delete what a listing names under a `destroyed` row. The failure the
|
|
1209
|
+
* checks leave when they refuse is an orphan object, which costs storage until something collects it —
|
|
1210
|
+
* deliberately the cheaper side of the trade.
|
|
1207
1211
|
*
|
|
1208
1212
|
* Written and published in one step, with no guard, an empty combine — a typo'd operand, an `exclude` that
|
|
1209
1213
|
* swallowed everything, an operand that had not loaded yet — would silently replace `dest` with an empty
|
|
@@ -1250,9 +1254,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1250
1254
|
}
|
|
1251
1255
|
const reason = result.reason;
|
|
1252
1256
|
if (reason === "superseded") {
|
|
1253
|
-
const wrote = result.size > 0 ? `generation ${result.generation} was written and
|
|
1257
|
+
const wrote = result.size > 0 ? `generation ${result.generation} was written and may not be current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
1254
1258
|
throw new WriteConflictError(
|
|
1255
|
-
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result
|
|
1259
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result may not be the destination's current generation: ${wrote}. The pointer may have moved, the row may have been created or rewritten (a retention policy does either), dropped or purged. Re-read the destination and re-run.`
|
|
1256
1260
|
);
|
|
1257
1261
|
}
|
|
1258
1262
|
return {
|
|
@@ -1269,7 +1273,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1269
1273
|
/** The one body of the forms above. */
|
|
1270
1274
|
async materializeMany(options) {
|
|
1271
1275
|
if (typeof options !== "object" || options === null) {
|
|
1272
|
-
throw new
|
|
1276
|
+
throw new ValidationError4(
|
|
1273
1277
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1274
1278
|
);
|
|
1275
1279
|
}
|
|
@@ -1278,13 +1282,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1278
1282
|
(k) => !MATERIALIZE_MANY_KEYS.includes(k)
|
|
1279
1283
|
);
|
|
1280
1284
|
if (unknown.length > 0) {
|
|
1281
|
-
throw new
|
|
1285
|
+
throw new ValidationError4(
|
|
1282
1286
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1283
1287
|
);
|
|
1284
1288
|
}
|
|
1285
1289
|
const dryRunIn = options.dryRun;
|
|
1286
1290
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1287
|
-
throw new
|
|
1291
|
+
throw new ValidationError4("materializeMany: dryRun must be a boolean");
|
|
1288
1292
|
}
|
|
1289
1293
|
const dryRun = dryRunIn === true;
|
|
1290
1294
|
const operandsIn = options.operands;
|
|
@@ -1296,7 +1300,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1296
1300
|
const feedIn = options.feed;
|
|
1297
1301
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1298
1302
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1299
|
-
throw new
|
|
1303
|
+
throw new ValidationError4(
|
|
1300
1304
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1301
1305
|
);
|
|
1302
1306
|
}
|
|
@@ -1306,22 +1310,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1306
1310
|
const budgetOption = options.budget;
|
|
1307
1311
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1308
1312
|
if (typeof pinOption !== "boolean")
|
|
1309
|
-
throw new
|
|
1313
|
+
throw new ValidationError4("materializeMany: pin must be a boolean");
|
|
1310
1314
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1311
|
-
throw new
|
|
1315
|
+
throw new ValidationError4("materializeMany: allowAbsentOperands must be a boolean");
|
|
1312
1316
|
}
|
|
1313
1317
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1314
|
-
throw new
|
|
1318
|
+
throw new ValidationError4("materializeMany: operands must be an object of name to segment");
|
|
1315
1319
|
}
|
|
1316
1320
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1317
|
-
throw new
|
|
1321
|
+
throw new ValidationError4("materializeMany: outputs must be a non-empty array");
|
|
1318
1322
|
}
|
|
1319
1323
|
const names = Object.keys(operandsIn);
|
|
1320
1324
|
const handles = /* @__PURE__ */ new Map();
|
|
1321
1325
|
for (const name of names) {
|
|
1322
1326
|
const handle = operandsIn[name];
|
|
1323
1327
|
if (!(handle instanceof Segment)) {
|
|
1324
|
-
throw new
|
|
1328
|
+
throw new ValidationError4(
|
|
1325
1329
|
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1326
1330
|
);
|
|
1327
1331
|
}
|
|
@@ -1329,14 +1333,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1329
1333
|
}
|
|
1330
1334
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1331
1335
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1332
|
-
throw new
|
|
1336
|
+
throw new ValidationError4(
|
|
1333
1337
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1334
1338
|
);
|
|
1335
1339
|
}
|
|
1336
1340
|
if (mayBeEmptyIn.length > 0) {
|
|
1337
1341
|
const first = mayBeEmptyIn[0];
|
|
1338
1342
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1339
|
-
throw new
|
|
1343
|
+
throw new ValidationError4(
|
|
1340
1344
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1341
1345
|
);
|
|
1342
1346
|
}
|
|
@@ -1346,28 +1350,28 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1346
1350
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1347
1351
|
outputsIn.forEach((raw, i) => {
|
|
1348
1352
|
if (typeof raw !== "object" || raw === null) {
|
|
1349
|
-
throw new
|
|
1353
|
+
throw new ValidationError4(`outputs[${i}] must be an object`);
|
|
1350
1354
|
}
|
|
1351
1355
|
const bad = Object.keys(raw).filter(
|
|
1352
1356
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1353
1357
|
);
|
|
1354
1358
|
if (bad.length > 0) {
|
|
1355
|
-
throw new
|
|
1359
|
+
throw new ValidationError4(
|
|
1356
1360
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1357
1361
|
);
|
|
1358
1362
|
}
|
|
1359
1363
|
const spec = raw;
|
|
1360
1364
|
const dest = spec.dest;
|
|
1361
1365
|
if (!(dest instanceof Segment)) {
|
|
1362
|
-
throw new
|
|
1366
|
+
throw new ValidationError4(`outputs[${i}].dest must be a segment from store.segment()`);
|
|
1363
1367
|
}
|
|
1364
1368
|
if (operandKeys.has(dest.key())) {
|
|
1365
|
-
throw new
|
|
1369
|
+
throw new ValidationError4(
|
|
1366
1370
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1367
1371
|
);
|
|
1368
1372
|
}
|
|
1369
1373
|
if (destKeys.has(dest.key())) {
|
|
1370
|
-
throw new
|
|
1374
|
+
throw new ValidationError4(
|
|
1371
1375
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1372
1376
|
);
|
|
1373
1377
|
}
|
|
@@ -1387,7 +1391,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1387
1391
|
...pinnedAt === void 0 ? {} : {
|
|
1388
1392
|
pinnedGeneration: pinnedAt.generation,
|
|
1389
1393
|
pinnedVersion: pinnedAt.version,
|
|
1390
|
-
pinnedFingerprint: pinnedAt.fingerprint
|
|
1394
|
+
pinnedFingerprint: pinnedAt.fingerprint
|
|
1391
1395
|
},
|
|
1392
1396
|
current: (read) => this.currentRowOf(seg.ref, read?.fingerprint === true)
|
|
1393
1397
|
};
|
|
@@ -1440,13 +1444,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1440
1444
|
held.set(key, { at, handle });
|
|
1441
1445
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1442
1446
|
if (!samePin(prior.at, at)) {
|
|
1443
|
-
throw new
|
|
1447
|
+
throw new ValidationError4(
|
|
1444
1448
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1445
1449
|
);
|
|
1446
1450
|
}
|
|
1447
1451
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1448
1452
|
if (!pinOption) {
|
|
1449
|
-
throw new
|
|
1453
|
+
throw new ValidationError4(
|
|
1450
1454
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1451
1455
|
);
|
|
1452
1456
|
}
|
|
@@ -1460,7 +1464,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1460
1464
|
else if (pinOption) toPin.push([key, handle]);
|
|
1461
1465
|
else pins.set(key, void 0);
|
|
1462
1466
|
}
|
|
1463
|
-
const metricsOn = this.metrics !==
|
|
1467
|
+
const metricsOn = this.metrics !== NOOP_METRICS2;
|
|
1464
1468
|
const work = async () => {
|
|
1465
1469
|
const taken = await mapWithConcurrency(
|
|
1466
1470
|
toPin,
|
|
@@ -1470,7 +1474,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1470
1474
|
for (const [key, at] of taken) pins.set(key, at);
|
|
1471
1475
|
const live = new Map([...pins].filter((e) => e[1] !== void 0));
|
|
1472
1476
|
const pinnedSource = new PinnedStorageChunkSource(crbm, live);
|
|
1473
|
-
const source = this.retryOptions === void 0 ? pinnedSource : new
|
|
1477
|
+
const source = this.retryOptions === void 0 ? pinnedSource : new RetryingStorageChunkSource2(pinnedSource, this.retryOptions);
|
|
1474
1478
|
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1475
1479
|
return runCombineMany(final, {
|
|
1476
1480
|
source,
|
|
@@ -1534,7 +1538,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1534
1538
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1535
1539
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1536
1540
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1537
|
-
throw new
|
|
1541
|
+
throw new ValidationError4(
|
|
1538
1542
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1539
1543
|
);
|
|
1540
1544
|
}
|
|
@@ -1568,7 +1572,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1568
1572
|
}
|
|
1569
1573
|
}
|
|
1570
1574
|
const row = await this.withRetries(() => registry.get(ref));
|
|
1571
|
-
return row === null ? null : { generation: row.currentGen, token: row
|
|
1575
|
+
return row === null ? null : { generation: row.currentGen, token: pointerIdOf(row) };
|
|
1572
1576
|
}
|
|
1573
1577
|
/** Throw what a read of, or a publish to, `handle` must throw now: its lease error, if it has one. */
|
|
1574
1578
|
assertLease(handle) {
|
|
@@ -1613,7 +1617,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1613
1617
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1614
1618
|
if (this.crbmSource !== void 0) {
|
|
1615
1619
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1616
|
-
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${
|
|
1620
|
+
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1617
1621
|
if (rowVersionOfLive(held) !== listed) this.engine.invalidate(ref);
|
|
1618
1622
|
}
|
|
1619
1623
|
return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
@@ -1633,10 +1637,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1633
1637
|
* can still hold it, and so can one above the pointer after a {@link CloudRoaring.rollback}, which the rollback
|
|
1634
1638
|
* could make current again. Each holder is deleted — below the pointer by a `keep: 0` collection, above it one
|
|
1635
1639
|
* by one, re-proved against the row first — and the generations above the pointer that never held the id stay
|
|
1636
|
-
* as rollback targets
|
|
1637
|
-
*
|
|
1638
|
-
*
|
|
1639
|
-
*
|
|
1640
|
+
* as rollback targets, but for any object no read of the segment can open, which goes whatever the id. An entry
|
|
1641
|
+
* says `erased: true` only once the call has listed the bucket and read what is left: **no generation of the
|
|
1642
|
+
* segment holds the id**. The returned per-segment record is your **erasure ledger** —
|
|
1643
|
+
* persist it / route it to your audit sink as the proof of deletion (when you pass `audit`, an audit event is also
|
|
1644
|
+
* emitted per segment erased: `segment.rewrite` for a rewrite, `segment.collect` where only other generations held
|
|
1645
|
+
* the id and nothing was rewritten). A segment where the erasure deleted only objects no read of it can open (a first
|
|
1646
|
+
* load's sealed under a key no row holds) found the id in none, so the ledger does not list it; the audit sink gets
|
|
1647
|
+
* `segment.collect` for it with no `fromGeneration`, which is the only record of that deletion.
|
|
1640
1648
|
*
|
|
1641
1649
|
* Uses the backend's **own** two halves, so the membership check and the rewrite provably run
|
|
1642
1650
|
* over the same generation. Requires the store built with a **backend** (throws
|
|
@@ -1658,13 +1666,21 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1658
1666
|
* an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
|
|
1659
1667
|
* that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger. A tombstoned
|
|
1660
1668
|
* segment is still searched: a cleartext object left under it that holds the id is deleted, with everything under
|
|
1661
|
-
* the tombstone, and the entry reads `erased: true`.
|
|
1669
|
+
* the tombstone, and the entry reads `erased: true`. So is a segment whose row names no generation yet (a retention
|
|
1670
|
+
* policy set before its first load): a first load's object that holds the id is deleted, once the row's `pointerId`
|
|
1671
|
+
* is renewed so that the load, if still running, is refused at its publish, and the entry reads `erased: true`. An
|
|
1672
|
+
* encrypted first load's object there cannot be searched, so it is deleted whatever the id: on an encrypted store,
|
|
1673
|
+
* an erasure of any id refuses an encrypted first load in flight that has written its object onto a segment made
|
|
1674
|
+
* ahead of its data. Each such delete, and each of a holder above the pointer, names the object the erasure read:
|
|
1675
|
+
* on a storage driver that reports `conditionalDelete`, another erasure of the id that deletes it first, followed by
|
|
1676
|
+
* a load that takes the number freed and publishes, does not lose that load's generation to this erasure's delete,
|
|
1677
|
+
* which is refused, and the entry says what the row says; on one that does not, it can.
|
|
1662
1678
|
*
|
|
1663
1679
|
* **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
|
|
1664
|
-
* another writer (a load, another erasure, or a rollback) moved the pointer
|
|
1665
|
-
* erase the id. Re-run: it erases the id if it is still there, and lists
|
|
1666
|
-
* erasure of the same id already removed it. Do not read `'superseded'` as
|
|
1667
|
-
* read it as "not done by this call, and the re-run settles it".
|
|
1680
|
+
* another writer (a load, another erasure, or a rollback) moved the pointer, or replaced an object the call meant to
|
|
1681
|
+
* delete, mid-call, so **this call** did not erase the id. Re-run: it erases the id if it is still there, and lists
|
|
1682
|
+
* nothing for the segment if a racing erasure of the same id already removed it. Do not read `'superseded'` as
|
|
1683
|
+
* "the id is still present" — read it as "not done by this call, and the re-run settles it".
|
|
1668
1684
|
* `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
|
|
1669
1685
|
* on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
|
|
1670
1686
|
* it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
|
|
@@ -1760,6 +1776,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1760
1776
|
* What the bucket holds, not what the segment has ever been — collection deletes superseded objects, so this is
|
|
1761
1777
|
* the grace window plus whatever has not been collected yet. It is the set {@link CloudRoaring.rollback} can
|
|
1762
1778
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1779
|
+
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1780
|
+
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
1763
1781
|
*
|
|
1764
1782
|
* Needs a backend.
|
|
1765
1783
|
*/
|
|
@@ -1844,35 +1862,58 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1844
1862
|
* (`segment.rollback`), because every other pointer move can be reconstructed from "a load happened" and this
|
|
1845
1863
|
* one cannot.
|
|
1846
1864
|
*
|
|
1847
|
-
* It refuses rather than guesses
|
|
1848
|
-
*
|
|
1849
|
-
*
|
|
1850
|
-
*
|
|
1851
|
-
*
|
|
1865
|
+
* It refuses rather than guesses. The row records, in `knownGens`, the generations its loads and rollbacks made
|
|
1866
|
+
* current, each with the footer checksum of its object, and a rollback moves only onto one of those: a generation the
|
|
1867
|
+
* row does not know (a load that wrote its object and died before the publish, a refused load's object, an object
|
|
1868
|
+
* sealed under a key no row holds, a load published after the point a registry was restored to) throws
|
|
1869
|
+
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1870
|
+
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1871
|
+
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1872
|
+
* crypto-shredded segment throws {@link ValidationError} because every generation of it is unreadable, and a target
|
|
1873
|
+
* that is cleartext under an encrypted segment, or encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1874
|
+
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1875
|
+
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1876
|
+
* gone it throws `NotFoundError` naming what is.
|
|
1852
1877
|
*
|
|
1853
1878
|
* The same tail read (and a range read when the target's index is longer than it) gives the target's id count and
|
|
1854
1879
|
* metadata, which the rollback writes into the row with the pointer, so a reader that sees the target as current sees
|
|
1855
1880
|
* what describes it. The store's keystore opens an encrypted target for this; without it, or when it cannot open the
|
|
1856
|
-
* key
|
|
1881
|
+
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1882
|
+
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1883
|
+
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1884
|
+
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one that landed, the row being the
|
|
1885
|
+
* one the swap leaves, is checked as any is, and otherwise its `TransientError` is thrown. After the swap the target's
|
|
1886
|
+
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1857
1887
|
*
|
|
1858
1888
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
1859
|
-
* and are then *above* `currentGen`, where collection never looks
|
|
1860
|
-
*
|
|
1861
|
-
* below its pointer
|
|
1862
|
-
*
|
|
1863
|
-
*
|
|
1889
|
+
* and are then *above* `currentGen`, where collection never looks, and still known to the row, so rolling forward onto
|
|
1890
|
+
* one needs no flag. They remain until loads pass them (the first load whose
|
|
1891
|
+
* number one of them holds numbers above them all, and collection then keeps `keep` of what is below its pointer, the
|
|
1892
|
+
* generation that load superseded first), {@link CloudRoaring.dropSegment}
|
|
1893
|
+
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and, when the
|
|
1894
|
+
* current generation does not hold the id, only those holding it and any no read of the segment can open, whatever
|
|
1895
|
+
* the id. An operator who has just undone a bad load should not have the
|
|
1864
1896
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
1865
1897
|
* erasure undoable.
|
|
1866
1898
|
*
|
|
1867
|
-
*
|
|
1868
|
-
*
|
|
1869
|
-
*
|
|
1899
|
+
* `{ allowUnknown: true }` moves onto a present object the row does not vouch for, or one whose checksum differs from
|
|
1900
|
+
* the recorded one, as long as it is the segment's kind of object, opens, and names the generation, and records it as
|
|
1901
|
+
* known (over a usable record; a row with none stays without one). On an encrypted segment, outside the two cases the pointer alone decides (below), it also needs the key: with no keystore, or one that holds none of the key's KEKs or
|
|
1902
|
+
* one that does not unwrap it, it is refused with `KeyUnavailableError` before the target is read, since an object
|
|
1903
|
+
* sealed under a key no row holds passes every check that needs no key, and a key service that did not answer in
|
|
1904
|
+
* time is thrown as `TransientError`, to retry. Every move the row did not vouch for is audited with `unknown: true`.
|
|
1905
|
+
* Two cases are decided by the pointer alone: a target below it needs no flag, no key and no checksum check, and the
|
|
1906
|
+
* move is audited the same way. They are a row with no usable `knownGens` (a registry that drops the field, or a pointer
|
|
1907
|
+
* moved by hand), and a target numbered below the bound of a full record (128 entries or more, leased ones included),
|
|
1908
|
+
* which is its least recently current entry that no lease names (an `*Into` destination keeps every generation in the
|
|
1909
|
+
* bucket and names only the newest 128). A generation a publish with a smaller `keep` no longer names but that stays in
|
|
1910
|
+
* the bucket (a lease spared it, or its delete failed) needs `allowUnknown` until a later listing collects it.
|
|
1870
1911
|
*
|
|
1871
1912
|
* Needs a backend.
|
|
1872
1913
|
*/
|
|
1873
1914
|
async rollback(ref, toGeneration, options = {}) {
|
|
1874
1915
|
validateSegmentRef(ref);
|
|
1875
|
-
checkOptions(options, "rollback", CALL_KEYS.rollback, ["
|
|
1916
|
+
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1876
1917
|
options ??= {};
|
|
1877
1918
|
const deps = this.lifecycleDeps("rollback");
|
|
1878
1919
|
try {
|
|
@@ -1929,10 +1970,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1929
1970
|
* published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
|
|
1930
1971
|
*
|
|
1931
1972
|
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
1932
|
-
* collection deliberately never looks — but only while the segment's registry row is unchanged or
|
|
1933
|
-
*
|
|
1934
|
-
*
|
|
1935
|
-
*
|
|
1973
|
+
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only in
|
|
1974
|
+
* its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1975
|
+
* encrypted segment's bucket. Under a row that is gone or keyed, one read of the object's footer must match what the
|
|
1976
|
+
* load wrote first. Once another write has otherwise changed the row, the generation number it holds may name another
|
|
1977
|
+
* writer's object, and a footer names bytes, not the load that wrote them, so it leaves the orphan rather than risk
|
|
1978
|
+
* deleting live data. A publish never adds the orphan to the row's list, and a load's listing deletes it once a
|
|
1979
|
+
* generation above it is current, unless the row's list names it; where the row records none (after a rollback, or
|
|
1980
|
+
* with a `keep` above 64), that listing's window can keep it and, at a `keep` up to 64, record it, and the publishes
|
|
1981
|
+
* after carry it until the window pushes it out. Until it goes, a rollback onto it is refused unless `allowUnknown` is passed, since the list never names it (a list seeded
|
|
1982
|
+
* over a row with none can); with the override it publishes what no load published.
|
|
1936
1983
|
*
|
|
1937
1984
|
* **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
|
|
1938
1985
|
* that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
|
|
@@ -2023,7 +2070,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2023
2070
|
async dropSegment(ref, options) {
|
|
2024
2071
|
validateSegmentRef(ref);
|
|
2025
2072
|
if (options === void 0 || options === null) {
|
|
2026
|
-
throw new
|
|
2073
|
+
throw new ValidationError4("dropSegment: options { confirmSegment } are required");
|
|
2027
2074
|
}
|
|
2028
2075
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2029
2076
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2145,8 +2192,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2145
2192
|
* Pass `purgeTombstones: false` to keep every tombstone, that row included. On a backend whose registry reports
|
|
2146
2193
|
* `conditionalDelete` (AWS S3, Azure Blob, the local filesystem and memory, by default; GCS when you set `conditionalDelete: true`)
|
|
2147
2194
|
* the purge removes the row from the bucket for good, by a delete the store applies only to the version it judged,
|
|
2148
|
-
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere
|
|
2149
|
-
*
|
|
2195
|
+
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere it
|
|
2196
|
+
* leaves a tombstone. Each retirement files a pointer in the due index
|
|
2150
2197
|
* under the day its tombstone's grace ends, so `scan: 'index'` purges as well as retires.
|
|
2151
2198
|
*
|
|
2152
2199
|
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
@@ -2191,7 +2238,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2191
2238
|
* reader converging, you need to signal them — this is the call to make when your own fan-out delivers.
|
|
2192
2239
|
*
|
|
2193
2240
|
* It also forgets any replacement this store found of a pin's object, so once it is called after a restore puts that
|
|
2194
|
-
* object back, the pin reads it again.
|
|
2241
|
+
* object back, the pin reads it again. A read of the segment already running on this store, a pin's and a combine's
|
|
2242
|
+
* with a pinned operand included, caches nothing more of what it began reading before the call, and a pinned one
|
|
2243
|
+
* opens its object again before its next chunk, failing with `NotFoundError` if the object is gone or replaced, or
|
|
2244
|
+
* its row is gone or destroyed.
|
|
2195
2245
|
*
|
|
2196
2246
|
* Synchronous, best-effort, and safe to call for a segment this store has never read.
|
|
2197
2247
|
*
|
|
@@ -2211,7 +2261,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2211
2261
|
*
|
|
2212
2262
|
* The pinned handle gets its own engine but **shares the store's chunk cache**, under keys of its own: the
|
|
2213
2263
|
* pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
|
|
2214
|
-
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
2264
|
+
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}). It
|
|
2265
|
+
* shares the store engine's invalidations too, so the store's writes reach a pinned read already running.
|
|
2215
2266
|
*/
|
|
2216
2267
|
async pinSegment(ref, leaseUntil, named) {
|
|
2217
2268
|
const crbm = this.crbmSource;
|
|
@@ -2290,7 +2341,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2290
2341
|
* An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
|
|
2291
2342
|
* retries, so a transient fault on a pinned read is retried as it would be on a live one.
|
|
2292
2343
|
*
|
|
2293
|
-
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment
|
|
2344
|
+
* It **shares the store's chunk cache and its memory bound**, under keys of its own for each pinned segment, and the
|
|
2345
|
+
* store engine's invalidations, so an invalidation of a pinned segment reaches a read of the pin already running.
|
|
2294
2346
|
*/
|
|
2295
2347
|
engineWithPins(pins) {
|
|
2296
2348
|
const crbm = this.crbmSource;
|
|
@@ -2301,12 +2353,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2301
2353
|
}
|
|
2302
2354
|
const pinned = new PinnedStorageChunkSource(crbm, pins);
|
|
2303
2355
|
return new SegmentEngine({
|
|
2304
|
-
storage: this.retryOptions === void 0 ? pinned : new
|
|
2356
|
+
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource2(pinned, this.retryOptions),
|
|
2305
2357
|
cache: this.cache,
|
|
2306
2358
|
codec: roaringCodec,
|
|
2307
2359
|
clock: this.clock,
|
|
2308
2360
|
metrics: this.metrics,
|
|
2309
|
-
budget: this.budget
|
|
2361
|
+
budget: this.budget,
|
|
2362
|
+
sharesInvalidationsWith: this.engine
|
|
2310
2363
|
});
|
|
2311
2364
|
}
|
|
2312
2365
|
/**
|
|
@@ -2332,7 +2385,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2332
2385
|
if (held.has(key)) {
|
|
2333
2386
|
const before = held.get(key);
|
|
2334
2387
|
if (!samePin(before, at)) {
|
|
2335
|
-
throw new
|
|
2388
|
+
throw new ValidationError4(
|
|
2336
2389
|
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`intersectInto(dest, [])\``
|
|
2337
2390
|
);
|
|
2338
2391
|
}
|
|
@@ -2346,7 +2399,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2346
2399
|
/** `op` under the store's retries, as the store's own reads run; bare when the store has retries off. */
|
|
2347
2400
|
withRetries(op) {
|
|
2348
2401
|
const r = this.retryOptions;
|
|
2349
|
-
return r === void 0 ? op() : withRetry(op, r.policy ??
|
|
2402
|
+
return r === void 0 ? op() : withRetry(op, r.policy ?? DEFAULT_RETRY_POLICY2, r);
|
|
2350
2403
|
}
|
|
2351
2404
|
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
2352
2405
|
requireRegistry(op) {
|
|
@@ -2370,17 +2423,27 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2370
2423
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2371
2424
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
2372
2425
|
* `summaries: true` it also opens each segment's current object (one tail read each) and reports a segment whose
|
|
2373
|
-
* row's summary says another id count or metadata than the object holds
|
|
2426
|
+
* row's summary names another object by its fingerprint, or says another id count or metadata than the object holds,
|
|
2427
|
+
* as `summary-mismatch`, which is what a
|
|
2374
2428
|
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
2375
2429
|
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
2376
2430
|
*
|
|
2431
|
+
* With `generations: true` it also holds every generation a row's ledger of known generations names, the current one
|
|
2432
|
+
* included, against its object: one footer read each (104 bytes, no key, not retried), reporting
|
|
2433
|
+
* `known-generation-mismatch` with the `generation` where the object under that number is not the one the row
|
|
2434
|
+
* recorded. A named generation that is not in the bucket is not reported. A row with a pointer whose ledger cannot
|
|
2435
|
+
* be used is counted in `generationsUnchecked`, and none of its objects is read.
|
|
2436
|
+
*
|
|
2377
2437
|
* It holds the registry rows it enumerates resident, at most 250,000 of them, and throws
|
|
2378
2438
|
* `BudgetExceededError` past that rather than report a partial scan as a whole one. This method takes no
|
|
2379
2439
|
* ceiling of its own: narrow the scan with `namespace`, or call `runConsistencyCheck` over the backend's
|
|
2380
2440
|
* `storage` and `registry` with a higher `maxScanSegments`.
|
|
2381
2441
|
*/
|
|
2382
2442
|
async checkConsistency(options = {}) {
|
|
2383
|
-
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2443
|
+
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, [
|
|
2444
|
+
"summaries",
|
|
2445
|
+
"generations"
|
|
2446
|
+
]);
|
|
2384
2447
|
options ??= {};
|
|
2385
2448
|
const deps = this.lifecycleDeps("checkConsistency");
|
|
2386
2449
|
return runConsistencyCheck(
|
|
@@ -2406,24 +2469,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2406
2469
|
const registry = this.requireRegistry("exportSegments");
|
|
2407
2470
|
return runExport(this, registry, sink, { ...opts, codec: opts.codec ?? roaringCodec });
|
|
2408
2471
|
}
|
|
2409
|
-
/**
|
|
2410
|
-
* Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
|
|
2411
|
-
* from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
2412
|
-
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
2413
|
-
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
2414
|
-
*/
|
|
2415
|
-
static estimateCost(input) {
|
|
2416
|
-
return estimateCost(input);
|
|
2417
|
-
}
|
|
2418
2472
|
};
|
|
2419
2473
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
2420
|
-
var
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2474
|
+
var refusing = (err) => (
|
|
2475
|
+
// eslint-disable-next-line require-yield
|
|
2476
|
+
async function* () {
|
|
2477
|
+
throw err;
|
|
2478
|
+
}
|
|
2479
|
+
);
|
|
2480
|
+
var failing = (err) => withBatches(refusing(err)(), refusing(err));
|
|
2427
2481
|
var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
|
|
2428
2482
|
var twoPins = (a, b) => {
|
|
2429
2483
|
const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
|
|
@@ -2441,9 +2495,12 @@ var readOptions = (options) => ({
|
|
|
2441
2495
|
});
|
|
2442
2496
|
var NO_SEGMENTS = [];
|
|
2443
2497
|
function segmentList(list, what) {
|
|
2444
|
-
if (!Array.isArray(list)) throw new
|
|
2498
|
+
if (!Array.isArray(list)) throw new ValidationError4(`${what} must be an array of segments`);
|
|
2445
2499
|
return list;
|
|
2446
2500
|
}
|
|
2501
|
+
function operandsLabel(verb) {
|
|
2502
|
+
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2503
|
+
}
|
|
2447
2504
|
var AS_IDS = {
|
|
2448
2505
|
failing,
|
|
2449
2506
|
guard: guardIds,
|
|
@@ -2458,10 +2515,8 @@ var CombineChunks = class {
|
|
|
2458
2515
|
chunks;
|
|
2459
2516
|
};
|
|
2460
2517
|
var AS_CHUNKS = {
|
|
2461
|
-
guard:
|
|
2462
|
-
failing: (err) => (
|
|
2463
|
-
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
|
|
2464
|
-
}),
|
|
2518
|
+
guard: guardItems,
|
|
2519
|
+
failing: (err) => refusing(err)(),
|
|
2465
2520
|
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2466
2521
|
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2467
2522
|
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
@@ -2499,6 +2554,7 @@ var Segment = class _Segment {
|
|
|
2499
2554
|
lease;
|
|
2500
2555
|
leaseHold;
|
|
2501
2556
|
static {
|
|
2557
|
+
Object.defineProperty(_Segment.prototype, SEGMENT_BRAND, { value: true });
|
|
2502
2558
|
makeSegment = (parts) => {
|
|
2503
2559
|
minting = true;
|
|
2504
2560
|
try {
|
|
@@ -2519,7 +2575,7 @@ var Segment = class _Segment {
|
|
|
2519
2575
|
*/
|
|
2520
2576
|
constructor(parts) {
|
|
2521
2577
|
if (!minting) {
|
|
2522
|
-
throw new
|
|
2578
|
+
throw new ValidationError4(
|
|
2523
2579
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2524
2580
|
);
|
|
2525
2581
|
}
|
|
@@ -2535,7 +2591,7 @@ var Segment = class _Segment {
|
|
|
2535
2591
|
if (parts.lease !== void 0) {
|
|
2536
2592
|
this.lease = Object.freeze({ holder: parts.lease.holder, until: parts.lease.until });
|
|
2537
2593
|
}
|
|
2538
|
-
this.metricsOn = parts.metrics !==
|
|
2594
|
+
this.metricsOn = parts.metrics !== NOOP_METRICS2;
|
|
2539
2595
|
}
|
|
2540
2596
|
/**
|
|
2541
2597
|
* **Hold this segment at the generation that is current right now**, for as long as you keep the handle.
|
|
@@ -2590,11 +2646,16 @@ var Segment = class _Segment {
|
|
|
2590
2646
|
* - `eraseSubject` invalidates a pin of each segment it scans that is not already destroyed.
|
|
2591
2647
|
*
|
|
2592
2648
|
* An invalidated pin opens its object again, and fails if that object is gone or replaced, or its row is gone or
|
|
2593
|
-
* destroyed.
|
|
2649
|
+
* destroyed. A read of the pin already running when the invalidation comes, a combine on this store with the pin as
|
|
2650
|
+
* an operand included, looks the pin up again before it hands out its next chunk: it carries on with what it holds
|
|
2651
|
+
* where the object is still there, and fails with `NotFoundError` where it is gone. It caches nothing more of what it
|
|
2652
|
+
* began reading before the invalidation, so the pin's next read opens the object again too.
|
|
2594
2653
|
* Anything else leaves the pin as it is: after a `destroySegment` beside this store, or an erasure, a drop or a
|
|
2595
2654
|
* retirement through another store, in this process or another, the pin answers from what it holds until this
|
|
2596
2655
|
* store's reader cache evicts the pin's reader and its chunk cache evicts the chunks the pin decoded, or
|
|
2597
|
-
* {@link CloudRoaring.invalidate} is called on this store.
|
|
2656
|
+
* {@link CloudRoaring.invalidate} is called on this store. A combine on another store that takes the pin as an
|
|
2657
|
+
* operand reads it through that store's reader and chunk cache, which only that store's own writes and its
|
|
2658
|
+
* `invalidate` reach. Where the
|
|
2598
2659
|
* object the pin reads has been deleted, by an erasure, a drop or a sweep, a chunk the pin has not cached fails at
|
|
2599
2660
|
* once. No timed refresh bounds any of that, as none bounds anything else a pin holds.
|
|
2600
2661
|
*
|
|
@@ -2617,7 +2678,7 @@ var Segment = class _Segment {
|
|
|
2617
2678
|
* {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
|
|
2618
2679
|
*/
|
|
2619
2680
|
async pin(options) {
|
|
2620
|
-
this.
|
|
2681
|
+
this.assertLease();
|
|
2621
2682
|
const leaseUntil = this.leaseUntilOf("pin", options);
|
|
2622
2683
|
return this.pinned(this.ref, leaseUntil);
|
|
2623
2684
|
}
|
|
@@ -2629,11 +2690,11 @@ var Segment = class _Segment {
|
|
|
2629
2690
|
leaseUntilOf(op, options) {
|
|
2630
2691
|
if (options === void 0 || options === null) return void 0;
|
|
2631
2692
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2632
|
-
throw new
|
|
2693
|
+
throw new ValidationError4(`${op}: options must be an object such as { leaseUntil }`);
|
|
2633
2694
|
}
|
|
2634
2695
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2635
2696
|
if (unknown.length > 0) {
|
|
2636
|
-
throw new
|
|
2697
|
+
throw new ValidationError4(
|
|
2637
2698
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2638
2699
|
);
|
|
2639
2700
|
}
|
|
@@ -2641,17 +2702,17 @@ var Segment = class _Segment {
|
|
|
2641
2702
|
if (until === void 0) return void 0;
|
|
2642
2703
|
const now = this.clock.now();
|
|
2643
2704
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2644
|
-
throw new
|
|
2705
|
+
throw new ValidationError4(
|
|
2645
2706
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2646
2707
|
);
|
|
2647
2708
|
}
|
|
2648
2709
|
if (until <= now) {
|
|
2649
|
-
throw new
|
|
2710
|
+
throw new ValidationError4(
|
|
2650
2711
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2651
2712
|
);
|
|
2652
2713
|
}
|
|
2653
2714
|
if (until > now + MAX_LEASE_MS) {
|
|
2654
|
-
throw new
|
|
2715
|
+
throw new ValidationError4(
|
|
2655
2716
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2656
2717
|
);
|
|
2657
2718
|
}
|
|
@@ -2707,10 +2768,12 @@ var Segment = class _Segment {
|
|
|
2707
2768
|
*/
|
|
2708
2769
|
assertSegments(handles) {
|
|
2709
2770
|
for (const h of handles) {
|
|
2710
|
-
if (h instanceof _Segment || typeof h === "object" && h !== null &&
|
|
2771
|
+
if (h instanceof _Segment || typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
2711
2772
|
continue;
|
|
2712
2773
|
}
|
|
2713
|
-
throw new
|
|
2774
|
+
throw new ValidationError4(
|
|
2775
|
+
"an operand must be a segment from `store.segment()` of a compatible copy of @cloudbitmaps/roaring; install one version of it"
|
|
2776
|
+
);
|
|
2714
2777
|
}
|
|
2715
2778
|
}
|
|
2716
2779
|
/**
|
|
@@ -2723,6 +2786,11 @@ var Segment = class _Segment {
|
|
|
2723
2786
|
if (err !== void 0) throw err;
|
|
2724
2787
|
}
|
|
2725
2788
|
}
|
|
2789
|
+
/** Throw this handle's lease error, if it has one. The calls that return a value instead of throwing use {@link leaseError}. */
|
|
2790
|
+
assertLease() {
|
|
2791
|
+
const err = this.leaseError();
|
|
2792
|
+
if (err !== void 0) throw err;
|
|
2793
|
+
}
|
|
2726
2794
|
/**
|
|
2727
2795
|
* Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
|
|
2728
2796
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
@@ -2746,16 +2814,16 @@ var Segment = class _Segment {
|
|
|
2746
2814
|
* is in `options`, the second.
|
|
2747
2815
|
*/
|
|
2748
2816
|
async pinAt(at, options) {
|
|
2749
|
-
this.
|
|
2817
|
+
this.assertLease();
|
|
2750
2818
|
const { generation, fingerprint } = at ?? {};
|
|
2751
2819
|
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => k !== "generation" && k !== "fingerprint") : [];
|
|
2752
2820
|
if (unknown.length > 0) {
|
|
2753
|
-
throw new
|
|
2821
|
+
throw new ValidationError4(
|
|
2754
2822
|
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { generation, fingerprint } only`
|
|
2755
2823
|
);
|
|
2756
2824
|
}
|
|
2757
2825
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2758
|
-
throw new
|
|
2826
|
+
throw new ValidationError4(
|
|
2759
2827
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2760
2828
|
);
|
|
2761
2829
|
}
|
|
@@ -2789,8 +2857,9 @@ var Segment = class _Segment {
|
|
|
2789
2857
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
2790
2858
|
}
|
|
2791
2859
|
/**
|
|
2792
|
-
* Time an op with the injected clock and emit an `op` metric on completion (success or throw).
|
|
2793
|
-
*
|
|
2860
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). With no sink wired it
|
|
2861
|
+
* reads no clock and emits nothing, but a call still passes through this async frame; `has()` and `count()` return the
|
|
2862
|
+
* engine's promise directly instead.
|
|
2794
2863
|
*/
|
|
2795
2864
|
async timed(name, fn) {
|
|
2796
2865
|
if (!this.metricsOn) return fn();
|
|
@@ -2801,18 +2870,23 @@ var Segment = class _Segment {
|
|
|
2801
2870
|
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
2802
2871
|
}
|
|
2803
2872
|
}
|
|
2804
|
-
/**
|
|
2873
|
+
/**
|
|
2874
|
+
* Membership: one chunk. A cold resolution of the segment is one registry read and a warm one none; then a cached
|
|
2875
|
+
* chunk answers with no request, and an uncached one is one range read, after a tail read when no reader of the
|
|
2876
|
+
* generation is open, unless that tail carried the chunk. Throws {@link ValidationError} on a bad id.
|
|
2877
|
+
*/
|
|
2805
2878
|
has(id) {
|
|
2806
2879
|
const lease = this.leaseError();
|
|
2807
2880
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2881
|
+
if (!this.metricsOn) return this.engine.has(this.ref, id);
|
|
2808
2882
|
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
2809
2883
|
}
|
|
2810
2884
|
/**
|
|
2811
2885
|
* Cardinality of the generation this handle reads: **one registry read when cold, none when warm, and no read of the
|
|
2812
2886
|
* object**, with **zero payload reads**. The registry row records the id count of the generation it names, written
|
|
2813
2887
|
* by the write that made it current, and a count answers from that. A row with no summary it can use (one written
|
|
2814
|
-
*
|
|
2815
|
-
* `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
2888
|
+
* without it, as a registry of your own may, one that names another generation, a sealed one that does not open)
|
|
2889
|
+
* sends the count to the `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
2816
2890
|
* restore) still counts the row's number, while a read of the object throws; `checkConsistency` is what finds it.
|
|
2817
2891
|
*
|
|
2818
2892
|
* What this trusts: the row's summary, or the index's sum, and no payload is decoded to confirm it. The summary is
|
|
@@ -2828,24 +2902,38 @@ var Segment = class _Segment {
|
|
|
2828
2902
|
count() {
|
|
2829
2903
|
const lease = this.leaseError();
|
|
2830
2904
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2905
|
+
if (!this.metricsOn) return this.engine.count(this.ref);
|
|
2831
2906
|
return this.timed("count", () => this.engine.count(this.ref));
|
|
2832
2907
|
}
|
|
2833
2908
|
/**
|
|
2834
|
-
* What the generation this handle reads is, from one resolution: its number, its id count
|
|
2835
|
-
*
|
|
2836
|
-
*
|
|
2837
|
-
*
|
|
2838
|
-
*
|
|
2839
|
-
*
|
|
2840
|
-
*
|
|
2841
|
-
*
|
|
2909
|
+
* What the generation this handle reads is, from one resolution: its number, its id count, the metadata it was
|
|
2910
|
+
* loaded with (absent when it has none), and `sizeBytes`, its object's bytes in storage. All four come from the
|
|
2911
|
+
* registry row when it has a summary this store can use, as `count()`'s do: the summary records the object's
|
|
2912
|
+
* fingerprint, its size with its checksum. That is **one registry read when cold, none when warm, and no read of the
|
|
2913
|
+
* object**. With no summary it can use, they come from the generation's object, opened once: its footer and index,
|
|
2914
|
+
* with no payload read, which is a tail read of the object more when cold (and a range read for an index longer than
|
|
2915
|
+
* the tail) and keeps the generation in the reader cache as a read does. On a pinned handle it answers for the
|
|
2916
|
+
* generation it pinned, from the pinned object. A segment with no generation answers
|
|
2917
|
+
* `{ generation: null, cardinality: 0, sizeBytes: null }`, and a store whose source cannot report a size answers
|
|
2918
|
+
* `sizeBytes: null`. A pointer that names a missing object (a torn restore) still answers from the row's summary, as
|
|
2919
|
+
* `count()` does, while a read of the object throws `NotFoundError`; with no summary to use, `stat()` throws it too.
|
|
2920
|
+
* `checkConsistency` is what finds such a row.
|
|
2921
|
+
*
|
|
2922
|
+
* Trust is as for `count()`: what the row's summary says is believed until the object is next opened. Every open of
|
|
2923
|
+
* the generation holds the object to the fingerprint the summary records, so another object under the number is
|
|
2924
|
+
* refused when it is opened, and holds its count against the summary's; a disagreement in count makes this process
|
|
2925
|
+
* stop using that summary, so `stat()` then answers from the object, whose index is checked for internal consistency
|
|
2926
|
+
* and whose count is the index's sum. A reader already open is not checked again until the row changes, it is
|
|
2927
|
+
* evicted or it is invalidated.
|
|
2928
|
+
*
|
|
2929
|
+
* `sizeBytes` is what `groundedReport` in `@cloudbitmaps/tools` prices storage from.
|
|
2842
2930
|
*
|
|
2843
2931
|
* ```ts
|
|
2844
|
-
* const { generation, cardinality, metadata } = await store.segment('active-30d').stat();
|
|
2932
|
+
* const { generation, cardinality, metadata, sizeBytes } = await store.segment('active-30d').stat();
|
|
2845
2933
|
* ```
|
|
2846
2934
|
*/
|
|
2847
2935
|
async stat() {
|
|
2848
|
-
this.
|
|
2936
|
+
this.assertLease();
|
|
2849
2937
|
return this.engine.stat(this.ref);
|
|
2850
2938
|
}
|
|
2851
2939
|
/**
|
|
@@ -2875,7 +2963,7 @@ var Segment = class _Segment {
|
|
|
2875
2963
|
engine.iterate(ref, bounds),
|
|
2876
2964
|
() => engine.iterateBatches(ref, bounds)
|
|
2877
2965
|
);
|
|
2878
|
-
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.
|
|
2966
|
+
return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease());
|
|
2879
2967
|
}
|
|
2880
2968
|
/**
|
|
2881
2969
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -2909,22 +2997,22 @@ var Segment = class _Segment {
|
|
|
2909
2997
|
*/
|
|
2910
2998
|
everyNth(n, options) {
|
|
2911
2999
|
const lease = this.leaseError();
|
|
2912
|
-
if (lease !== void 0) return
|
|
3000
|
+
if (lease !== void 0) return refusing(lease)();
|
|
2913
3001
|
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
2914
|
-
if (bad !== void 0) return
|
|
3002
|
+
if (bad !== void 0) return refusing(bad)();
|
|
2915
3003
|
if (this.pinnedAt === void 0) {
|
|
2916
|
-
return
|
|
3004
|
+
return refusing(
|
|
2917
3005
|
new UnsupportedError(
|
|
2918
3006
|
"everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
|
|
2919
3007
|
)
|
|
2920
|
-
);
|
|
3008
|
+
)();
|
|
2921
3009
|
}
|
|
2922
3010
|
const stream = this.engine.everyNth(
|
|
2923
3011
|
this.ref,
|
|
2924
3012
|
n,
|
|
2925
3013
|
options == null ? void 0 : rangeOf(options)
|
|
2926
3014
|
);
|
|
2927
|
-
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.
|
|
3015
|
+
return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease());
|
|
2928
3016
|
}
|
|
2929
3017
|
/**
|
|
2930
3018
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -2964,28 +3052,7 @@ var Segment = class _Segment {
|
|
|
2964
3052
|
intersect(others, options) {
|
|
2965
3053
|
const bad = optionsError(options, "intersect", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
2966
3054
|
if (bad !== void 0) return failing(bad);
|
|
2967
|
-
return this.
|
|
2968
|
-
}
|
|
2969
|
-
intersectAs(out, others, options) {
|
|
2970
|
-
let exclude;
|
|
2971
|
-
let handles;
|
|
2972
|
-
try {
|
|
2973
|
-
exclude = this.excludesOf(options);
|
|
2974
|
-
handles = [this, ...segmentList(others, "intersect: `others`"), ...exclude];
|
|
2975
|
-
this.assertSegments(handles);
|
|
2976
|
-
this.assertLeases(handles);
|
|
2977
|
-
} catch (err) {
|
|
2978
|
-
return out.failing(err);
|
|
2979
|
-
}
|
|
2980
|
-
let engine;
|
|
2981
|
-
try {
|
|
2982
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
2983
|
-
} catch (err) {
|
|
2984
|
-
return out.failing(err);
|
|
2985
|
-
}
|
|
2986
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
2987
|
-
const opts = this.refsIn(options, exclude);
|
|
2988
|
-
return this.guarded(out, handles, out.intersect(engine, refs, opts));
|
|
3055
|
+
return this.combineAs(AS_IDS, "intersect", others, options ?? void 0);
|
|
2989
3056
|
}
|
|
2990
3057
|
/**
|
|
2991
3058
|
* Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
|
|
@@ -3009,21 +3076,7 @@ var Segment = class _Segment {
|
|
|
3009
3076
|
*/
|
|
3010
3077
|
async intersectInto(dest, others, options) {
|
|
3011
3078
|
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3012
|
-
|
|
3013
|
-
...segmentList(others, "intersectInto: `others`"),
|
|
3014
|
-
...this.excludesOf(options)
|
|
3015
|
-
];
|
|
3016
|
-
this.assertIntoHandles(dest, operands);
|
|
3017
|
-
this.combineEngine([this, ...operands]);
|
|
3018
|
-
return this.timed(
|
|
3019
|
-
"intersectInto",
|
|
3020
|
-
() => this.materialize(
|
|
3021
|
-
dest.ref,
|
|
3022
|
-
new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
|
|
3023
|
-
"intersectInto",
|
|
3024
|
-
options
|
|
3025
|
-
)
|
|
3026
|
-
);
|
|
3079
|
+
return this.combineInto("intersectInto", dest, others, options);
|
|
3027
3080
|
}
|
|
3028
3081
|
/**
|
|
3029
3082
|
* `this ∪ others…`, minus `options.exclude` — streamed ascending.
|
|
@@ -3038,44 +3091,12 @@ var Segment = class _Segment {
|
|
|
3038
3091
|
union(others, options) {
|
|
3039
3092
|
const bad = optionsError(options, "union", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
3040
3093
|
if (bad !== void 0) return failing(bad);
|
|
3041
|
-
return this.
|
|
3042
|
-
}
|
|
3043
|
-
unionAs(out, others, options) {
|
|
3044
|
-
let exclude;
|
|
3045
|
-
let handles;
|
|
3046
|
-
try {
|
|
3047
|
-
exclude = this.excludesOf(options);
|
|
3048
|
-
handles = [this, ...segmentList(others, "union: `others`"), ...exclude];
|
|
3049
|
-
this.assertSegments(handles);
|
|
3050
|
-
this.assertLeases(handles);
|
|
3051
|
-
} catch (err) {
|
|
3052
|
-
return out.failing(err);
|
|
3053
|
-
}
|
|
3054
|
-
let engine;
|
|
3055
|
-
try {
|
|
3056
|
-
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
3057
|
-
} catch (err) {
|
|
3058
|
-
return out.failing(err);
|
|
3059
|
-
}
|
|
3060
|
-
const refs = [this.ref, ...others.map((o) => o.ref)];
|
|
3061
|
-
const opts = this.refsIn(options, exclude);
|
|
3062
|
-
return this.guarded(out, handles, out.union(engine, refs, opts));
|
|
3094
|
+
return this.combineAs(AS_IDS, "union", others, options ?? void 0);
|
|
3063
3095
|
}
|
|
3064
3096
|
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3065
3097
|
async unionInto(dest, others, options) {
|
|
3066
3098
|
checkOptions(options, "unionInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3067
|
-
|
|
3068
|
-
this.assertIntoHandles(dest, operands);
|
|
3069
|
-
this.combineEngine([this, ...operands]);
|
|
3070
|
-
return this.timed(
|
|
3071
|
-
"unionInto",
|
|
3072
|
-
() => this.materialize(
|
|
3073
|
-
dest.ref,
|
|
3074
|
-
new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
|
|
3075
|
-
"unionInto",
|
|
3076
|
-
options
|
|
3077
|
-
)
|
|
3078
|
-
);
|
|
3099
|
+
return this.combineInto("unionInto", dest, others, options);
|
|
3079
3100
|
}
|
|
3080
3101
|
/**
|
|
3081
3102
|
* `this \ (excludes…)` — streamed ascending. Suppression on its own.
|
|
@@ -3091,12 +3112,26 @@ var Segment = class _Segment {
|
|
|
3091
3112
|
andNot(excludes, options) {
|
|
3092
3113
|
const bad = optionsError(options, "andNot", CALL_KEYS.andNot, ["allowAbsentOperands"]);
|
|
3093
3114
|
if (bad !== void 0) return failing(bad);
|
|
3094
|
-
return this.
|
|
3115
|
+
return this.combineAs(AS_IDS, "andNot", excludes, options ?? void 0);
|
|
3095
3116
|
}
|
|
3096
|
-
|
|
3117
|
+
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3118
|
+
async andNotInto(dest, excludes, options) {
|
|
3119
|
+
checkOptions(options, "andNotInto", CALL_KEYS.andNotInto, [
|
|
3120
|
+
"allowEmpty",
|
|
3121
|
+
"allowAbsentOperands"
|
|
3122
|
+
]);
|
|
3123
|
+
return this.combineInto("andNotInto", dest, excludes, options);
|
|
3124
|
+
}
|
|
3125
|
+
/**
|
|
3126
|
+
* The one body of the three combines, read as ids or as chunks. `operands` is `others` for `intersect` and
|
|
3127
|
+
* `union`, and `excludes` for `andNot`, which takes no `exclude` option.
|
|
3128
|
+
*/
|
|
3129
|
+
combineAs(out, verb, operands, options) {
|
|
3130
|
+
let exclude;
|
|
3097
3131
|
let handles;
|
|
3098
3132
|
try {
|
|
3099
|
-
|
|
3133
|
+
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3134
|
+
handles = [this, ...segmentList(operands, operandsLabel(verb)), ...exclude];
|
|
3100
3135
|
this.assertSegments(handles);
|
|
3101
3136
|
this.assertLeases(handles);
|
|
3102
3137
|
} catch (err) {
|
|
@@ -3104,77 +3139,54 @@ var Segment = class _Segment {
|
|
|
3104
3139
|
}
|
|
3105
3140
|
let engine;
|
|
3106
3141
|
try {
|
|
3107
|
-
engine = this.combineEngine([this, ...
|
|
3142
|
+
engine = this.combineEngine([this, ...operands, ...exclude]) ?? this.engine;
|
|
3108
3143
|
} catch (err) {
|
|
3109
3144
|
return out.failing(err);
|
|
3110
3145
|
}
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3146
|
+
if (verb === "andNot") {
|
|
3147
|
+
const refs2 = operands.map((o) => o.ref);
|
|
3148
|
+
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3149
|
+
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
3150
|
+
}
|
|
3151
|
+
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3152
|
+
const opts = this.refsIn(options, exclude);
|
|
3153
|
+
return this.guarded(
|
|
3154
|
+
out,
|
|
3155
|
+
handles,
|
|
3156
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
3157
|
+
);
|
|
3115
3158
|
}
|
|
3116
|
-
/**
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
]);
|
|
3122
|
-
this.assertIntoHandles(dest, segmentList(excludes, "andNotInto: `excludes`"));
|
|
3123
|
-
this.combineEngine([this, ...excludes]);
|
|
3159
|
+
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
3160
|
+
combineInto(name, dest, operands, options) {
|
|
3161
|
+
const verb = name.slice(0, -"Into".length);
|
|
3162
|
+
const all = verb === "andNot" ? segmentList(operands, `${name}: \`excludes\``) : [...segmentList(operands, `${name}: \`others\``), ...this.excludesOf(options)];
|
|
3163
|
+
this.assertIntoHandles(dest, all);
|
|
3164
|
+
this.combineEngine([this, ...all]);
|
|
3124
3165
|
return this.timed(
|
|
3125
|
-
|
|
3166
|
+
name,
|
|
3126
3167
|
() => this.materialize(
|
|
3127
3168
|
dest.ref,
|
|
3128
|
-
new CombineChunks(this.
|
|
3129
|
-
|
|
3169
|
+
new CombineChunks(this.combineAs(AS_CHUNKS, verb, operands, options)),
|
|
3170
|
+
name,
|
|
3130
3171
|
options
|
|
3131
3172
|
)
|
|
3132
3173
|
);
|
|
3133
3174
|
}
|
|
3134
|
-
/**
|
|
3135
|
-
* Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
|
|
3136
|
-
* reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
|
|
3137
|
-
* storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
|
|
3138
|
-
* refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
|
|
3139
|
-
* the lose-zone).
|
|
3140
|
-
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
3141
|
-
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
3142
|
-
*/
|
|
3143
|
-
async costReport(options) {
|
|
3144
|
-
checkOptions(options, "costReport", CALL_KEYS.costReport);
|
|
3145
|
-
const shape = options?.workload;
|
|
3146
|
-
if (shape !== void 0 && shape !== null && (typeof shape !== "object" || Array.isArray(shape))) {
|
|
3147
|
-
throw new ValidationError2("costReport: workload must be an object");
|
|
3148
|
-
}
|
|
3149
|
-
this.assertLeases([this]);
|
|
3150
|
-
const canMeasure = this.engine.supportsStorageSize;
|
|
3151
|
-
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
3152
|
-
const refreshMs = this.engine.pointerRefreshMs;
|
|
3153
|
-
const given = options?.workload === void 0 || options.workload === null ? void 0 : picked(options.workload, CALL_KEYS.workload);
|
|
3154
|
-
const workload = refreshMs === void 0 || given?.genTtlMs !== void 0 ? given : { ...given, genTtlMs: refreshMs };
|
|
3155
|
-
return groundedReport({
|
|
3156
|
-
storageBytes: size?.sizeBytes ?? 0,
|
|
3157
|
-
grounded: canMeasure,
|
|
3158
|
-
workload,
|
|
3159
|
-
pricing: options?.pricing,
|
|
3160
|
-
extraNotes: canMeasure ? void 0 : ["storage source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
3161
|
-
});
|
|
3162
|
-
}
|
|
3163
3175
|
};
|
|
3164
3176
|
|
|
3165
3177
|
// src/bin/export-segments.ts
|
|
3166
3178
|
function parseConfig(env) {
|
|
3167
3179
|
const root = env.CR_EXPORT_ROOT;
|
|
3168
3180
|
if (root === void 0 || root === "") {
|
|
3169
|
-
throw new
|
|
3181
|
+
throw new ValidationError5("CR_EXPORT_ROOT is required (the local-filesystem storage root)");
|
|
3170
3182
|
}
|
|
3171
3183
|
const out = env.CR_EXPORT_OUT;
|
|
3172
3184
|
if (out === void 0 || out === "") {
|
|
3173
|
-
throw new
|
|
3185
|
+
throw new ValidationError5("CR_EXPORT_OUT is required (the output directory for the dump)");
|
|
3174
3186
|
}
|
|
3175
3187
|
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3176
3188
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3177
|
-
throw new
|
|
3189
|
+
throw new ValidationError5(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3178
3190
|
}
|
|
3179
3191
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
3180
3192
|
return { root, out, format, namespace };
|
|
@@ -3189,7 +3201,7 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3189
3201
|
}
|
|
3190
3202
|
const found = await lstat(dir);
|
|
3191
3203
|
const refuse = (why) => {
|
|
3192
|
-
throw new
|
|
3204
|
+
throw new ValidationError5(`the export directory is not one this tool made: ${why}`);
|
|
3193
3205
|
};
|
|
3194
3206
|
if (found.isSymbolicLink()) refuse("a namespace directory is a symbolic link");
|
|
3195
3207
|
if (!found.isDirectory()) refuse("a namespace path is not a directory");
|
|
@@ -3198,10 +3210,17 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3198
3210
|
if ((found.mode & 63) !== 0) refuse("a namespace directory is open to group or others");
|
|
3199
3211
|
}
|
|
3200
3212
|
}
|
|
3213
|
+
var MANIFEST_NAME = "manifest.json";
|
|
3201
3214
|
function fsSink(out) {
|
|
3202
3215
|
return {
|
|
3203
3216
|
async open(ref, ext) {
|
|
3204
|
-
const
|
|
3217
|
+
const part = namespacePathPart(ref.namespace);
|
|
3218
|
+
if (part.toLowerCase() === MANIFEST_NAME) {
|
|
3219
|
+
throw new ValidationError5(
|
|
3220
|
+
`a namespace named ${MANIFEST_NAME} (in any letter case) cannot be exported to a directory: its folder would be the manifest's path`
|
|
3221
|
+
);
|
|
3222
|
+
}
|
|
3223
|
+
const dir = join(out, part);
|
|
3205
3224
|
await ensureOwnDirectory(out, dir);
|
|
3206
3225
|
const finalPath = join(dir, `${encodeNameForPath(ref.segment)}${ext}`);
|
|
3207
3226
|
const tmpPath = `${finalPath}.${randomUUID()}.part`;
|
|
@@ -3228,6 +3247,10 @@ function fsSink(out) {
|
|
|
3228
3247
|
await handle.close().catch(() => {
|
|
3229
3248
|
});
|
|
3230
3249
|
await rm(tmpPath, { force: true });
|
|
3250
|
+
await rmdir(dir).catch((err) => {
|
|
3251
|
+
if (err.code !== "ENOTEMPTY" && err.code !== "EEXIST" && err.code !== "ENOENT")
|
|
3252
|
+
throw err;
|
|
3253
|
+
});
|
|
3231
3254
|
}
|
|
3232
3255
|
};
|
|
3233
3256
|
}
|
|
@@ -3238,10 +3261,10 @@ async function requireStoreRoot(root) {
|
|
|
3238
3261
|
try {
|
|
3239
3262
|
entries = await readdir(root);
|
|
3240
3263
|
} catch {
|
|
3241
|
-
throw new
|
|
3264
|
+
throw new ValidationError5("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3242
3265
|
}
|
|
3243
3266
|
if (entries.length === 0) {
|
|
3244
|
-
throw new
|
|
3267
|
+
throw new ValidationError5(
|
|
3245
3268
|
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3246
3269
|
);
|
|
3247
3270
|
}
|
|
@@ -3255,7 +3278,13 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3255
3278
|
await requireStoreRoot(config.root);
|
|
3256
3279
|
const storage = new LocalFsStorage(config.root);
|
|
3257
3280
|
const store = new CloudRoaring({ storage });
|
|
3258
|
-
await
|
|
3281
|
+
const priorManifest = await lstat(join(config.out, MANIFEST_NAME)).catch(() => null);
|
|
3282
|
+
if (priorManifest?.isDirectory() === true) {
|
|
3283
|
+
throw new ValidationError5(
|
|
3284
|
+
`${join(config.out, MANIFEST_NAME)} is a directory, so the manifest cannot be written there: remove or rename it, or export to a fresh directory`
|
|
3285
|
+
);
|
|
3286
|
+
}
|
|
3287
|
+
await rm(join(config.out, MANIFEST_NAME), { force: true });
|
|
3259
3288
|
const manifest = await store.exportSegments(fsSink(config.out), {
|
|
3260
3289
|
format: config.format,
|
|
3261
3290
|
namespace: config.namespace
|
|
@@ -3282,7 +3311,8 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3282
3311
|
format: manifest.format,
|
|
3283
3312
|
totalSegments: manifest.totalSegments,
|
|
3284
3313
|
totalIds: manifest.totalIds,
|
|
3285
|
-
failed: manifest.failed.length
|
|
3314
|
+
failed: manifest.failed.length,
|
|
3315
|
+
skipped: manifest.skipped.length
|
|
3286
3316
|
});
|
|
3287
3317
|
return manifest;
|
|
3288
3318
|
}
|