@cloudbitmaps/roaring 0.18.3 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +10 -7
- package/README.md +3 -3
- package/dist/bin/export-segments.js +452 -473
- package/dist/bin/export-segments.js.map +3 -3
- package/dist/index.d.ts +134 -220
- package/dist/index.js +440 -473
- package/dist/index.js.map +3 -3
- package/dist/testing/conformance.d.ts +29 -2
- package/package.json +2 -2
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// src/bin/export-segments.ts
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { realpathSync } from "node:fs";
|
|
6
|
-
import { lstat, mkdir, open, rename, rm, writeFile } from "node:fs/promises";
|
|
6
|
+
import { lstat, mkdir, open, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import { pathToFileURL } from "node:url";
|
|
9
9
|
import { encodeNameForPath, namespacePathPart } from "@cloudbitmaps/core/driver-kit";
|
|
@@ -28,7 +28,6 @@ import {
|
|
|
28
28
|
UnsupportedError,
|
|
29
29
|
ValidationError as ValidationError2,
|
|
30
30
|
WriteConflictError,
|
|
31
|
-
MIN_EXPIRES_AT_MS,
|
|
32
31
|
checkBudget,
|
|
33
32
|
collectWithinBudget,
|
|
34
33
|
excludingReservedRows,
|
|
@@ -36,16 +35,15 @@ import {
|
|
|
36
35
|
eraseIdFromSegment,
|
|
37
36
|
estimateCost,
|
|
38
37
|
groundedReport,
|
|
38
|
+
judgeLoad,
|
|
39
39
|
loadSegment,
|
|
40
40
|
loadSegmentChunks,
|
|
41
41
|
compileCombineMany,
|
|
42
|
-
prepareHeld,
|
|
43
42
|
rebindCombineMany,
|
|
44
43
|
runCombineMany,
|
|
45
44
|
mapWithConcurrency,
|
|
46
45
|
resolveBudget,
|
|
47
46
|
resolvePerOpBudget,
|
|
48
|
-
reapRegistryTombstones,
|
|
49
47
|
retireExpired,
|
|
50
48
|
runConsistencyCheck,
|
|
51
49
|
runExport,
|
|
@@ -620,7 +618,7 @@ import {
|
|
|
620
618
|
eraseNamespace,
|
|
621
619
|
excludingReservedRows as excludingReservedRows2,
|
|
622
620
|
readRetentionPolicy,
|
|
623
|
-
MIN_EXPIRES_AT_MS
|
|
621
|
+
MIN_EXPIRES_AT_MS,
|
|
624
622
|
LEASE_SKEW_MS as LEASE_SKEW_MS2,
|
|
625
623
|
MAX_LEASE_MS as MAX_LEASE_MS2,
|
|
626
624
|
MAX_LEASES_PER_SEGMENT,
|
|
@@ -649,6 +647,10 @@ import {
|
|
|
649
647
|
isLeaseExpiredError,
|
|
650
648
|
isLeaseLimitError,
|
|
651
649
|
isStaleOperandError,
|
|
650
|
+
isUnsupportedError,
|
|
651
|
+
isCapabilityError,
|
|
652
|
+
isBudgetExceededError,
|
|
653
|
+
isKeyUnavailableError,
|
|
652
654
|
CrbmReader,
|
|
653
655
|
BufferReader,
|
|
654
656
|
CountingMetricsSink,
|
|
@@ -666,12 +668,25 @@ var SystemRng = class {
|
|
|
666
668
|
};
|
|
667
669
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
668
670
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
671
|
+
function rowVersionOfLive(version) {
|
|
672
|
+
if (version === null) return null;
|
|
673
|
+
const cut = version.lastIndexOf("#");
|
|
674
|
+
return cut < 0 ? version : version.slice(0, cut);
|
|
675
|
+
}
|
|
669
676
|
function validateConcurrency(concurrency) {
|
|
670
677
|
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
671
678
|
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
672
679
|
}
|
|
673
680
|
}
|
|
674
681
|
function requireScope(options, op) {
|
|
682
|
+
if (options.namespace !== void 0) {
|
|
683
|
+
if (typeof options.namespace !== "string") {
|
|
684
|
+
throw new ValidationError2(
|
|
685
|
+
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
689
|
+
}
|
|
675
690
|
refuseReservedNamespace(options.namespace);
|
|
676
691
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
677
692
|
throw new ValidationError2(
|
|
@@ -679,9 +694,154 @@ function requireScope(options, op) {
|
|
|
679
694
|
);
|
|
680
695
|
}
|
|
681
696
|
}
|
|
697
|
+
var keysOf = () => (keys) => keys;
|
|
698
|
+
var CALL_KEYS = {
|
|
699
|
+
range: keysOf()(["after", "through"]),
|
|
700
|
+
andNot: keysOf()([
|
|
701
|
+
"after",
|
|
702
|
+
"through",
|
|
703
|
+
"concurrency",
|
|
704
|
+
"budget",
|
|
705
|
+
"allowAbsentOperands"
|
|
706
|
+
]),
|
|
707
|
+
combine: keysOf()([
|
|
708
|
+
"after",
|
|
709
|
+
"through",
|
|
710
|
+
"concurrency",
|
|
711
|
+
"budget",
|
|
712
|
+
"allowAbsentOperands",
|
|
713
|
+
"exclude"
|
|
714
|
+
]),
|
|
715
|
+
into: keysOf()([
|
|
716
|
+
"after",
|
|
717
|
+
"through",
|
|
718
|
+
"concurrency",
|
|
719
|
+
"budget",
|
|
720
|
+
"allowAbsentOperands",
|
|
721
|
+
"exclude",
|
|
722
|
+
"audit",
|
|
723
|
+
"allowEmpty",
|
|
724
|
+
"guard",
|
|
725
|
+
"metadata",
|
|
726
|
+
"keep"
|
|
727
|
+
]),
|
|
728
|
+
andNotInto: keysOf()([
|
|
729
|
+
"after",
|
|
730
|
+
"through",
|
|
731
|
+
"concurrency",
|
|
732
|
+
"budget",
|
|
733
|
+
"allowAbsentOperands",
|
|
734
|
+
"audit",
|
|
735
|
+
"allowEmpty",
|
|
736
|
+
"guard",
|
|
737
|
+
"metadata",
|
|
738
|
+
"keep"
|
|
739
|
+
]),
|
|
740
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
741
|
+
segments: keysOf()(["namespace"]),
|
|
742
|
+
subjectReport: keysOf()([
|
|
743
|
+
"namespace",
|
|
744
|
+
"allNamespaces",
|
|
745
|
+
"concurrency",
|
|
746
|
+
"budget"
|
|
747
|
+
]),
|
|
748
|
+
eraseSubject: keysOf()([
|
|
749
|
+
"namespace",
|
|
750
|
+
"allNamespaces",
|
|
751
|
+
"audit",
|
|
752
|
+
"concurrency",
|
|
753
|
+
"budget"
|
|
754
|
+
]),
|
|
755
|
+
rollback: keysOf()([
|
|
756
|
+
"audit",
|
|
757
|
+
"allowForward"
|
|
758
|
+
]),
|
|
759
|
+
dropSegment: keysOf()([
|
|
760
|
+
"confirmSegment",
|
|
761
|
+
"dryRun",
|
|
762
|
+
"audit"
|
|
763
|
+
]),
|
|
764
|
+
retireExpired: keysOf()([
|
|
765
|
+
"namespace",
|
|
766
|
+
"now",
|
|
767
|
+
"limit",
|
|
768
|
+
"dryRun",
|
|
769
|
+
"audit",
|
|
770
|
+
"maxScanSegments",
|
|
771
|
+
"scan",
|
|
772
|
+
"shards",
|
|
773
|
+
"totalShards",
|
|
774
|
+
"lookbackBuckets",
|
|
775
|
+
"purgeTombstones",
|
|
776
|
+
"tombstoneGraceMs"
|
|
777
|
+
]),
|
|
778
|
+
checkConsistency: keysOf()([
|
|
779
|
+
"namespace",
|
|
780
|
+
"concurrency",
|
|
781
|
+
"summaries"
|
|
782
|
+
]),
|
|
783
|
+
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"]),
|
|
784
|
+
costReport: keysOf()(["pricing", "workload"]),
|
|
785
|
+
workload: keysOf()([
|
|
786
|
+
"readsPerSec",
|
|
787
|
+
"intersectsPerSec",
|
|
788
|
+
"cacheHitRate",
|
|
789
|
+
"chunksPerIntersect",
|
|
790
|
+
"operandsPerIntersect",
|
|
791
|
+
"loadsPerMonth",
|
|
792
|
+
"requestsPerLoad",
|
|
793
|
+
"hotSegments",
|
|
794
|
+
"readerProcesses",
|
|
795
|
+
"genTtlMs",
|
|
796
|
+
"retirementsPerMonth",
|
|
797
|
+
"purgesPerMonth",
|
|
798
|
+
"conditionalDelete"
|
|
799
|
+
])
|
|
800
|
+
};
|
|
801
|
+
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
802
|
+
function optionsError(options, op, known, booleans = []) {
|
|
803
|
+
if (options === void 0 || options === null) return void 0;
|
|
804
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
805
|
+
return new ValidationError2(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
806
|
+
}
|
|
807
|
+
const bag = options;
|
|
808
|
+
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
809
|
+
if (unknown.length > 0) {
|
|
810
|
+
return new ValidationError2(
|
|
811
|
+
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
const audit = bag.audit;
|
|
815
|
+
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
816
|
+
return new ValidationError2(
|
|
817
|
+
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
for (const name of booleans) {
|
|
821
|
+
const v = bag[name];
|
|
822
|
+
if (v !== void 0 && typeof v !== "boolean") {
|
|
823
|
+
return new ValidationError2(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
return void 0;
|
|
827
|
+
}
|
|
828
|
+
function picked(options, keys) {
|
|
829
|
+
const out = {};
|
|
830
|
+
if (options === void 0 || options === null) return out;
|
|
831
|
+
for (const key of keys) {
|
|
832
|
+
const value = options[key];
|
|
833
|
+
if (value !== void 0) out[key] = value;
|
|
834
|
+
}
|
|
835
|
+
return out;
|
|
836
|
+
}
|
|
837
|
+
function checkOptions(options, op, known, booleans) {
|
|
838
|
+
const err = optionsError(options, op, known, booleans);
|
|
839
|
+
if (err !== void 0) throw err;
|
|
840
|
+
}
|
|
682
841
|
var viewOf;
|
|
683
842
|
var MATERIALIZE_MANY_KEYS = [
|
|
684
843
|
"operands",
|
|
844
|
+
"dryRun",
|
|
685
845
|
"outputs",
|
|
686
846
|
"keep",
|
|
687
847
|
"pin",
|
|
@@ -870,8 +1030,29 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
870
1030
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
871
1031
|
);
|
|
872
1032
|
}
|
|
1033
|
+
/**
|
|
1034
|
+
* A seam the store calls through must have the methods it calls. Without this a `clock` with no `now`, an `rng` with
|
|
1035
|
+
* no `next` or a keystore with no `createDek` builds a store that throws a raw `TypeError` at its first load or read.
|
|
1036
|
+
*/
|
|
1037
|
+
static rejectHollowSeams(options) {
|
|
1038
|
+
const seams = [
|
|
1039
|
+
[options.seams?.clock, "seams.clock", ["now", "sleep"]],
|
|
1040
|
+
[options.seams?.rng, "seams.rng", ["next"]],
|
|
1041
|
+
[options.encryption?.keystore, "encryption.keystore", ["createDek", "openDek"]]
|
|
1042
|
+
];
|
|
1043
|
+
for (const [seam, name, needs] of seams) {
|
|
1044
|
+
if (seam === void 0 || seam === null) continue;
|
|
1045
|
+
const has = (m) => typeof seam[m] === "function";
|
|
1046
|
+
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1047
|
+
throw new ValidationError2(
|
|
1048
|
+
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
873
1053
|
constructor(options) {
|
|
874
1054
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1055
|
+
_CloudRoaring.rejectHollowSeams(options);
|
|
875
1056
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
876
1057
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
877
1058
|
const metrics = safeMetrics(options.metrics ?? NOOP_METRICS);
|
|
@@ -887,6 +1068,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
887
1068
|
let storage = resolved.source;
|
|
888
1069
|
if (options.retry !== false) {
|
|
889
1070
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
1071
|
+
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
1072
|
+
throw new ValidationError2("retry.onRetry must be a function");
|
|
1073
|
+
}
|
|
890
1074
|
const policy = {
|
|
891
1075
|
maxAttempts: ov.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
892
1076
|
baseDelayMs: ov.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
|
|
@@ -962,37 +1146,51 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
962
1146
|
readRetry: this.retryOptions
|
|
963
1147
|
};
|
|
964
1148
|
}
|
|
965
|
-
/**
|
|
1149
|
+
/**
|
|
1150
|
+
* Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). The options
|
|
1151
|
+
* are `{ namespace }` and nothing else: anything else is refused with {@link ValidationError}, by name, since a
|
|
1152
|
+
* misspelt `namespace` would otherwise address the default namespace. An `expiresAt` among them is refused with its own
|
|
1153
|
+
* message: a handle carries no deadline, and one left unread would serve the data past it. A key whose value is
|
|
1154
|
+
* `undefined` is read as absent, so a spread of options keeps working.
|
|
1155
|
+
*/
|
|
966
1156
|
segment(name, options) {
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1157
|
+
if (options !== void 0 && options !== null) {
|
|
1158
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1159
|
+
throw new ValidationError2("segment: options must be an object such as { namespace }");
|
|
1160
|
+
}
|
|
1161
|
+
if (options.expiresAt !== void 0) {
|
|
1162
|
+
throw new ValidationError2(
|
|
1163
|
+
"segment: `expiresAt` is not an option of a handle; check a deadline where you read, or record it with `store.setRetention(ref, { expiresAt })` and run `store.retireExpired()`"
|
|
1164
|
+
);
|
|
1165
|
+
}
|
|
1166
|
+
const unknown = Object.keys(options).filter(
|
|
1167
|
+
(k) => k !== "namespace" && options[k] !== void 0
|
|
1168
|
+
);
|
|
1169
|
+
if (unknown.length > 0) {
|
|
973
1170
|
throw new ValidationError2(
|
|
974
|
-
`segment:
|
|
1171
|
+
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
975
1172
|
);
|
|
976
1173
|
}
|
|
977
1174
|
}
|
|
1175
|
+
const ref = { segment: name, namespace: options?.namespace };
|
|
1176
|
+
validateSegmentRef(ref);
|
|
1177
|
+
refuseReservedNamespace(ref.namespace);
|
|
978
1178
|
return makeSegment({
|
|
979
1179
|
engine: this.engine,
|
|
980
1180
|
ref,
|
|
981
1181
|
clock: this.clock,
|
|
982
1182
|
metrics: this.metrics,
|
|
983
1183
|
materialize: (dest, ids, op, options2) => this.materialize(dest, ids, op, options2),
|
|
984
|
-
pinned: (r,
|
|
985
|
-
combineEngine: (handles) => this.engineForCombine(handles)
|
|
986
|
-
expiresAt
|
|
1184
|
+
pinned: (r, l, at) => this.pinSegment(r, l, at),
|
|
1185
|
+
combineEngine: (handles) => this.engineForCombine(handles)
|
|
987
1186
|
});
|
|
988
1187
|
}
|
|
989
1188
|
/**
|
|
990
1189
|
* Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
|
|
991
1190
|
* materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
|
|
992
|
-
* publish on the row's token
|
|
993
|
-
*
|
|
994
|
-
*
|
|
995
|
-
* forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
1191
|
+
* publish on the row's token, and every load that finds none on that absence, guarded or not; a guarded load (the
|
|
1192
|
+
* default, since the empty refusal needs the size of the current generation) also fences on the pointer it judged
|
|
1193
|
+
* (`expectFrom`). The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
996
1194
|
* within `cache.genTtlMs`).
|
|
997
1195
|
*
|
|
998
1196
|
* This routes through `loadSegment` rather than writing the generation itself, and that is the whole point
|
|
@@ -1019,9 +1217,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1019
1217
|
* **A `WriteConflictError` does not by itself mean nothing was published**, and that is worth knowing
|
|
1020
1218
|
* before you write the retry. `'superseded'` covers five different causes — the write-once PUT collided,
|
|
1021
1219
|
* the pointer moved, a row appeared where the load found none, the row's token changed, the row was purged —
|
|
1022
|
-
* and only the first
|
|
1023
|
-
* that is not a supersession at all
|
|
1024
|
-
*
|
|
1220
|
+
* and only the first two are always the "somebody beat us" the name suggests. A row can appear, and a token can
|
|
1221
|
+
* change, on a write that is not a supersession at all: a `setRetention` on the destination creates its row when it
|
|
1222
|
+
* has none, and rewrites it when it has one. On top of that, the collection pass that runs AFTER a successful
|
|
1223
|
+
* publish can raise the same error. So: treat it as "re-read the destination and decide",
|
|
1025
1224
|
* never as "the write did not happen".
|
|
1026
1225
|
*
|
|
1027
1226
|
* **And by default it collects nothing**, unlike `load()`. See the `keep` default below.
|
|
@@ -1053,7 +1252,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1053
1252
|
if (reason === "superseded") {
|
|
1054
1253
|
const wrote = result.size > 0 ? `generation ${result.generation} was written and is not current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
1055
1254
|
throw new WriteConflictError(
|
|
1056
|
-
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result is not the destination's current generation: ${wrote}. The pointer may have moved, the row may have been rewritten (a retention policy does
|
|
1255
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result is not the destination's current generation: ${wrote}. The pointer may have moved, the row may have been created or rewritten (a retention policy does either), dropped or purged. Re-read the destination and re-run.`
|
|
1057
1256
|
);
|
|
1058
1257
|
}
|
|
1059
1258
|
return {
|
|
@@ -1067,131 +1266,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1067
1266
|
collected: result.collected
|
|
1068
1267
|
};
|
|
1069
1268
|
}
|
|
1070
|
-
/**
|
|
1071
|
-
* **Hold a set of ids in memory**, to be an operand of {@link CloudRoaring.materializeMany} and of no other verb.
|
|
1072
|
-
*
|
|
1073
|
-
* ```ts
|
|
1074
|
-
* const vip = await store.memory([7, 42, 99_001]);
|
|
1075
|
-
* await store.materializeMany({ operands: { vip, engaged: store.segment('cond-engaged') }, outputs: [...], keep: 12 });
|
|
1076
|
-
* vip.release();
|
|
1077
|
-
* ```
|
|
1078
|
-
*
|
|
1079
|
-
* It takes what {@link CloudRoaring.load} takes as input (ids as a sync or async iterable or a typed array, `{ bitmap }`
|
|
1080
|
-
* or `{ serialized }`) and checks it the same way, with the same errors: the size cap, the structure, the safe
|
|
1081
|
-
* deserializer and each id's range. **Only a real `Uint32Array` skips the per-id range check**; any other typed array goes
|
|
1082
|
-
* through it, so `Int32Array([-1])` and `Float64Array([1.5])` throw {@link ValidationError}. An object with an `ascending`
|
|
1083
|
-
* key is refused: ids that arrive in key order are a `feed`. An empty set is accepted here, and refused by
|
|
1084
|
-
* `materializeMany` unless its name is in `mayBeEmpty`.
|
|
1085
|
-
*
|
|
1086
|
-
* The ids are held as the chunks a stored generation would hold, until {@link MemoryOperand.release}; between calls that
|
|
1087
|
-
* memory is the caller's, and during a call it counts against `maxBufferedBytes`. There is no size option: the load's
|
|
1088
|
-
* own cap applies to a serialized input.
|
|
1089
|
-
*
|
|
1090
|
-
* **Erasure.** Only `eraseSubject` moves the store's counter (at its start and at its end). One that started in this store
|
|
1091
|
-
* since the handle began to be made, or is still running, makes a call that reads the handle fail the outputs that read it with
|
|
1092
|
-
* {@link StaleOperandError} (`reason: 'erased'`), before any request and again just before each publish. An erasure in
|
|
1093
|
-
* another store or another process is not seen, nor are `dropSegment`, `eraseNamespace`, `destroySegment` and the free
|
|
1094
|
-
* function `eraseIdFromSegment`, which do not move it: the handle is the caller's copy, so build it from the source of truth at
|
|
1095
|
-
* the start of each refresh and release it at the end.
|
|
1096
|
-
*
|
|
1097
|
-
* @throws {ValidationError} where a load of the same input would.
|
|
1098
|
-
*/
|
|
1099
|
-
async memory(input) {
|
|
1100
|
-
const epochAt = this.epochNow();
|
|
1101
|
-
const chunks = await prepareHeld(input, roaringCodec, this.clock);
|
|
1102
|
-
return makeMemoryOperand({ store: this, chunks, epochAt });
|
|
1103
|
-
}
|
|
1104
|
-
/**
|
|
1105
|
-
* **Many outputs, each operand read once per group.** Each output is an expression over named stored operands (`and`, `or`, `andNot`, nested
|
|
1106
|
-
* to depth 64) published as a new generation of its own `dest`, exactly as an `*Into` would publish it, but computed
|
|
1107
|
-
* together: each operand's chunks are read once for all the outputs that use them, instead of once per output and
|
|
1108
|
-
* once more for every scratch segment a nested expression would otherwise need.
|
|
1109
|
-
*
|
|
1110
|
-
* ```ts
|
|
1111
|
-
* const run = await store.materializeMany({
|
|
1112
|
-
* operands: { us: store.segment('us'), engaged: store.segment('engaged'), optOut: store.segment('opt-out') },
|
|
1113
|
-
* outputs: [
|
|
1114
|
-
* { dest: store.segment('send-1'), expr: { and: ['us', 'engaged'] }, exclude: ['optOut'] },
|
|
1115
|
-
* { dest: store.segment('send-2'), expr: { andNot: ['us', 'engaged'] } },
|
|
1116
|
-
* ],
|
|
1117
|
-
* keep: 3,
|
|
1118
|
-
* });
|
|
1119
|
-
* run.outputs[0]; // a MaterializeResult, or { published: false, error }
|
|
1120
|
-
* ```
|
|
1121
|
-
*
|
|
1122
|
-
* **Results.** `run.outputs[i]` is index-aligned with `outputs` and is exactly what output `i`'s `*Into` would have
|
|
1123
|
-
* returned (a {@link MaterializeResult}: published, or refused with its `reason`), or `{ published: false, error }` for
|
|
1124
|
-
* what it would have thrown. One output's refusal or error never stops another. The call itself throws only for bad
|
|
1125
|
-
* input ({@link ValidationError} naming the output index and path, before any request), an operand that names no
|
|
1126
|
-
* segment (unless `allowAbsentOperands`), and a `budget` its own plan exceeds, the last two before any chunk is read.
|
|
1127
|
-
*
|
|
1128
|
-
* **One generation per operand.** By default every stored operand is pinned for the call: read at the generation it had
|
|
1129
|
-
* when the call started, however long the call runs and whatever loads land meanwhile. That is one generation per
|
|
1130
|
-
* operand, not one instant across operands (two pins are two registry reads). `pin: false` reads live, as a combine does.
|
|
1131
|
-
*
|
|
1132
|
-
* **Every pinned operand an output subtracts is re-checked.** An output subtracts an operand when it is in its `exclude`
|
|
1133
|
-
* or under any entry of an `andNot` after the first, at any depth. Immediately before the publishes the call re-reads the
|
|
1134
|
-
* registry row of each such operand and compares its generation and incarnation, and the fingerprint of the current
|
|
1135
|
-
* object where both reads carry one, with what was pinned (a name deleted and created again is a replacement though it
|
|
1136
|
-
* is generation 0 again). If one was replaced, each output that subtracts it is
|
|
1137
|
-
* not published and carries {@link StaleOperandError}; the others publish. It costs one registry read per such operand per
|
|
1138
|
-
* group, and `stats.operands` carries every operand's pinned, start and end generation and whether it moved.
|
|
1139
|
-
*
|
|
1140
|
-
* **Memory.** Everything the pass holds resident is counted against `maxBufferedBytes` (default 256 MiB): each output's
|
|
1141
|
-
* buffered chunks, the operand streams' ranges, the chunks being evaluated, and the plan itself, and a call whose plan alone
|
|
1142
|
-
* passes it is refused before any chunk is read. When the outputs do not fit, they run in groups, each reading its operands
|
|
1143
|
-
* once (so the operand reads grow with the total output size over the budget), and a group's buffers are released only as
|
|
1144
|
-
* its publishes settle. A chunk count the index understated does not grow a group past the budget: the largest buffer is
|
|
1145
|
-
* dropped and that output re-runs alone. An output that cannot fit alone has `BudgetExceededError` as its result. The count
|
|
1146
|
-
* is the pass's own: process memory is more, measured at 1.6 times the ledger's high water on Linux and 2.5 to 5 times on
|
|
1147
|
-
* macOS on the in-memory backend (the guide has the shapes and what is not counted).
|
|
1148
|
-
*
|
|
1149
|
-
* **Untrusted bytes.** Every operand chunk is decoded through the checks on untrusted tier data. A key an operand's index
|
|
1150
|
-
* lists whose bytes are missing is an error for the outputs that read the operand, never an empty chunk.
|
|
1151
|
-
*
|
|
1152
|
-
* **Handles.** An expired or released handle anywhere in the call (an operand or a `dest`) is refused before any request,
|
|
1153
|
-
* as on the `*Into` verbs. A leased operand's lease, and an operand's deadline, are checked before each chunk key
|
|
1154
|
-
* is read: a lapse fails only the outputs that read that operand, never reads empty. Each `dest`'s lease and deadline
|
|
1155
|
-
* are checked again just before its publish.
|
|
1156
|
-
*
|
|
1157
|
-
* **The chunk cache is bypassed:** the pass reads through the storage source and never writes the decoded-chunk cache, so
|
|
1158
|
-
* a batch does not evict other readers' hot chunks. Each publish still invalidates its `dest` as an `*Into` does.
|
|
1159
|
-
*
|
|
1160
|
-
* **A limit of the form.** An output cannot name another output of the same call: a `dest` that is also an operand is
|
|
1161
|
-
* refused. Chain two calls.
|
|
1162
|
-
*
|
|
1163
|
-
* **Held operands.** An operand may be a {@link MemoryOperand} from {@link CloudRoaring.memory}: ids the caller holds,
|
|
1164
|
-
* read beside the stored ones and giving the same output byte for byte. It is read from memory (no request, and the
|
|
1165
|
-
* shared chunk cache never sees it), counts against `maxBufferedBytes` while the call runs, and is refused when empty
|
|
1166
|
-
* unless its name is in `mayBeEmpty`. A handle from another store, or a released one, is a {@link ValidationError}. An
|
|
1167
|
-
* erasure in this store since the handle was made fails the outputs that read it with {@link StaleOperandError}
|
|
1168
|
-
* (`reason: 'erased'`).
|
|
1169
|
-
*
|
|
1170
|
-
* **A feed.** `feed: { names, records, counts }` adds operands that arrive as records in chunk-key order, for conditions
|
|
1171
|
-
* too many to hold or store. Each record is checked before the pass sees it (see {@link MaterializeManyFeedRecord}): a bad
|
|
1172
|
-
* feed is refused, never read as fewer members, and **a fed output is published only after the whole feed was read and every
|
|
1173
|
-
* end-of-feed check passed**, so a bad feed refuses every fed output and publishes none of them; outputs that name only
|
|
1174
|
-
* stored operands are unaffected. At the end of the feed `counts` must equal the ids the feed held for each name, and a
|
|
1175
|
-
* name that appeared in no record is refused unless it is in `mayBeEmpty`. A feed is read once, so the call runs its outputs
|
|
1176
|
-
* as one group and **`maxBufferedBytes` is required**: each record is converted to compressed bitmaps as it arrives, the key
|
|
1177
|
-
* in hand and its fed outputs are counted against the budget, and the call fails as soon as the count passes it, with
|
|
1178
|
-
* {@link BudgetExceededError} for every fed output. **Erasure:** a fed call is refused at its next record, and before each
|
|
1179
|
-
* fed publish, once `eraseSubject` has started in this store ({@link StaleOperandError}, `reason: 'erased'`).
|
|
1180
|
-
*
|
|
1181
|
-
* ```ts
|
|
1182
|
-
* await store.materializeMany({
|
|
1183
|
-
* operands: { unsub: store.segment('global-unsub') },
|
|
1184
|
-
* feed: {
|
|
1185
|
-
* names: ['us', 'engaged'],
|
|
1186
|
-
* records: warehouseRecords(), // yields { key, operands: { us: Uint32Array, engaged: Uint32Array } } by key
|
|
1187
|
-
* counts: () => warehouseCounts(),
|
|
1188
|
-
* },
|
|
1189
|
-
* maxBufferedBytes: 512 * 1024 * 1024,
|
|
1190
|
-
* outputs: [{ dest: store.segment('send-1'), expr: { and: ['us', 'engaged'] }, exclude: ['unsub'] }],
|
|
1191
|
-
* keep: 12,
|
|
1192
|
-
* });
|
|
1193
|
-
* ```
|
|
1194
|
-
*/
|
|
1269
|
+
/** The one body of the forms above. */
|
|
1195
1270
|
async materializeMany(options) {
|
|
1196
1271
|
if (typeof options !== "object" || options === null) {
|
|
1197
1272
|
throw new ValidationError2(
|
|
@@ -1207,12 +1282,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1207
1282
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1208
1283
|
);
|
|
1209
1284
|
}
|
|
1285
|
+
const dryRunIn = options.dryRun;
|
|
1286
|
+
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1287
|
+
throw new ValidationError2("materializeMany: dryRun must be a boolean");
|
|
1288
|
+
}
|
|
1289
|
+
const dryRun = dryRunIn === true;
|
|
1210
1290
|
const operandsIn = options.operands;
|
|
1211
1291
|
const outputsIn = options.outputs;
|
|
1212
1292
|
const keep = options.keep;
|
|
1213
1293
|
const after = options.after;
|
|
1214
1294
|
const through = options.through;
|
|
1215
|
-
const pinOption = options.pin
|
|
1295
|
+
const pinOption = options.pin === void 0 ? true : options.pin;
|
|
1216
1296
|
const feedIn = options.feed;
|
|
1217
1297
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1218
1298
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
@@ -1238,50 +1318,26 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1238
1318
|
}
|
|
1239
1319
|
const names = Object.keys(operandsIn);
|
|
1240
1320
|
const handles = /* @__PURE__ */ new Map();
|
|
1241
|
-
const heldHandles = /* @__PURE__ */ new Map();
|
|
1242
1321
|
for (const name of names) {
|
|
1243
1322
|
const handle = operandsIn[name];
|
|
1244
|
-
if (handle instanceof MemoryOperand) {
|
|
1245
|
-
const view = heldOf(handle);
|
|
1246
|
-
if (view === void 0) {
|
|
1247
|
-
throw new ValidationError2(
|
|
1248
|
-
`materializeMany: operands["${name}"] is not a memory operand from store.memory()`
|
|
1249
|
-
);
|
|
1250
|
-
}
|
|
1251
|
-
if (view.store !== this) {
|
|
1252
|
-
throw new ValidationError2(
|
|
1253
|
-
`materializeMany: operands["${name}"] was made by another store: a memory operand belongs to the store that made it`
|
|
1254
|
-
);
|
|
1255
|
-
}
|
|
1256
|
-
if (view.chunks.released) {
|
|
1257
|
-
throw new ValidationError2(
|
|
1258
|
-
`materializeMany: operands["${name}"] was released: make another with store.memory()`
|
|
1259
|
-
);
|
|
1260
|
-
}
|
|
1261
|
-
heldHandles.set(name, view);
|
|
1262
|
-
continue;
|
|
1263
|
-
}
|
|
1264
1323
|
if (!(handle instanceof Segment)) {
|
|
1265
1324
|
throw new ValidationError2(
|
|
1266
|
-
`materializeMany: operands["${name}"] must be a segment from store.segment()
|
|
1325
|
+
`materializeMany: operands["${name}"] must be a segment from store.segment()`
|
|
1267
1326
|
);
|
|
1268
1327
|
}
|
|
1269
1328
|
handles.set(name, handle);
|
|
1270
1329
|
}
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
for (const name of mayBeEmptyIn) {
|
|
1277
|
-
if (typeof name === "string" && heldHandles.has(name)) heldMayBeEmpty.add(name);
|
|
1278
|
-
else rest.push(name);
|
|
1279
|
-
}
|
|
1280
|
-
fedMayBeEmpty = rest;
|
|
1330
|
+
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1331
|
+
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1332
|
+
throw new ValidationError2(
|
|
1333
|
+
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1334
|
+
);
|
|
1281
1335
|
}
|
|
1282
|
-
if (
|
|
1336
|
+
if (mayBeEmptyIn.length > 0) {
|
|
1337
|
+
const first = mayBeEmptyIn[0];
|
|
1338
|
+
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1283
1339
|
throw new ValidationError2(
|
|
1284
|
-
|
|
1340
|
+
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1285
1341
|
);
|
|
1286
1342
|
}
|
|
1287
1343
|
}
|
|
@@ -1321,26 +1377,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1321
1377
|
const storeCeiling = this.budget === null || this.budget.maxRequests !== DEFAULT_BUDGET.maxRequests ? this.budget : void 0;
|
|
1322
1378
|
const budget = budgetOption === void 0 ? storeCeiling : resolvePerOpBudget(budgetOption, this.budget);
|
|
1323
1379
|
const operandSpecs = (pins2) => names.map((name) => {
|
|
1324
|
-
const view = heldHandles.get(name);
|
|
1325
|
-
if (view !== void 0) {
|
|
1326
|
-
return {
|
|
1327
|
-
name,
|
|
1328
|
-
ref: { segment: name },
|
|
1329
|
-
held: {
|
|
1330
|
-
chunks: view.chunks,
|
|
1331
|
-
moved: () => this.epochMoved(view.epochAt),
|
|
1332
|
-
mayBeEmpty: heldMayBeEmpty.has(name)
|
|
1333
|
-
}
|
|
1334
|
-
};
|
|
1335
|
-
}
|
|
1336
1380
|
const handle = handles.get(name);
|
|
1337
1381
|
const seg = viewOf(handle);
|
|
1338
1382
|
const pinnedAt = pins2.get(handle.key());
|
|
1339
|
-
const guarded = seg.expiresAt !== void 0 || handle.lease !== void 0;
|
|
1340
1383
|
return {
|
|
1341
1384
|
name,
|
|
1342
1385
|
ref: seg.ref,
|
|
1343
|
-
...
|
|
1386
|
+
...handle.lease === void 0 ? {} : { check: () => this.assertLease(handle) },
|
|
1344
1387
|
...pinnedAt === void 0 ? {} : {
|
|
1345
1388
|
pinnedGeneration: pinnedAt.generation,
|
|
1346
1389
|
pinnedVersion: pinnedAt.version,
|
|
@@ -1351,18 +1394,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1351
1394
|
});
|
|
1352
1395
|
const request = (pins2) => ({
|
|
1353
1396
|
operands: operandSpecs(pins2),
|
|
1354
|
-
outputs: outputs.map(({ spec, dest }
|
|
1397
|
+
outputs: outputs.map(({ spec, dest }) => ({
|
|
1355
1398
|
expr: spec.expr,
|
|
1356
1399
|
...spec.exclude === void 0 ? {} : { exclude: spec.exclude },
|
|
1357
1400
|
...spec.allowEmpty === void 0 ? {} : { allowEmpty: spec.allowEmpty },
|
|
1358
1401
|
...spec.guard === void 0 ? {} : { guard: spec.guard },
|
|
1359
1402
|
...spec.metadata === void 0 ? {} : { metadata: spec.metadata },
|
|
1360
1403
|
...spec.keep === void 0 ? {} : { keep: spec.keep },
|
|
1361
|
-
beforePublish: () => this.
|
|
1404
|
+
beforePublish: () => this.assertLease(dest),
|
|
1362
1405
|
// What a settled publish proves it sent: a published one wrote its object and moved the pointer, a refused one
|
|
1363
|
-
// wrote its object and deleted it.
|
|
1364
|
-
requestsOf: (result) =>
|
|
1365
|
-
publish: (chunks, write) => this.materialize(viewOf(dest).ref, new CombineChunks(chunks), "materializeMany", {
|
|
1406
|
+
// wrote its object and deleted it. A dry run's judgement wrote nothing, and read the row and maybe the object.
|
|
1407
|
+
requestsOf: (result) => "dryRun" in result ? { get: result.reads, put: 0 } : { get: 0, put: result.published ? 2 : 1 },
|
|
1408
|
+
publish: (chunks, write) => dryRun ? this.judgeOutput(viewOf(dest).ref, chunks, write) : this.materialize(viewOf(dest).ref, new CombineChunks(chunks), "materializeMany", {
|
|
1366
1409
|
...write,
|
|
1367
1410
|
...spec.audit === void 0 ? {} : { audit: spec.audit }
|
|
1368
1411
|
})
|
|
@@ -1375,11 +1418,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1375
1418
|
concurrency,
|
|
1376
1419
|
...budget === void 0 ? {} : { budget },
|
|
1377
1420
|
...allowAbsentOperands === void 0 ? {} : { allowAbsentOperands },
|
|
1378
|
-
...feedIn === void 0 ? {} : { feed: this.feedOf(feedIn,
|
|
1421
|
+
...feedIn === void 0 ? {} : { feed: this.feedOf(feedIn, mayBeEmptyIn, epochAt) },
|
|
1422
|
+
...dryRun ? { dryRun: true } : {}
|
|
1379
1423
|
});
|
|
1380
1424
|
const compiled = compileCombineMany(request(/* @__PURE__ */ new Map()));
|
|
1381
|
-
for (const { dest } of outputs) this.
|
|
1382
|
-
for (const handle of handles.values()) this.
|
|
1425
|
+
for (const { dest } of outputs) this.assertLease(dest);
|
|
1426
|
+
for (const handle of handles.values()) this.assertLease(handle);
|
|
1383
1427
|
this.lifecycleDeps("materializeMany");
|
|
1384
1428
|
const crbm = this.crbmSource;
|
|
1385
1429
|
if (crbm === void 0) {
|
|
@@ -1416,25 +1460,64 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1416
1460
|
else if (pinOption) toPin.push([key, handle]);
|
|
1417
1461
|
else pins.set(key, void 0);
|
|
1418
1462
|
}
|
|
1419
|
-
const
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1463
|
+
const metricsOn = this.metrics !== NOOP_METRICS;
|
|
1464
|
+
const work = async () => {
|
|
1465
|
+
const taken = await mapWithConcurrency(
|
|
1466
|
+
toPin,
|
|
1467
|
+
PIN_PARALLELISM,
|
|
1468
|
+
async ([key, handle]) => [key, (await handle.pin()).pinnedAt]
|
|
1469
|
+
);
|
|
1470
|
+
for (const [key, at] of taken) pins.set(key, at);
|
|
1471
|
+
const live = new Map([...pins].filter((e) => e[1] !== void 0));
|
|
1472
|
+
const pinnedSource = new PinnedStorageChunkSource(crbm, live);
|
|
1473
|
+
const source = this.retryOptions === void 0 ? pinnedSource : new RetryingStorageChunkSource(pinnedSource, this.retryOptions);
|
|
1474
|
+
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1475
|
+
return runCombineMany(final, {
|
|
1476
|
+
source,
|
|
1477
|
+
codec: roaringCodec,
|
|
1478
|
+
clock: this.clock,
|
|
1479
|
+
...metricsOn ? { metrics: this.metrics } : {}
|
|
1480
|
+
});
|
|
1481
|
+
};
|
|
1482
|
+
const run = metricsOn ? await timeOp(this.metrics, this.clock, "materializeMany", work) : await work();
|
|
1483
|
+
const results = run.outputs.map((o) => {
|
|
1484
|
+
if (!o.ok) {
|
|
1485
|
+
return {
|
|
1433
1486
|
published: false,
|
|
1434
1487
|
error: o.error instanceof Error ? o.error : new Error(String(o.error))
|
|
1435
|
-
}
|
|
1436
|
-
|
|
1437
|
-
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
if (!("dryRun" in o.value)) return o.value;
|
|
1491
|
+
const v = o.value;
|
|
1492
|
+
const judged = {
|
|
1493
|
+
dryRun: true,
|
|
1494
|
+
published: false,
|
|
1495
|
+
cardinality: v.cardinality,
|
|
1496
|
+
cardinalityBefore: v.cardinalityBefore,
|
|
1497
|
+
...v.wouldRefuse === void 0 ? {} : { wouldRefuse: v.wouldRefuse }
|
|
1498
|
+
};
|
|
1499
|
+
return judged;
|
|
1500
|
+
});
|
|
1501
|
+
return dryRun ? { outputs: results, stats: run.stats } : { outputs: results, stats: run.stats };
|
|
1502
|
+
}
|
|
1503
|
+
/**
|
|
1504
|
+
* A dry run's publish of one output: count what the finished chunks hold, and judge that size against `dest` as a
|
|
1505
|
+
* load would, writing nothing.
|
|
1506
|
+
*/
|
|
1507
|
+
async judgeOutput(dest, chunks, write) {
|
|
1508
|
+
let cardinality = 0;
|
|
1509
|
+
for await (const chunk of chunks) cardinality += chunk.bitmap.size;
|
|
1510
|
+
const judged = await judgeLoad(dest, cardinality, this.lifecycleDeps("materializeMany"), {
|
|
1511
|
+
...write.allowEmpty === void 0 ? {} : { allowEmpty: write.allowEmpty },
|
|
1512
|
+
...write.guard === void 0 ? {} : { guard: write.guard }
|
|
1513
|
+
});
|
|
1514
|
+
return {
|
|
1515
|
+
dryRun: true,
|
|
1516
|
+
published: false,
|
|
1517
|
+
cardinality,
|
|
1518
|
+
cardinalityBefore: judged.cardinalityBefore,
|
|
1519
|
+
...judged.wouldRefuse === void 0 ? {} : { wouldRefuse: judged.wouldRefuse },
|
|
1520
|
+
reads: judged.reads
|
|
1438
1521
|
};
|
|
1439
1522
|
}
|
|
1440
1523
|
epochNow() {
|
|
@@ -1487,20 +1570,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1487
1570
|
const row = await this.withRetries(() => registry.get(ref));
|
|
1488
1571
|
return row === null ? null : { generation: row.currentGen, token: row.token };
|
|
1489
1572
|
}
|
|
1490
|
-
/** Throw what a read of `handle` must throw now: its lease error,
|
|
1491
|
-
|
|
1492
|
-
const
|
|
1493
|
-
const lease = view.leaseError();
|
|
1573
|
+
/** Throw what a read of, or a publish to, `handle` must throw now: its lease error, if it has one. */
|
|
1574
|
+
assertLease(handle) {
|
|
1575
|
+
const lease = viewOf(handle).leaseError();
|
|
1494
1576
|
if (lease !== void 0) throw lease;
|
|
1495
|
-
if (view.expired()) {
|
|
1496
|
-
throw new ValidationError2(
|
|
1497
|
-
`${op}: refusing to read "${view.ref.segment}", whose handle has expired \u2014 an expired handle reads as empty, so a generation built from it would be short. Open the handle without \`expiresAt\` if you meant to materialise.`
|
|
1498
|
-
);
|
|
1499
|
-
}
|
|
1500
|
-
}
|
|
1501
|
-
/** {@link assertReadable} for a destination, named by where it sits in the call. */
|
|
1502
|
-
assertWritable(where, dest) {
|
|
1503
|
-
this.assertReadable(where, dest);
|
|
1504
1577
|
}
|
|
1505
1578
|
/**
|
|
1506
1579
|
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?**
|
|
@@ -1518,6 +1591,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1518
1591
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
1519
1592
|
*/
|
|
1520
1593
|
async subjectReport(id, options = {}) {
|
|
1594
|
+
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1595
|
+
options ??= {};
|
|
1521
1596
|
const registry = this.requireRegistry("subjectReport");
|
|
1522
1597
|
requireScope(options, "subjectReport");
|
|
1523
1598
|
validateConcurrency(options.concurrency);
|
|
@@ -1539,7 +1614,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1539
1614
|
if (this.crbmSource !== void 0) {
|
|
1540
1615
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1541
1616
|
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${String(rec.token)}`;
|
|
1542
|
-
if (held !== listed) this.engine.invalidate(ref);
|
|
1617
|
+
if (rowVersionOfLive(held) !== listed) this.engine.invalidate(ref);
|
|
1543
1618
|
}
|
|
1544
1619
|
return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
1545
1620
|
}
|
|
@@ -1581,8 +1656,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1581
1656
|
* every generation below its new pointer — the one this rewrite is streaming, and the object this rewrite
|
|
1582
1657
|
* just wrote — so the loser can find its own inputs deleted mid-flight. That surfaces as an outcome rather than
|
|
1583
1658
|
* an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
|
|
1584
|
-
* that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger
|
|
1585
|
-
*
|
|
1659
|
+
* that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger. A tombstoned
|
|
1660
|
+
* segment is still searched: a cleartext object left under it that holds the id is deleted, with everything under
|
|
1661
|
+
* the tombstone, and the entry reads `erased: true`.
|
|
1586
1662
|
*
|
|
1587
1663
|
* **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
|
|
1588
1664
|
* another writer (a load, another erasure, or a rollback) moved the pointer mid-call, so **this call** did not
|
|
@@ -1611,6 +1687,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1611
1687
|
*/
|
|
1612
1688
|
async eraseSubject(id, options = {}) {
|
|
1613
1689
|
this.erasureEpoch++;
|
|
1690
|
+
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1691
|
+
options ??= {};
|
|
1614
1692
|
this.erasuresRunning++;
|
|
1615
1693
|
try {
|
|
1616
1694
|
return await this.eraseSubjectRun(id, options);
|
|
@@ -1639,7 +1717,6 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1639
1717
|
recs,
|
|
1640
1718
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1641
1719
|
async (rec) => {
|
|
1642
|
-
if (rec.status === "destroyed") return null;
|
|
1643
1720
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1644
1721
|
try {
|
|
1645
1722
|
const result = await eraseIdFromSegment(
|
|
@@ -1703,10 +1780,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1703
1780
|
* exists. It is `false` for a segment whose row was minted ahead of its first load (by `setRetention`) and
|
|
1704
1781
|
* for a `destroyed` tombstone, because a read answers empty in both cases.
|
|
1705
1782
|
*
|
|
1706
|
-
*
|
|
1783
|
+
* One state answers `true` where a read still gives you nothing: a torn restore (a live pointer whose object
|
|
1707
1784
|
* was deleted) makes reads of the object *throw* rather than answer empty, while a cold `count()` answers the number
|
|
1708
|
-
* its row records — `checkConsistency` is the call for that
|
|
1709
|
-
* a handle carrying an expired `expiresAt` reads empty by a rule that lives on the handle, not the row.
|
|
1785
|
+
* its row records — `checkConsistency` is the call for that.
|
|
1710
1786
|
*
|
|
1711
1787
|
* Not a lock: the answer can change the moment it returns. If it has to hold, use the fence built for that —
|
|
1712
1788
|
* `load`'s `guard`, or `expectFrom`/`expectToken` on a publish. Needs a storage backend.
|
|
@@ -1749,6 +1825,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1749
1825
|
* ```
|
|
1750
1826
|
*/
|
|
1751
1827
|
segments(options = {}) {
|
|
1828
|
+
checkOptions(options, "segments", CALL_KEYS.segments);
|
|
1829
|
+
options ??= {};
|
|
1752
1830
|
if (options.namespace !== void 0) {
|
|
1753
1831
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
1754
1832
|
refuseReservedNamespace(options.namespace);
|
|
@@ -1794,6 +1872,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1794
1872
|
*/
|
|
1795
1873
|
async rollback(ref, toGeneration, options = {}) {
|
|
1796
1874
|
validateSegmentRef(ref);
|
|
1875
|
+
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowForward"]);
|
|
1876
|
+
options ??= {};
|
|
1797
1877
|
const deps = this.lifecycleDeps("rollback");
|
|
1798
1878
|
try {
|
|
1799
1879
|
return await rollbackSegment(ref, toGeneration, deps, options);
|
|
@@ -1886,6 +1966,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1886
1966
|
*/
|
|
1887
1967
|
async load(ref, input, options = {}) {
|
|
1888
1968
|
validateSegmentRef(ref);
|
|
1969
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
1970
|
+
options ??= {};
|
|
1889
1971
|
const deps = this.lifecycleDeps("load");
|
|
1890
1972
|
try {
|
|
1891
1973
|
return await loadSegment(ref, bitmapAsLoadInput(input), deps, options);
|
|
@@ -1940,6 +2022,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1940
2022
|
*/
|
|
1941
2023
|
async dropSegment(ref, options) {
|
|
1942
2024
|
validateSegmentRef(ref);
|
|
2025
|
+
if (options === void 0 || options === null) {
|
|
2026
|
+
throw new ValidationError2("dropSegment: options { confirmSegment } are required");
|
|
2027
|
+
}
|
|
2028
|
+
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
1943
2029
|
const deps = this.lifecycleDeps("dropSegment");
|
|
1944
2030
|
try {
|
|
1945
2031
|
return await dropSegment(
|
|
@@ -2067,69 +2153,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2067
2153
|
* because retiring a segment deletes its storage objects. `now` defaults to the store's clock.
|
|
2068
2154
|
*/
|
|
2069
2155
|
async retireExpired(options = {}) {
|
|
2156
|
+
checkOptions(options, "retireExpired", CALL_KEYS.retireExpired, ["dryRun", "purgeTombstones"]);
|
|
2157
|
+
const opts = picked(options, CALL_KEYS.retireExpired);
|
|
2070
2158
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2071
2159
|
const result = await retireExpired(
|
|
2072
2160
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2073
|
-
{ ...
|
|
2161
|
+
{ ...opts, now: opts.now ?? this.clock.now() }
|
|
2074
2162
|
);
|
|
2075
|
-
if (
|
|
2163
|
+
if (opts.dryRun !== true) {
|
|
2076
2164
|
for (const entry of result.entries) {
|
|
2077
2165
|
this.engine.invalidate({ segment: entry.segment, namespace: entry.namespace });
|
|
2078
2166
|
}
|
|
2079
2167
|
}
|
|
2080
2168
|
return result;
|
|
2081
2169
|
}
|
|
2082
|
-
/**
|
|
2083
|
-
* **Remove the registry rows that releases before 0.12.0 left as `deleted: true` envelopes.** Those releases never
|
|
2084
|
-
* removed a row they deleted, and no later call removes it either: it is invisible to every read, and every full
|
|
2085
|
-
* listing of the registry still reads it (one GET each). This is the one call that removes them. It is an admin call,
|
|
2086
|
-
* not a sweep, and nothing schedules it: run it once after the last process on a release before 0.12.0 is gone.
|
|
2087
|
-
*
|
|
2088
|
-
* ```ts
|
|
2089
|
-
* const preview = await store.reapRegistryTombstones({ dryRun: true });
|
|
2090
|
-
* console.log(`would remove ${preview.wouldReap} of ${preview.examined} rows read`);
|
|
2091
|
-
*
|
|
2092
|
-
* const done = await store.reapRegistryTombstones({ confirmNoLegacyWriters: true });
|
|
2093
|
-
* if (done.limited) runAgain(); // `limit` (default 1,000) stopped the run with rows unread
|
|
2094
|
-
* ```
|
|
2095
|
-
*
|
|
2096
|
-
* **What it removes.** Only a row that is `deleted: true` and whose token carries no incarnation id, which is what a
|
|
2097
|
-
* release before 0.12.0 left, or a 0.12.0 or later release left by deleting a row born before 0.12.0 (both are safe to
|
|
2098
|
-
* remove), whatever its `status`. It never touches a live row, a `destroyed` row that is not `deleted` (a crypto-shred's
|
|
2099
|
-
* or a `dropSegment`'s tombstone: the attestation of an erasure, which it cannot tell from another), or a deleted row
|
|
2100
|
-
* with an incarnation id, and it never touches a generation. **So it does not clean a bucket completely**: the
|
|
2101
|
-
* `destroyed` tombstones `dropSegment` leaves stay, so do live rows written before 0.12.0, and so does a tombstone a
|
|
2102
|
-
* 0.12.0 or later release wrote while `conditionalDelete` was off (GCS by default, S3 on a custom endpoint): every full
|
|
2103
|
-
* listing still reads it, and this call neither removes it nor runs on such a store.
|
|
2104
|
-
* It also covers the library's own bookkeeping rows when `namespace` is not set, by the same rule. The default
|
|
2105
|
-
* namespace cannot be scoped to: leave `namespace` out to cover it.
|
|
2106
|
-
*
|
|
2107
|
-
* **The two things it asks.** `confirmNoLegacyWriters: true` is required for a real run (without it, a
|
|
2108
|
-
* `ValidationError`, and nothing is requested): your statement that no process on a release before 0.12.0 writes
|
|
2109
|
-
* this registry, since one that did would, re-creating a removed name, issue the removed row's tokens again. And the
|
|
2110
|
-
* registry must apply a delete under a precondition: where its `conditionalDelete` capability is off (GCS by default,
|
|
2111
|
-
* S3 on a custom endpoint) it throws `CapabilityError` before any request, dry run included. A registry that keeps
|
|
2112
|
-
* no such rows (the in-memory and local-filesystem ones) throws `UnsupportedError`.
|
|
2113
|
-
*
|
|
2114
|
-
* **Each removal is fenced** on the version it read, so a `create` that lands over the envelope first wins, the
|
|
2115
|
-
* delete is refused, and the row is counted as `skipped.raced` (which also counts a row already gone). That holds where
|
|
2116
|
-
* the backend applies the precondition, which auto-detection checks (S3 only on an AWS host). On an endpoint that
|
|
2117
|
-
* ignores `If-Match` (MinIO, fake-gcs-server), a `conditionalDelete: true` you set makes this an unfenced delete: run
|
|
2118
|
-
* it with every writer stopped. On a versioned bucket a delete leaves a delete marker and the earlier version, record
|
|
2119
|
-
* included, so the name is not erased from the bucket. A `create` that had already read the envelope and
|
|
2120
|
-
* whose write meets the removal throws `WriteConflictError`, which this library does not retry: retry the create, and
|
|
2121
|
-
* it finds no row and writes one.
|
|
2122
|
-
*
|
|
2123
|
-
* **Cost** over R rows read and E removed: `ceil(R / 1000)` LIST requests, R GETs and E DELETEs (a dry run, no
|
|
2124
|
-
* DELETEs). `limit` bounds the removals, and the run stops listing once it is spent. **A run is not resumable**: each
|
|
2125
|
-
* lists and reads from the start (R GETs whatever the `limit`), a dry run with a `limit` reports the same first rows
|
|
2126
|
-
* every time, so `wouldReap` is a total only when `limited` is `false`, and `limited` is `true` whenever the limit was
|
|
2127
|
-
* spent with keys left, removable or not. An unreadable object stops the run with an error naming its key, and
|
|
2128
|
-
* removes nothing more: it may be a newer release's row. Scope the run with `namespace` to get past one.
|
|
2129
|
-
*/
|
|
2130
|
-
async reapRegistryTombstones(options = {}) {
|
|
2131
|
-
return reapRegistryTombstones(this.requireRegistry("reapRegistryTombstones"), options);
|
|
2132
|
-
}
|
|
2133
2170
|
/**
|
|
2134
2171
|
* **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
|
|
2135
2172
|
* through a path this store did not perform itself.
|
|
@@ -2176,7 +2213,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2176
2213
|
* pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
|
|
2177
2214
|
* those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
|
|
2178
2215
|
*/
|
|
2179
|
-
async pinSegment(ref,
|
|
2216
|
+
async pinSegment(ref, leaseUntil, named) {
|
|
2180
2217
|
const crbm = this.crbmSource;
|
|
2181
2218
|
if (crbm === void 0) {
|
|
2182
2219
|
throw new UnsupportedError(
|
|
@@ -2236,9 +2273,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2236
2273
|
clock: this.clock,
|
|
2237
2274
|
metrics: this.metrics,
|
|
2238
2275
|
materialize: (dest, ids, op, options) => this.materialize(dest, ids, op, options),
|
|
2239
|
-
pinned: (r,
|
|
2276
|
+
pinned: (r, l, at2) => this.pinSegment(r, l, at2),
|
|
2240
2277
|
combineEngine: (handles) => this.engineForCombine(handles),
|
|
2241
|
-
expiresAt,
|
|
2242
2278
|
pinnedAt,
|
|
2243
2279
|
lease: hold
|
|
2244
2280
|
});
|
|
@@ -2344,6 +2380,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2344
2380
|
* `storage` and `registry` with a higher `maxScanSegments`.
|
|
2345
2381
|
*/
|
|
2346
2382
|
async checkConsistency(options = {}) {
|
|
2383
|
+
checkOptions(options, "checkConsistency", CALL_KEYS.checkConsistency, ["summaries"]);
|
|
2384
|
+
options ??= {};
|
|
2347
2385
|
const deps = this.lifecycleDeps("checkConsistency");
|
|
2348
2386
|
return runConsistencyCheck(
|
|
2349
2387
|
{ storage: deps.storage, registry: deps.registry, keystore: deps.keystore },
|
|
@@ -2363,15 +2401,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2363
2401
|
* CLI wraps this with a filesystem sink.
|
|
2364
2402
|
*/
|
|
2365
2403
|
async exportSegments(sink, options = {}) {
|
|
2404
|
+
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments);
|
|
2405
|
+
const opts = picked(options, CALL_KEYS.exportSegments);
|
|
2366
2406
|
const registry = this.requireRegistry("exportSegments");
|
|
2367
|
-
return runExport(this, registry, sink, {
|
|
2368
|
-
...options,
|
|
2369
|
-
codec: options.codec ?? roaringCodec
|
|
2370
|
-
});
|
|
2407
|
+
return runExport(this, registry, sink, { ...opts, codec: opts.codec ?? roaringCodec });
|
|
2371
2408
|
}
|
|
2372
2409
|
/**
|
|
2373
2410
|
* Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
|
|
2374
2411
|
* from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
2412
|
+
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
2413
|
+
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
2375
2414
|
*/
|
|
2376
2415
|
static estimateCost(input) {
|
|
2377
2416
|
return estimateCost(input);
|
|
@@ -2401,24 +2440,17 @@ var readOptions = (options) => ({
|
|
|
2401
2440
|
allowAbsentOperands: options.allowAbsentOperands
|
|
2402
2441
|
});
|
|
2403
2442
|
var NO_SEGMENTS = [];
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
}
|
|
2409
|
-
};
|
|
2443
|
+
function segmentList(list, what) {
|
|
2444
|
+
if (!Array.isArray(list)) throw new ValidationError2(`${what} must be an array of segments`);
|
|
2445
|
+
return list;
|
|
2446
|
+
}
|
|
2410
2447
|
var AS_IDS = {
|
|
2411
|
-
none: EMPTY_IDS,
|
|
2412
2448
|
failing,
|
|
2413
2449
|
guard: guardIds,
|
|
2414
2450
|
intersect: (engine, refs, opts) => withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts)),
|
|
2415
2451
|
union: (engine, refs, opts) => withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts)),
|
|
2416
2452
|
andNot: (engine, base, refs, opts) => withBatches(engine.andNot(base, refs, opts), () => engine.andNotBatches(base, refs, opts))
|
|
2417
2453
|
};
|
|
2418
|
-
var NO_CHUNKS = {
|
|
2419
|
-
async *[Symbol.asyncIterator]() {
|
|
2420
|
-
}
|
|
2421
|
-
};
|
|
2422
2454
|
var CombineChunks = class {
|
|
2423
2455
|
constructor(chunks) {
|
|
2424
2456
|
this.chunks = chunks;
|
|
@@ -2426,7 +2458,6 @@ var CombineChunks = class {
|
|
|
2426
2458
|
chunks;
|
|
2427
2459
|
};
|
|
2428
2460
|
var AS_CHUNKS = {
|
|
2429
|
-
none: NO_CHUNKS,
|
|
2430
2461
|
guard: guardChunks,
|
|
2431
2462
|
failing: (err) => ({
|
|
2432
2463
|
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
|
|
@@ -2436,39 +2467,14 @@ var AS_CHUNKS = {
|
|
|
2436
2467
|
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
2437
2468
|
};
|
|
2438
2469
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
heldOf = (operand) => operand.view;
|
|
2446
|
-
makeMemoryOperand = (view) => {
|
|
2447
|
-
mintingMemory = true;
|
|
2448
|
-
try {
|
|
2449
|
-
return new _MemoryOperand(view);
|
|
2450
|
-
} finally {
|
|
2451
|
-
mintingMemory = false;
|
|
2452
|
-
}
|
|
2453
|
-
};
|
|
2454
|
-
}
|
|
2455
|
-
/** Not constructible: a handle comes from {@link CloudRoaring.memory}. */
|
|
2456
|
-
constructor(view) {
|
|
2457
|
-
if (!mintingMemory) {
|
|
2458
|
-
throw new ValidationError2(
|
|
2459
|
-
"a MemoryOperand is not constructed directly; call `await store.memory(ids)`"
|
|
2460
|
-
);
|
|
2461
|
-
}
|
|
2462
|
-
this.view = view;
|
|
2463
|
-
}
|
|
2464
|
-
/**
|
|
2465
|
-
* Zero and drop the bytes it holds. Any later use throws {@link ValidationError}; releasing twice does nothing. A call
|
|
2466
|
-
* still reading it fails the outputs that read it.
|
|
2467
|
-
*/
|
|
2468
|
-
release() {
|
|
2469
|
-
this.view.chunks.release();
|
|
2470
|
+
async function timeOp(metrics, clock, name, fn) {
|
|
2471
|
+
const startedAt = clock.now();
|
|
2472
|
+
try {
|
|
2473
|
+
return await fn();
|
|
2474
|
+
} finally {
|
|
2475
|
+
metrics.onEvent({ kind: "op", name, ms: Math.max(0, clock.now() - startedAt) });
|
|
2470
2476
|
}
|
|
2471
|
-
}
|
|
2477
|
+
}
|
|
2472
2478
|
var makeSegment;
|
|
2473
2479
|
var minting = false;
|
|
2474
2480
|
var Segment = class _Segment {
|
|
@@ -2480,8 +2486,6 @@ var Segment = class _Segment {
|
|
|
2480
2486
|
materialize;
|
|
2481
2487
|
pinned;
|
|
2482
2488
|
combineEngine;
|
|
2483
|
-
/** Absolute epoch-ms deadline from {@link SegmentOptions.expiresAt}; `undefined` ⇒ this handle never expires. */
|
|
2484
|
-
expiresAt;
|
|
2485
2489
|
/**
|
|
2486
2490
|
* The generation this handle is held at, when it came from {@link Segment.pin}. Read by the store so a
|
|
2487
2491
|
* pinned handle passed as an **operand** is still read at its pin rather than live.
|
|
@@ -2506,9 +2510,7 @@ var Segment = class _Segment {
|
|
|
2506
2510
|
viewOf = (segment) => ({
|
|
2507
2511
|
ref: segment.ref,
|
|
2508
2512
|
pinnedAt: segment.pinnedAt,
|
|
2509
|
-
|
|
2510
|
-
leaseError: () => segment.leaseError(),
|
|
2511
|
-
expired: () => segment.expired()
|
|
2513
|
+
leaseError: () => segment.leaseError()
|
|
2512
2514
|
});
|
|
2513
2515
|
}
|
|
2514
2516
|
/**
|
|
@@ -2528,7 +2530,6 @@ var Segment = class _Segment {
|
|
|
2528
2530
|
this.materialize = parts.materialize;
|
|
2529
2531
|
this.pinned = parts.pinned;
|
|
2530
2532
|
this.combineEngine = parts.combineEngine;
|
|
2531
|
-
this.expiresAt = parts.expiresAt;
|
|
2532
2533
|
this.pinnedAt = parts.pinnedAt;
|
|
2533
2534
|
this.leaseHold = parts.lease;
|
|
2534
2535
|
if (parts.lease !== void 0) {
|
|
@@ -2618,12 +2619,12 @@ var Segment = class _Segment {
|
|
|
2618
2619
|
async pin(options) {
|
|
2619
2620
|
this.assertLeases([this]);
|
|
2620
2621
|
const leaseUntil = this.leaseUntilOf("pin", options);
|
|
2621
|
-
return this.pinned(this.ref,
|
|
2622
|
+
return this.pinned(this.ref, leaseUntil);
|
|
2622
2623
|
}
|
|
2623
2624
|
/**
|
|
2624
2625
|
* Hold what this pin names until `leaseUntil`, or `undefined` for a pin with no lease. Refuses an option it does not
|
|
2625
|
-
* know, a value that is not epoch-milliseconds in the future, one past the longest lease,
|
|
2626
|
-
*
|
|
2626
|
+
* know, a value that is not epoch-milliseconds in the future, and one past the longest lease, before any request is
|
|
2627
|
+
* made.
|
|
2627
2628
|
*/
|
|
2628
2629
|
leaseUntilOf(op, options) {
|
|
2629
2630
|
if (options === void 0 || options === null) return void 0;
|
|
@@ -2654,11 +2655,6 @@ var Segment = class _Segment {
|
|
|
2654
2655
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2655
2656
|
);
|
|
2656
2657
|
}
|
|
2657
|
-
if (this.expiresAt !== void 0 && until > this.expiresAt) {
|
|
2658
|
-
throw new ValidationError2(
|
|
2659
|
-
`${op}: leaseUntil outlasts this handle's own expiresAt, which would have it read empty while it still holds the generation`
|
|
2660
|
-
);
|
|
2661
|
-
}
|
|
2662
2658
|
return until;
|
|
2663
2659
|
}
|
|
2664
2660
|
/**
|
|
@@ -2683,7 +2679,7 @@ var Segment = class _Segment {
|
|
|
2683
2679
|
}
|
|
2684
2680
|
/**
|
|
2685
2681
|
* The error a read of this handle must throw, or `undefined` when it holds no lease or the lease is live. One property
|
|
2686
|
-
* read on a handle with no lease.
|
|
2682
|
+
* read on a handle with no lease. A read past a lease throws; it never reads empty.
|
|
2687
2683
|
*/
|
|
2688
2684
|
leaseError() {
|
|
2689
2685
|
const hold = this.leaseHold;
|
|
@@ -2705,9 +2701,21 @@ var Segment = class _Segment {
|
|
|
2705
2701
|
}
|
|
2706
2702
|
return void 0;
|
|
2707
2703
|
}
|
|
2704
|
+
/**
|
|
2705
|
+
* Refuse with a {@link ValidationError} anything among `handles` that is not a segment, a value a JavaScript caller
|
|
2706
|
+
* passed. It runs first in every combine, before the lease check reads the handles.
|
|
2707
|
+
*/
|
|
2708
|
+
assertSegments(handles) {
|
|
2709
|
+
for (const h of handles) {
|
|
2710
|
+
if (h instanceof _Segment || typeof h === "object" && h !== null && typeof h.leaseError === "function") {
|
|
2711
|
+
continue;
|
|
2712
|
+
}
|
|
2713
|
+
throw new ValidationError2("an operand must be a segment from store.segment()");
|
|
2714
|
+
}
|
|
2715
|
+
}
|
|
2708
2716
|
/**
|
|
2709
2717
|
* Throw the first lease error among `handles`: this handle and every operand and exclude of a call. It runs before
|
|
2710
|
-
*
|
|
2718
|
+
* the engine, so no operand, exclude or cached reader can answer past a lease.
|
|
2711
2719
|
*/
|
|
2712
2720
|
assertLeases(handles) {
|
|
2713
2721
|
for (const h of handles) {
|
|
@@ -2752,7 +2760,7 @@ var Segment = class _Segment {
|
|
|
2752
2760
|
);
|
|
2753
2761
|
}
|
|
2754
2762
|
const leaseUntil = this.leaseUntilOf("pinAt", options);
|
|
2755
|
-
return this.pinned(this.ref,
|
|
2763
|
+
return this.pinned(this.ref, leaseUntil, { generation, fingerprint });
|
|
2756
2764
|
}
|
|
2757
2765
|
/**
|
|
2758
2766
|
* An opaque string that names this handle's segment, namespace included: two handles of one segment have the
|
|
@@ -2764,49 +2772,17 @@ var Segment = class _Segment {
|
|
|
2764
2772
|
return segmentKey(this.ref);
|
|
2765
2773
|
}
|
|
2766
2774
|
/**
|
|
2767
|
-
*
|
|
2768
|
-
*
|
|
2769
|
-
*
|
|
2770
|
-
* Deliberately evaluated per call rather than cached: a long-lived handle created before its deadline must
|
|
2771
|
-
* start reading empty the moment the deadline passes, without the caller re-creating it.
|
|
2772
|
-
*/
|
|
2773
|
-
expired() {
|
|
2774
|
-
return this.expiresAt !== void 0 && this.clock.now() >= this.expiresAt;
|
|
2775
|
-
}
|
|
2776
|
-
/**
|
|
2777
|
-
* Refuse a materialisation that involves an **expired** handle. Called by the three `*Into` verbs only.
|
|
2778
|
-
*
|
|
2779
|
-
* On the read verbs an expired handle answers empty, which is the point of lazy expiry. On a *write* the same
|
|
2780
|
-
* rule would be destructive in a way nobody asks for: `a.intersectInto(dest, [b])` where `b`'s deadline has
|
|
2781
|
-
* quietly passed would, unrefused, replace `dest` with an **empty generation** — a wipe, reported as a successful
|
|
2782
|
-
* write, with the cause (a deadline on a handle somewhere) nowhere in the result. Reads degrade to empty; writes must not.
|
|
2783
|
-
*
|
|
2784
|
-
* So every handle in the call is checked, `dest` included: an expired `dest` does not change the bytes written,
|
|
2785
|
-
* but a caller who put a deadline on the thing they are writing into has said something contradictory and is
|
|
2786
|
-
* better told than guessed at. Open a handle without `expiresAt` to write, or drop the deadline.
|
|
2787
|
-
*
|
|
2788
|
-
* (The broader guard — refusing to publish an empty or implausible generation over a non-empty one, with an
|
|
2789
|
-
* `allowEmpty` override — covers these verbs too: they route through the same guarded write path as
|
|
2790
|
-
* {@link CloudRoaring.load}. The two stay separate because they differ in kind. That one is a REPORTED
|
|
2791
|
-
* refusal a caller may legitimately override; an expired handle is a wiring mistake, so it THROWS, before
|
|
2792
|
-
* any object is written — and `allowEmpty: true` does not reach it.)
|
|
2775
|
+
* Refuse an `*Into` whose handles cannot be read, before any object is written: anything that is not a segment
|
|
2776
|
+
* ({@link ValidationError}), and a handle (this one, `dest` or an operand) whose lease has ended or was released
|
|
2777
|
+
* ({@link LeaseExpiredError}). Called by the three `*Into` verbs only.
|
|
2793
2778
|
*
|
|
2794
2779
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2795
2780
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
2796
2781
|
* awaiting.
|
|
2797
2782
|
*/
|
|
2798
|
-
|
|
2783
|
+
assertIntoHandles(dest, operands) {
|
|
2784
|
+
this.assertSegments([dest, ...operands]);
|
|
2799
2785
|
this.assertLeases([this, dest, ...operands]);
|
|
2800
|
-
const stale = [];
|
|
2801
|
-
for (const seg of [this, dest, ...operands]) {
|
|
2802
|
-
if (seg.expired())
|
|
2803
|
-
stale.push(seg.ref.namespace ? `${seg.ref.namespace}/${seg.ref.segment}` : seg.ref.segment);
|
|
2804
|
-
}
|
|
2805
|
-
if (stale.length > 0) {
|
|
2806
|
-
throw new ValidationError2(
|
|
2807
|
-
`${op}: refusing to publish a generation while these handles have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired handle reads as empty, so this would write an empty (or short) generation over "${dest.ref.segment}". Open the handles without \`expiresAt\` if you meant to materialise, or drop the deadline.`
|
|
2808
|
-
);
|
|
2809
|
-
}
|
|
2810
2786
|
}
|
|
2811
2787
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2812
2788
|
guarded(out, handles, result) {
|
|
@@ -2829,7 +2805,6 @@ var Segment = class _Segment {
|
|
|
2829
2805
|
has(id) {
|
|
2830
2806
|
const lease = this.leaseError();
|
|
2831
2807
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2832
|
-
if (this.expired()) return Promise.resolve(false);
|
|
2833
2808
|
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
2834
2809
|
}
|
|
2835
2810
|
/**
|
|
@@ -2853,15 +2828,14 @@ var Segment = class _Segment {
|
|
|
2853
2828
|
count() {
|
|
2854
2829
|
const lease = this.leaseError();
|
|
2855
2830
|
if (lease !== void 0) return Promise.reject(lease);
|
|
2856
|
-
if (this.expired()) return Promise.resolve(0);
|
|
2857
2831
|
return this.timed("count", () => this.engine.count(this.ref));
|
|
2858
2832
|
}
|
|
2859
2833
|
/**
|
|
2860
2834
|
* What the generation this handle reads is, from one resolution: its number, its id count and the metadata it
|
|
2861
2835
|
* was loaded with (absent when it has none). It is what answers {@link Segment.count}, so the three describe one
|
|
2862
2836
|
* generation and cannot straddle a publish. One registry read when cold, none while warm, and none on a pinned
|
|
2863
|
-
* handle, which answers for the generation it pinned. A segment with no generation
|
|
2864
|
-
*
|
|
2837
|
+
* handle, which answers for the generation it pinned. A segment with no generation answers
|
|
2838
|
+
* `{ generation: null, cardinality: 0 }`.
|
|
2865
2839
|
*
|
|
2866
2840
|
* Trust is as for `count()`: the registry row's word, not confirmed against the object until the object is
|
|
2867
2841
|
* opened, when a disagreement makes this process stop using that row's summary.
|
|
@@ -2872,7 +2846,6 @@ var Segment = class _Segment {
|
|
|
2872
2846
|
*/
|
|
2873
2847
|
async stat() {
|
|
2874
2848
|
this.assertLeases([this]);
|
|
2875
|
-
if (this.expired()) return { generation: null, cardinality: 0 };
|
|
2876
2849
|
return this.engine.stat(this.ref);
|
|
2877
2850
|
}
|
|
2878
2851
|
/**
|
|
@@ -2887,13 +2860,13 @@ var Segment = class _Segment {
|
|
|
2887
2860
|
* ```
|
|
2888
2861
|
*
|
|
2889
2862
|
* Each bound is optional and an integer in `0..4294967295`; a bad one throws {@link ValidationError} when the
|
|
2890
|
-
* stream is first read. `after >= through` is an empty range, which reads nothing.
|
|
2891
|
-
* without checking its options, as every read of one does.
|
|
2863
|
+
* stream is first read. `after >= through` is an empty range, which reads nothing.
|
|
2892
2864
|
*/
|
|
2893
2865
|
iterate(options) {
|
|
2894
2866
|
const lease = this.leaseError();
|
|
2895
2867
|
if (lease !== void 0) return failing(lease);
|
|
2896
|
-
|
|
2868
|
+
const bad = optionsError(options, "iterate", CALL_KEYS.range);
|
|
2869
|
+
if (bad !== void 0) return failing(bad);
|
|
2897
2870
|
const range = options == null ? void 0 : rangeOf(options);
|
|
2898
2871
|
const none = range === void 0 || range.after === void 0 && range.through === void 0;
|
|
2899
2872
|
const { engine, ref } = this;
|
|
@@ -2926,7 +2899,7 @@ var Segment = class _Segment {
|
|
|
2926
2899
|
*
|
|
2927
2900
|
* A live handle is refused with {@link UnsupportedError} at the first read, because its counts and its chunks could
|
|
2928
2901
|
* come from two generations and name the wrong id. `n` that is not a positive integer throws
|
|
2929
|
-
* {@link ValidationError} at the first read, as does a bad bound.
|
|
2902
|
+
* {@link ValidationError} at the first read, as does a bad bound.
|
|
2930
2903
|
*
|
|
2931
2904
|
* ```ts
|
|
2932
2905
|
* const audience = await store.segment('audience').pin();
|
|
@@ -2937,7 +2910,8 @@ var Segment = class _Segment {
|
|
|
2937
2910
|
everyNth(n, options) {
|
|
2938
2911
|
const lease = this.leaseError();
|
|
2939
2912
|
if (lease !== void 0) return failing(lease);
|
|
2940
|
-
|
|
2913
|
+
const bad = optionsError(options, "everyNth", CALL_KEYS.range);
|
|
2914
|
+
if (bad !== void 0) return failing(bad);
|
|
2941
2915
|
if (this.pinnedAt === void 0) {
|
|
2942
2916
|
return failing(
|
|
2943
2917
|
new UnsupportedError(
|
|
@@ -2975,30 +2949,8 @@ var Segment = class _Segment {
|
|
|
2975
2949
|
*/
|
|
2976
2950
|
excludesOf(options) {
|
|
2977
2951
|
const exclude = options?.exclude;
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
/**
|
|
2981
|
-
* The refusal for a combine whose **exclusion** has expired, or `undefined` when none has. Unlike an expired
|
|
2982
|
-
* operand, which reads empty, an exclusion is a suppression or opt-out list: left out silently, the result
|
|
2983
|
-
* would include the very ids it was passed to remove. So it is refused with a {@link ValidationError} naming each
|
|
2984
|
-
* expired exclusion, as the `*Into` verbs refuse an expired handle.
|
|
2985
|
-
*
|
|
2986
|
-
* It is judged against the injected clock when the combine is called, before the engine is asked for anything:
|
|
2987
|
-
* no request is made, and an expired exclusion that names a segment that does not exist is refused as expired,
|
|
2988
|
-
* not as an absent operand. It runs ahead of the expired-operand shortcuts, so a combine that would read empty
|
|
2989
|
-
* anyway still refuses a lapsed exclusion. A stream already being read is not re-checked when an exclusion
|
|
2990
|
-
* expires part-way, as an operand expiring part-way does not stop it either.
|
|
2991
|
-
*/
|
|
2992
|
-
expiredExcludes(op, excludes) {
|
|
2993
|
-
let stale;
|
|
2994
|
-
for (const e of excludes) {
|
|
2995
|
-
if (!e.expired()) continue;
|
|
2996
|
-
(stale ??= []).push(e.ref.namespace ? `${e.ref.namespace}/${e.ref.segment}` : e.ref.segment);
|
|
2997
|
-
}
|
|
2998
|
-
if (stale === void 0) return void 0;
|
|
2999
|
-
return new ValidationError2(
|
|
3000
|
-
`${op}: refusing to read while these exclusions have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired exclusion would subtract nothing, so the result would include the ids it was passed to remove. Renew the exclusion's \`expiresAt\`, open it without one, or leave it out of the call.`
|
|
3001
|
-
);
|
|
2952
|
+
if (exclude == null) return NO_SEGMENTS;
|
|
2953
|
+
return segmentList(exclude, "`exclude`").length === 0 ? NO_SEGMENTS : exclude;
|
|
3002
2954
|
}
|
|
3003
2955
|
/**
|
|
3004
2956
|
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
@@ -3010,19 +2962,21 @@ var Segment = class _Segment {
|
|
|
3010
2962
|
* override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
3011
2963
|
*/
|
|
3012
2964
|
intersect(others, options) {
|
|
3013
|
-
|
|
2965
|
+
const bad = optionsError(options, "intersect", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
2966
|
+
if (bad !== void 0) return failing(bad);
|
|
2967
|
+
return this.intersectAs(AS_IDS, others, options ?? void 0);
|
|
3014
2968
|
}
|
|
3015
2969
|
intersectAs(out, others, options) {
|
|
3016
|
-
|
|
3017
|
-
|
|
2970
|
+
let exclude;
|
|
2971
|
+
let handles;
|
|
3018
2972
|
try {
|
|
2973
|
+
exclude = this.excludesOf(options);
|
|
2974
|
+
handles = [this, ...segmentList(others, "intersect: `others`"), ...exclude];
|
|
2975
|
+
this.assertSegments(handles);
|
|
3019
2976
|
this.assertLeases(handles);
|
|
3020
2977
|
} catch (err) {
|
|
3021
2978
|
return out.failing(err);
|
|
3022
2979
|
}
|
|
3023
|
-
const refused = this.expiredExcludes("intersect", exclude);
|
|
3024
|
-
if (refused) return out.failing(refused);
|
|
3025
|
-
if (this.expired() || others.some((o) => o.expired())) return out.none;
|
|
3026
2980
|
let engine;
|
|
3027
2981
|
try {
|
|
3028
2982
|
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
@@ -3046,16 +3000,21 @@ var Segment = class _Segment {
|
|
|
3046
3000
|
* write is **refused and reported**: `published: false`, `reason: 'empty'`, and `dest` keeps what it had.
|
|
3047
3001
|
* Pass `allowEmpty: true` when emptying the destination is the point.
|
|
3048
3002
|
*
|
|
3049
|
-
* `guard` adds the same plausibility bounds `load()` takes — `minCardinality` and `
|
|
3003
|
+
* `guard` adds the same plausibility bounds `load()` takes — `minCardinality`, `minRetained` and `maxGrowth` — judged
|
|
3050
3004
|
* against what `dest` held before. A refusal is **reported, not thrown**, exactly as on `load()`; branch on
|
|
3051
3005
|
* `published`. A lost race is the one outcome that still throws ({@link WriteConflictError}), because a
|
|
3052
3006
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3053
3007
|
*
|
|
3054
|
-
* A call involving
|
|
3008
|
+
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
3055
3009
|
*/
|
|
3056
3010
|
async intersectInto(dest, others, options) {
|
|
3057
|
-
|
|
3058
|
-
|
|
3011
|
+
checkOptions(options, "intersectInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3012
|
+
const operands = [
|
|
3013
|
+
...segmentList(others, "intersectInto: `others`"),
|
|
3014
|
+
...this.excludesOf(options)
|
|
3015
|
+
];
|
|
3016
|
+
this.assertIntoHandles(dest, operands);
|
|
3017
|
+
this.combineEngine([this, ...operands]);
|
|
3059
3018
|
return this.timed(
|
|
3060
3019
|
"intersectInto",
|
|
3061
3020
|
() => this.materialize(
|
|
@@ -3077,27 +3036,21 @@ var Segment = class _Segment {
|
|
|
3077
3036
|
* segment once (`unionInto`, or a load) is the cheaper shape.
|
|
3078
3037
|
*/
|
|
3079
3038
|
union(others, options) {
|
|
3080
|
-
|
|
3039
|
+
const bad = optionsError(options, "union", CALL_KEYS.combine, ["allowAbsentOperands"]);
|
|
3040
|
+
if (bad !== void 0) return failing(bad);
|
|
3041
|
+
return this.unionAs(AS_IDS, others, options ?? void 0);
|
|
3081
3042
|
}
|
|
3082
3043
|
unionAs(out, others, options) {
|
|
3083
|
-
|
|
3084
|
-
|
|
3044
|
+
let exclude;
|
|
3045
|
+
let handles;
|
|
3085
3046
|
try {
|
|
3047
|
+
exclude = this.excludesOf(options);
|
|
3048
|
+
handles = [this, ...segmentList(others, "union: `others`"), ...exclude];
|
|
3049
|
+
this.assertSegments(handles);
|
|
3086
3050
|
this.assertLeases(handles);
|
|
3087
3051
|
} catch (err) {
|
|
3088
3052
|
return out.failing(err);
|
|
3089
3053
|
}
|
|
3090
|
-
const refused = this.expiredExcludes("union", exclude);
|
|
3091
|
-
if (refused) return out.failing(refused);
|
|
3092
|
-
const live = others.filter((o) => !o.expired());
|
|
3093
|
-
if (this.expired()) {
|
|
3094
|
-
if (live.length === 0) return out.none;
|
|
3095
|
-
return live[0].unionAs(out, live.slice(1), options);
|
|
3096
|
-
}
|
|
3097
|
-
if (live.length === 0 && others.length > 0) {
|
|
3098
|
-
return exclude.length > 0 ? this.andNotAs(out, [...exclude], options) : this.unionAs(out, [], options);
|
|
3099
|
-
}
|
|
3100
|
-
if (live.length !== others.length) return this.unionAs(out, live, options);
|
|
3101
3054
|
let engine;
|
|
3102
3055
|
try {
|
|
3103
3056
|
engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
|
|
@@ -3110,8 +3063,10 @@ var Segment = class _Segment {
|
|
|
3110
3063
|
}
|
|
3111
3064
|
/** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3112
3065
|
async unionInto(dest, others, options) {
|
|
3113
|
-
|
|
3114
|
-
|
|
3066
|
+
checkOptions(options, "unionInto", CALL_KEYS.into, ["allowEmpty", "allowAbsentOperands"]);
|
|
3067
|
+
const operands = [...segmentList(others, "unionInto: `others`"), ...this.excludesOf(options)];
|
|
3068
|
+
this.assertIntoHandles(dest, operands);
|
|
3069
|
+
this.combineEngine([this, ...operands]);
|
|
3115
3070
|
return this.timed(
|
|
3116
3071
|
"unionInto",
|
|
3117
3072
|
() => this.materialize(
|
|
@@ -3130,26 +3085,23 @@ var Segment = class _Segment {
|
|
|
3130
3085
|
* suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
|
|
3131
3086
|
* opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
|
|
3132
3087
|
*
|
|
3133
|
-
* An expired handle in `excludes` is refused, not skipped: the stream throws {@link ValidationError} naming it,
|
|
3134
|
-
* before any request is made, because a lapsed suppression list would otherwise let through the ids it was
|
|
3135
|
-
* passed to remove. An expired `this` is empty. `exclude` on `intersect` and `union` follows the same rule.
|
|
3136
|
-
*
|
|
3137
3088
|
* To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
|
|
3138
3089
|
* the suppression folds into the same pass rather than materializing an intermediate segment first.
|
|
3139
3090
|
*/
|
|
3140
3091
|
andNot(excludes, options) {
|
|
3141
|
-
|
|
3092
|
+
const bad = optionsError(options, "andNot", CALL_KEYS.andNot, ["allowAbsentOperands"]);
|
|
3093
|
+
if (bad !== void 0) return failing(bad);
|
|
3094
|
+
return this.andNotAs(AS_IDS, excludes, options ?? void 0);
|
|
3142
3095
|
}
|
|
3143
3096
|
andNotAs(out, excludes, options) {
|
|
3144
|
-
|
|
3097
|
+
let handles;
|
|
3145
3098
|
try {
|
|
3099
|
+
handles = [this, ...segmentList(excludes, "andNot: `excludes`")];
|
|
3100
|
+
this.assertSegments(handles);
|
|
3146
3101
|
this.assertLeases(handles);
|
|
3147
3102
|
} catch (err) {
|
|
3148
3103
|
return out.failing(err);
|
|
3149
3104
|
}
|
|
3150
|
-
const refused = this.expiredExcludes("andNot", excludes);
|
|
3151
|
-
if (refused) return out.failing(refused);
|
|
3152
|
-
if (this.expired()) return out.none;
|
|
3153
3105
|
let engine;
|
|
3154
3106
|
try {
|
|
3155
3107
|
engine = this.combineEngine([this, ...excludes]) ?? this.engine;
|
|
@@ -3163,7 +3115,11 @@ var Segment = class _Segment {
|
|
|
3163
3115
|
}
|
|
3164
3116
|
/** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
|
|
3165
3117
|
async andNotInto(dest, excludes, options) {
|
|
3166
|
-
|
|
3118
|
+
checkOptions(options, "andNotInto", CALL_KEYS.andNotInto, [
|
|
3119
|
+
"allowEmpty",
|
|
3120
|
+
"allowAbsentOperands"
|
|
3121
|
+
]);
|
|
3122
|
+
this.assertIntoHandles(dest, segmentList(excludes, "andNotInto: `excludes`"));
|
|
3167
3123
|
this.combineEngine([this, ...excludes]);
|
|
3168
3124
|
return this.timed(
|
|
3169
3125
|
"andNotInto",
|
|
@@ -3181,13 +3137,21 @@ var Segment = class _Segment {
|
|
|
3181
3137
|
* storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
|
|
3182
3138
|
* refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
|
|
3183
3139
|
* the lose-zone).
|
|
3140
|
+
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
3141
|
+
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
3184
3142
|
*/
|
|
3185
3143
|
async costReport(options) {
|
|
3144
|
+
checkOptions(options, "costReport", CALL_KEYS.costReport);
|
|
3145
|
+
const shape = options?.workload;
|
|
3146
|
+
if (shape !== void 0 && shape !== null && (typeof shape !== "object" || Array.isArray(shape))) {
|
|
3147
|
+
throw new ValidationError2("costReport: workload must be an object");
|
|
3148
|
+
}
|
|
3186
3149
|
this.assertLeases([this]);
|
|
3187
3150
|
const canMeasure = this.engine.supportsStorageSize;
|
|
3188
3151
|
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
3189
3152
|
const refreshMs = this.engine.pointerRefreshMs;
|
|
3190
|
-
const
|
|
3153
|
+
const given = options?.workload === void 0 || options.workload === null ? void 0 : picked(options.workload, CALL_KEYS.workload);
|
|
3154
|
+
const workload = refreshMs === void 0 || given?.genTtlMs !== void 0 ? given : { ...given, genTtlMs: refreshMs };
|
|
3191
3155
|
return groundedReport({
|
|
3192
3156
|
storageBytes: size?.sizeBytes ?? 0,
|
|
3193
3157
|
grounded: canMeasure,
|
|
@@ -3208,7 +3172,7 @@ function parseConfig(env) {
|
|
|
3208
3172
|
if (out === void 0 || out === "") {
|
|
3209
3173
|
throw new Error("CR_EXPORT_OUT is required (the output directory for the dump)");
|
|
3210
3174
|
}
|
|
3211
|
-
const format = env.CR_EXPORT_FORMAT
|
|
3175
|
+
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3212
3176
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3213
3177
|
throw new Error(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3214
3178
|
}
|
|
@@ -3269,14 +3233,29 @@ function fsSink(out) {
|
|
|
3269
3233
|
}
|
|
3270
3234
|
};
|
|
3271
3235
|
}
|
|
3236
|
+
async function requireStoreRoot(root) {
|
|
3237
|
+
let entries;
|
|
3238
|
+
try {
|
|
3239
|
+
entries = await readdir(root);
|
|
3240
|
+
} catch {
|
|
3241
|
+
throw new ValidationError3("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3242
|
+
}
|
|
3243
|
+
if (entries.length === 0) {
|
|
3244
|
+
throw new ValidationError3(
|
|
3245
|
+
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3246
|
+
);
|
|
3247
|
+
}
|
|
3248
|
+
}
|
|
3272
3249
|
var log = (obj) => {
|
|
3273
3250
|
process.stdout.write(`${JSON.stringify(obj)}
|
|
3274
3251
|
`);
|
|
3275
3252
|
};
|
|
3276
3253
|
async function main(env = process.env, now = () => Date.now()) {
|
|
3277
3254
|
const config = parseConfig(env);
|
|
3255
|
+
await requireStoreRoot(config.root);
|
|
3278
3256
|
const storage = new LocalFsStorage(config.root);
|
|
3279
3257
|
const store = new CloudRoaring({ storage });
|
|
3258
|
+
await rm(join(config.out, "manifest.json"), { force: true });
|
|
3280
3259
|
const manifest = await store.exportSegments(fsSink(config.out), {
|
|
3281
3260
|
format: config.format,
|
|
3282
3261
|
namespace: config.namespace
|