@cloudbitmaps/roaring 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +2 -2
- package/PRIVACY.md +24 -18
- package/README.md +28 -11
- package/dist/bin/export-segments.js +1980 -319
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +263 -61
- package/dist/index.js +1996 -315
- package/dist/index.js.map +4 -4
- package/dist/roaring-codec.d.ts +36 -105
- package/package.json +4 -5
|
@@ -21,17 +21,20 @@ import {
|
|
|
21
21
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
22
22
|
SegmentEngine,
|
|
23
23
|
LEASE_SKEW_MS,
|
|
24
|
+
BudgetExceededError,
|
|
24
25
|
LeaseExpiredError,
|
|
25
26
|
MAX_LEASE_MS,
|
|
26
27
|
NotFoundError,
|
|
27
28
|
UnsupportedError as UnsupportedError2,
|
|
28
|
-
ValidationError as
|
|
29
|
+
ValidationError as ValidationError6,
|
|
30
|
+
errorKind,
|
|
29
31
|
WriteConflictError,
|
|
30
32
|
checkBudget,
|
|
31
33
|
collectWithinBudget,
|
|
32
34
|
excludingReservedRows,
|
|
33
35
|
dropSegment,
|
|
34
36
|
eraseIdFromSegment,
|
|
37
|
+
findIdUnderTombstone,
|
|
35
38
|
judgeLoad,
|
|
36
39
|
loadSegment,
|
|
37
40
|
loadSegmentChunks,
|
|
@@ -52,7 +55,12 @@ import {
|
|
|
52
55
|
splitId,
|
|
53
56
|
takeLease
|
|
54
57
|
} from "@cloudbitmaps/core";
|
|
55
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
pointerIdOf,
|
|
60
|
+
requireBool,
|
|
61
|
+
requirePositiveInt,
|
|
62
|
+
validateSegmentRef
|
|
63
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
56
64
|
|
|
57
65
|
// src/open-charging-storage.ts
|
|
58
66
|
var OpenChargingStorage = class {
|
|
@@ -68,6 +76,7 @@ var OpenChargingStorage = class {
|
|
|
68
76
|
}
|
|
69
77
|
inner;
|
|
70
78
|
charge;
|
|
79
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
71
80
|
free = /* @__PURE__ */ new Set();
|
|
72
81
|
capabilities() {
|
|
73
82
|
return this.inner.capabilities();
|
|
@@ -80,9 +89,15 @@ var OpenChargingStorage = class {
|
|
|
80
89
|
getRange(key, offset, length) {
|
|
81
90
|
return this.inner.getRange(key, offset, length);
|
|
82
91
|
}
|
|
83
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
94
|
+
* generation is charged at its first open only.
|
|
95
|
+
*/
|
|
84
96
|
async getTail(key, maxBytes) {
|
|
85
|
-
if (!this.free.has(key.generation))
|
|
97
|
+
if (!this.free.has(key.generation)) {
|
|
98
|
+
this.charge();
|
|
99
|
+
this.free.add(key.generation);
|
|
100
|
+
}
|
|
86
101
|
return this.inner.getTail(key, maxBytes);
|
|
87
102
|
}
|
|
88
103
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -96,7 +111,12 @@ var OpenChargingStorage = class {
|
|
|
96
111
|
|
|
97
112
|
// src/index.ts
|
|
98
113
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
99
|
-
import {
|
|
114
|
+
import {
|
|
115
|
+
destroySegment as coreDestroySegment,
|
|
116
|
+
eraseNamespace as coreEraseNamespace
|
|
117
|
+
} from "@cloudbitmaps/core";
|
|
118
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
119
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
100
120
|
|
|
101
121
|
// src/reserved-namespace.ts
|
|
102
122
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -110,8 +130,10 @@ function refuseReservedNamespace(namespace) {
|
|
|
110
130
|
}
|
|
111
131
|
|
|
112
132
|
// src/roaring-codec.ts
|
|
113
|
-
import
|
|
114
|
-
|
|
133
|
+
import { ValidationError as ValidationError3, decodeSerialized } from "@cloudbitmaps/core";
|
|
134
|
+
|
|
135
|
+
// src/ts-roaring/bitmap.ts
|
|
136
|
+
import { IntegrityError as IntegrityError3, ValidationError as ValidationError2 } from "@cloudbitmaps/core";
|
|
115
137
|
|
|
116
138
|
// src/portable/layout.ts
|
|
117
139
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -129,45 +151,10 @@ function overrun(bytes, offset, length, what) {
|
|
|
129
151
|
`portable roaring: ${what} needs bytes [${offset}, ${offset + length}) of a ${bytes.byteLength}-byte buffer`
|
|
130
152
|
);
|
|
131
153
|
}
|
|
132
|
-
function
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
var SINGLE_RUN_HEADER_BYTES = 9;
|
|
137
|
-
function* containerPayloads(bytes) {
|
|
138
|
-
if (bytes.byteLength === 0) return;
|
|
139
|
-
const { view, count, runFlagsAt, descriptiveAt, bodiesAt } = header(bytes);
|
|
140
|
-
let pos = bodiesAt;
|
|
141
|
-
let previousKey = -1;
|
|
142
|
-
for (let i = 0; i < count; i++) {
|
|
143
|
-
const key = view.getUint16(descriptiveAt + i * 4, true);
|
|
144
|
-
if (key <= previousKey) {
|
|
145
|
-
throw new IntegrityError(
|
|
146
|
-
`portable roaring: container keys are not strictly ascending (${previousKey} then ${key} at index ${i})`
|
|
147
|
-
);
|
|
148
|
-
}
|
|
149
|
-
previousKey = key;
|
|
150
|
-
const cardinality = view.getUint16(descriptiveAt + i * 4 + 2, true) + 1;
|
|
151
|
-
const isRun = runFlagsAt >= 0 && (bytes[runFlagsAt + (i >>> 3)] & 1 << (i & 7)) !== 0;
|
|
152
|
-
const kind = isRun ? RUN : cardinality > ARRAY_MAX_CARDINALITY ? BITMAP : ARRAY;
|
|
153
|
-
const size = checkContainer(bytes, view, pos, kind, cardinality, i);
|
|
154
|
-
const headerBytes = isRun ? SINGLE_RUN_HEADER_BYTES : SINGLE_HEADER_BYTES;
|
|
155
|
-
const payload = new Uint8Array(headerBytes + size);
|
|
156
|
-
const out = new DataView(payload.buffer);
|
|
157
|
-
if (isRun) {
|
|
158
|
-
out.setUint32(0, SERIAL_COOKIE, true);
|
|
159
|
-
payload[4] = 1;
|
|
160
|
-
out.setUint16(7, cardinality - 1, true);
|
|
161
|
-
} else {
|
|
162
|
-
out.setUint32(0, SERIAL_COOKIE_NO_RUNCONTAINER, true);
|
|
163
|
-
out.setUint32(4, 1, true);
|
|
164
|
-
out.setUint16(10, cardinality - 1, true);
|
|
165
|
-
out.setUint32(12, SINGLE_HEADER_BYTES, true);
|
|
166
|
-
}
|
|
167
|
-
payload.set(bytes.subarray(pos, pos + size), headerBytes);
|
|
168
|
-
pos += size;
|
|
169
|
-
yield { chunkKey: key, payload, cardinality };
|
|
170
|
-
}
|
|
154
|
+
function parsePortableLayout(bytes) {
|
|
155
|
+
const containers = [];
|
|
156
|
+
walk(bytes, containers);
|
|
157
|
+
return containers;
|
|
171
158
|
}
|
|
172
159
|
function header(bytes) {
|
|
173
160
|
const length = bytes.byteLength;
|
|
@@ -318,163 +305,1579 @@ function popcount(view, at) {
|
|
|
318
305
|
return bits;
|
|
319
306
|
}
|
|
320
307
|
|
|
321
|
-
// src/roaring
|
|
322
|
-
var
|
|
323
|
-
var
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
308
|
+
// src/ts-roaring/containers.ts
|
|
309
|
+
var ARRAY2 = 0;
|
|
310
|
+
var BITSET = 1;
|
|
311
|
+
var RUN2 = 2;
|
|
312
|
+
var WORDS = 2048;
|
|
313
|
+
var BITSET_BYTES = 8192;
|
|
314
|
+
var GALLOP_SKEW = 16;
|
|
315
|
+
var LITTLE_ENDIAN = new Uint8Array(Uint16Array.of(1).buffer)[0] === 1;
|
|
316
|
+
var Container = class {
|
|
317
|
+
constructor(kind, data, off, len, card, shared) {
|
|
318
|
+
this.kind = kind;
|
|
319
|
+
this.data = data;
|
|
320
|
+
this.off = off;
|
|
321
|
+
this.len = len;
|
|
322
|
+
this.card = card;
|
|
323
|
+
this.shared = shared;
|
|
324
|
+
}
|
|
325
|
+
kind;
|
|
326
|
+
data;
|
|
327
|
+
off;
|
|
328
|
+
len;
|
|
329
|
+
card;
|
|
330
|
+
shared;
|
|
331
|
+
};
|
|
332
|
+
function lows16(c) {
|
|
333
|
+
return c.data;
|
|
334
|
+
}
|
|
335
|
+
function words32(c) {
|
|
336
|
+
return c.data;
|
|
337
|
+
}
|
|
338
|
+
function arrayContainer(values, card) {
|
|
339
|
+
return new Container(ARRAY2, values, 0, card, card, false);
|
|
340
|
+
}
|
|
341
|
+
function bitsetContainer(words, card) {
|
|
342
|
+
return new Container(BITSET, words, 0, 0, card, false);
|
|
343
|
+
}
|
|
344
|
+
function runContainer(pairs, runs, card) {
|
|
345
|
+
return new Container(RUN2, pairs, 0, runs, card, false);
|
|
346
|
+
}
|
|
347
|
+
var valueScratch = new Uint16Array(8192);
|
|
348
|
+
var pairScratch = new Uint16Array(8192);
|
|
349
|
+
var SCRATCH_KEEP = 65536;
|
|
350
|
+
function values16(n) {
|
|
351
|
+
if (valueScratch.length >= n) return valueScratch;
|
|
352
|
+
if (n > SCRATCH_KEEP) return new Uint16Array(n);
|
|
353
|
+
valueScratch = new Uint16Array(Math.max(n, Math.min(2 * valueScratch.length, SCRATCH_KEEP)));
|
|
354
|
+
return valueScratch;
|
|
355
|
+
}
|
|
356
|
+
function pairs16(n) {
|
|
357
|
+
if (pairScratch.length >= n) return pairScratch;
|
|
358
|
+
if (n > SCRATCH_KEEP) return new Uint16Array(n);
|
|
359
|
+
pairScratch = new Uint16Array(Math.max(n, Math.min(2 * pairScratch.length, SCRATCH_KEEP)));
|
|
360
|
+
return pairScratch;
|
|
361
|
+
}
|
|
362
|
+
function arrayOfScratch(scratch, n) {
|
|
363
|
+
return n === 0 ? null : arrayContainer(scratch.slice(0, n), n);
|
|
364
|
+
}
|
|
365
|
+
function popcount32(word) {
|
|
366
|
+
let w = word - (word >>> 1 & 1431655765);
|
|
367
|
+
w = (w & 858993459) + (w >>> 2 & 858993459);
|
|
368
|
+
return Math.imul(w + (w >>> 4) & 252645135, 16843009) >>> 24;
|
|
369
|
+
}
|
|
370
|
+
function popcountWords(d) {
|
|
371
|
+
let total = 0;
|
|
372
|
+
let ones = 0;
|
|
373
|
+
let twos = 0;
|
|
374
|
+
let fours = 0;
|
|
375
|
+
let eights = 0;
|
|
376
|
+
for (let i = 0; i < WORDS; i += 16) {
|
|
377
|
+
let u;
|
|
378
|
+
let twosA;
|
|
379
|
+
let twosB;
|
|
380
|
+
let foursA;
|
|
381
|
+
let foursB;
|
|
382
|
+
let x = d[i];
|
|
383
|
+
let y = d[i + 1];
|
|
384
|
+
u = ones ^ x;
|
|
385
|
+
twosA = ones & x | u & y;
|
|
386
|
+
ones = u ^ y;
|
|
387
|
+
x = d[i + 2];
|
|
388
|
+
y = d[i + 3];
|
|
389
|
+
u = ones ^ x;
|
|
390
|
+
twosB = ones & x | u & y;
|
|
391
|
+
ones = u ^ y;
|
|
392
|
+
u = twos ^ twosA;
|
|
393
|
+
foursA = twos & twosA | u & twosB;
|
|
394
|
+
twos = u ^ twosB;
|
|
395
|
+
x = d[i + 4];
|
|
396
|
+
y = d[i + 5];
|
|
397
|
+
u = ones ^ x;
|
|
398
|
+
twosA = ones & x | u & y;
|
|
399
|
+
ones = u ^ y;
|
|
400
|
+
x = d[i + 6];
|
|
401
|
+
y = d[i + 7];
|
|
402
|
+
u = ones ^ x;
|
|
403
|
+
twosB = ones & x | u & y;
|
|
404
|
+
ones = u ^ y;
|
|
405
|
+
u = twos ^ twosA;
|
|
406
|
+
foursB = twos & twosA | u & twosB;
|
|
407
|
+
twos = u ^ twosB;
|
|
408
|
+
u = fours ^ foursA;
|
|
409
|
+
const eightsA = fours & foursA | u & foursB;
|
|
410
|
+
fours = u ^ foursB;
|
|
411
|
+
x = d[i + 8];
|
|
412
|
+
y = d[i + 9];
|
|
413
|
+
u = ones ^ x;
|
|
414
|
+
twosA = ones & x | u & y;
|
|
415
|
+
ones = u ^ y;
|
|
416
|
+
x = d[i + 10];
|
|
417
|
+
y = d[i + 11];
|
|
418
|
+
u = ones ^ x;
|
|
419
|
+
twosB = ones & x | u & y;
|
|
420
|
+
ones = u ^ y;
|
|
421
|
+
u = twos ^ twosA;
|
|
422
|
+
foursA = twos & twosA | u & twosB;
|
|
423
|
+
twos = u ^ twosB;
|
|
424
|
+
x = d[i + 12];
|
|
425
|
+
y = d[i + 13];
|
|
426
|
+
u = ones ^ x;
|
|
427
|
+
twosA = ones & x | u & y;
|
|
428
|
+
ones = u ^ y;
|
|
429
|
+
x = d[i + 14];
|
|
430
|
+
y = d[i + 15];
|
|
431
|
+
u = ones ^ x;
|
|
432
|
+
twosB = ones & x | u & y;
|
|
433
|
+
ones = u ^ y;
|
|
434
|
+
u = twos ^ twosA;
|
|
435
|
+
foursB = twos & twosA | u & twosB;
|
|
436
|
+
twos = u ^ twosB;
|
|
437
|
+
u = fours ^ foursA;
|
|
438
|
+
const eightsB = fours & foursA | u & foursB;
|
|
439
|
+
fours = u ^ foursB;
|
|
440
|
+
u = eights ^ eightsA;
|
|
441
|
+
const sixteens = eights & eightsA | u & eightsB;
|
|
442
|
+
eights = u ^ eightsB;
|
|
443
|
+
total += popcount32(sixteens);
|
|
444
|
+
}
|
|
445
|
+
return 16 * total + 8 * popcount32(eights) + 4 * popcount32(fours) + 2 * popcount32(twos) + popcount32(ones);
|
|
446
|
+
}
|
|
447
|
+
function setRange(words, start, end) {
|
|
448
|
+
const first = start >>> 5;
|
|
449
|
+
const last = end >>> 5;
|
|
450
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
451
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
452
|
+
if (first === last) {
|
|
453
|
+
words[first] = words[first] | head & tail;
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
words[first] = words[first] | head;
|
|
457
|
+
for (let i = first + 1; i < last; i++) words[i] = 4294967295;
|
|
458
|
+
words[last] = words[last] | tail;
|
|
459
|
+
}
|
|
460
|
+
function clearRange(words, start, end) {
|
|
461
|
+
const first = start >>> 5;
|
|
462
|
+
const last = end >>> 5;
|
|
463
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
464
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
465
|
+
if (first === last) {
|
|
466
|
+
words[first] = words[first] & ~(head & tail);
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
469
|
+
words[first] = words[first] & ~head;
|
|
470
|
+
for (let i = first + 1; i < last; i++) words[i] = 0;
|
|
471
|
+
words[last] = words[last] & ~tail;
|
|
472
|
+
}
|
|
473
|
+
function fromSortedIds(ids, from, to) {
|
|
474
|
+
const n = to - from;
|
|
475
|
+
if (n <= ARRAY_MAX_CARDINALITY) {
|
|
476
|
+
const lows = new Uint16Array(n);
|
|
477
|
+
let count = 0;
|
|
478
|
+
let previous = -1;
|
|
479
|
+
for (let i = from; i < to; i++) {
|
|
480
|
+
const low = ids[i] & 65535;
|
|
481
|
+
if (low !== previous) lows[count++] = low;
|
|
482
|
+
previous = low;
|
|
483
|
+
}
|
|
484
|
+
return arrayContainer(count === n ? lows : lows.slice(0, count), count);
|
|
485
|
+
}
|
|
486
|
+
const words = new Uint32Array(WORDS);
|
|
487
|
+
for (let i = from; i < to; i++) {
|
|
488
|
+
const v = ids[i] & 65535;
|
|
489
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
490
|
+
}
|
|
491
|
+
return fromWordsCard(words, popcountWords(words));
|
|
492
|
+
}
|
|
493
|
+
function fromWordsCard(words, card) {
|
|
494
|
+
if (card === 0) return null;
|
|
495
|
+
if (card > ARRAY_MAX_CARDINALITY) return bitsetContainer(words, card);
|
|
496
|
+
const out = new Uint16Array(card);
|
|
497
|
+
wordsToLows(words, out);
|
|
498
|
+
return arrayContainer(out, card);
|
|
499
|
+
}
|
|
500
|
+
function wordsToLows(words, out) {
|
|
501
|
+
let k = 0;
|
|
502
|
+
for (let i = 0; i < WORDS; i++) {
|
|
503
|
+
let w = words[i];
|
|
504
|
+
const base = i * 32 + 31;
|
|
505
|
+
while (w !== 0) {
|
|
506
|
+
const lowest = w & -w;
|
|
507
|
+
out[k++] = base - Math.clz32(lowest);
|
|
508
|
+
w ^= lowest;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
return k;
|
|
512
|
+
}
|
|
513
|
+
function writeLows(c, out) {
|
|
514
|
+
if (c.kind === ARRAY2) {
|
|
515
|
+
out.set(lows16(c).subarray(c.off, c.off + c.len));
|
|
516
|
+
} else if (c.kind === BITSET) {
|
|
517
|
+
wordsToLows(words32(c), out);
|
|
518
|
+
} else {
|
|
519
|
+
const u16 = lows16(c);
|
|
520
|
+
let k = 0;
|
|
521
|
+
for (let r = 0; r < c.len; r++) {
|
|
522
|
+
const start = u16[c.off + 2 * r];
|
|
523
|
+
const end = start + u16[c.off + 2 * r + 1];
|
|
524
|
+
for (let v = start; v <= end; v++) out[k++] = v;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
function appendIds(c, out, at, base) {
|
|
529
|
+
if (c.kind === ARRAY2) {
|
|
530
|
+
const u162 = lows16(c);
|
|
531
|
+
const end = c.off + c.len;
|
|
532
|
+
if (base === 0) {
|
|
533
|
+
out.set(u162.subarray(c.off, end), at);
|
|
534
|
+
return at + c.len;
|
|
535
|
+
}
|
|
536
|
+
for (let i = c.off; i < end; i++) out[at++] = base + u162[i];
|
|
537
|
+
return at;
|
|
538
|
+
}
|
|
539
|
+
if (c.kind === BITSET) {
|
|
540
|
+
const words = words32(c);
|
|
541
|
+
for (let i = 0; i < WORDS; i++) {
|
|
542
|
+
let w = words[i];
|
|
543
|
+
const top = base + i * 32 + 31;
|
|
544
|
+
while (w !== 0) {
|
|
545
|
+
const lowest = w & -w;
|
|
546
|
+
out[at++] = top - Math.clz32(lowest);
|
|
547
|
+
w ^= lowest;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return at;
|
|
551
|
+
}
|
|
552
|
+
const u16 = lows16(c);
|
|
553
|
+
for (let r = 0; r < c.len; r++) {
|
|
554
|
+
const start = u16[c.off + 2 * r];
|
|
555
|
+
const last = start + u16[c.off + 2 * r + 1];
|
|
556
|
+
if (last - start < LONG_RUN) {
|
|
557
|
+
for (let v = base + start; v <= base + last; v++) out[at++] = v;
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
const from = at;
|
|
561
|
+
out.set(iota().subarray(start, last + 1), at);
|
|
562
|
+
at += last - start + 1;
|
|
563
|
+
if (base !== 0) for (let i = from; i < at; i++) out[i] = out[i] + base;
|
|
564
|
+
}
|
|
565
|
+
return at;
|
|
566
|
+
}
|
|
567
|
+
var LONG_RUN = 48;
|
|
568
|
+
var iotaTable;
|
|
569
|
+
function iota() {
|
|
570
|
+
if (iotaTable === void 0) {
|
|
571
|
+
iotaTable = new Uint16Array(65536);
|
|
572
|
+
for (let i = 0; i < 65536; i++) iotaTable[i] = i;
|
|
573
|
+
}
|
|
574
|
+
return iotaTable;
|
|
575
|
+
}
|
|
576
|
+
function toWords(c) {
|
|
577
|
+
if (c.kind === BITSET) return words32(c);
|
|
578
|
+
const words = new Uint32Array(WORDS);
|
|
579
|
+
const u16 = lows16(c);
|
|
580
|
+
if (c.kind === ARRAY2) {
|
|
581
|
+
for (let i = c.off; i < c.off + c.len; i++) {
|
|
582
|
+
const v = u16[i];
|
|
583
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
584
|
+
}
|
|
585
|
+
} else {
|
|
586
|
+
for (let r = 0; r < c.len; r++) {
|
|
587
|
+
const start = u16[c.off + 2 * r];
|
|
588
|
+
setRange(words, start, start + u16[c.off + 2 * r + 1]);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
return words;
|
|
592
|
+
}
|
|
593
|
+
function demote(c) {
|
|
594
|
+
if (c.kind !== RUN2) return c;
|
|
595
|
+
if (c.card > ARRAY_MAX_CARDINALITY) return bitsetContainer(toWords(c), c.card);
|
|
596
|
+
const out = new Uint16Array(c.card);
|
|
597
|
+
writeLows(c, out);
|
|
598
|
+
return arrayContainer(out, c.card);
|
|
599
|
+
}
|
|
600
|
+
function cloneContainer(c) {
|
|
601
|
+
if (c.kind === BITSET) return bitsetContainer(words32(c).slice(), c.card);
|
|
602
|
+
const copy = lows16(c).slice(c.off, c.off + (c.kind === ARRAY2 ? c.len : 2 * c.len));
|
|
603
|
+
return new Container(c.kind, copy, 0, c.len, c.card, false);
|
|
604
|
+
}
|
|
605
|
+
function writableWords(c) {
|
|
606
|
+
return c.shared ? words32(c).slice() : words32(c);
|
|
607
|
+
}
|
|
608
|
+
function lowerBound(data, off, len, low, stride) {
|
|
609
|
+
let lo = 0;
|
|
610
|
+
let hi = len;
|
|
611
|
+
while (lo < hi) {
|
|
612
|
+
const mid = lo + hi >>> 1;
|
|
613
|
+
if (data[off + mid * stride] < low) lo = mid + 1;
|
|
614
|
+
else hi = mid;
|
|
615
|
+
}
|
|
616
|
+
return lo;
|
|
617
|
+
}
|
|
618
|
+
function gallop(data, off, from, end, target) {
|
|
619
|
+
let step = 1;
|
|
620
|
+
let lo = from;
|
|
621
|
+
let hi = from;
|
|
622
|
+
while (hi < end && data[off + hi] < target) {
|
|
623
|
+
lo = hi + 1;
|
|
624
|
+
hi += step;
|
|
625
|
+
step *= 2;
|
|
626
|
+
}
|
|
627
|
+
if (hi > end) hi = end;
|
|
628
|
+
while (lo < hi) {
|
|
629
|
+
const mid = lo + hi >>> 1;
|
|
630
|
+
if (data[off + mid] < target) lo = mid + 1;
|
|
631
|
+
else hi = mid;
|
|
632
|
+
}
|
|
633
|
+
return lo;
|
|
634
|
+
}
|
|
635
|
+
function containerHas(c, low) {
|
|
636
|
+
if (c.kind === BITSET)
|
|
637
|
+
return (c.data[low >>> 5] >>> (low & 31) & 1) === 1;
|
|
638
|
+
const u16 = c.data;
|
|
639
|
+
if (c.kind === ARRAY2) {
|
|
640
|
+
const at = lowerBound(u16, c.off, c.len, low, 1);
|
|
641
|
+
return at < c.len && u16[c.off + at] === low;
|
|
642
|
+
}
|
|
643
|
+
const after = lowerBound(u16, c.off, c.len, low + 1, 2);
|
|
644
|
+
if (after === 0) return false;
|
|
645
|
+
const r = c.off + 2 * (after - 1);
|
|
646
|
+
return low <= u16[r] + u16[r + 1];
|
|
647
|
+
}
|
|
648
|
+
function containerAdd(c, low) {
|
|
649
|
+
if (containerHas(c, low)) return c;
|
|
650
|
+
const plain = demote(c);
|
|
651
|
+
if (plain.kind === BITSET) {
|
|
652
|
+
const words = writableWords(plain);
|
|
653
|
+
words[low >>> 5] = words[low >>> 5] | 1 << (low & 31);
|
|
654
|
+
return bitsetContainer(words, plain.card + 1);
|
|
655
|
+
}
|
|
656
|
+
if (plain.card === ARRAY_MAX_CARDINALITY) {
|
|
657
|
+
const words = toWords(plain);
|
|
658
|
+
words[low >>> 5] = words[low >>> 5] | 1 << (low & 31);
|
|
659
|
+
return bitsetContainer(words, plain.card + 1);
|
|
660
|
+
}
|
|
661
|
+
const old = lows16(plain);
|
|
662
|
+
const at = lowerBound(old, plain.off, plain.len, low, 1);
|
|
663
|
+
let u16 = old;
|
|
664
|
+
if (!plain.shared && plain.off === 0 && old.length > plain.len) {
|
|
665
|
+
u16.copyWithin(at + 1, at, plain.len);
|
|
666
|
+
} else {
|
|
667
|
+
u16 = new Uint16Array(Math.min(ARRAY_MAX_CARDINALITY, Math.max(8, plain.len * 2)));
|
|
668
|
+
u16.set(old.subarray(plain.off, plain.off + at));
|
|
669
|
+
u16.set(old.subarray(plain.off + at, plain.off + plain.len), at + 1);
|
|
670
|
+
}
|
|
671
|
+
u16[at] = low;
|
|
672
|
+
return arrayContainer(u16, plain.card + 1);
|
|
673
|
+
}
|
|
674
|
+
function containerRemove(c, low) {
|
|
675
|
+
if (!containerHas(c, low)) return c;
|
|
676
|
+
if (c.card === 1) return null;
|
|
677
|
+
const plain = demote(c);
|
|
678
|
+
if (plain.kind === BITSET) {
|
|
679
|
+
const words = writableWords(plain);
|
|
680
|
+
words[low >>> 5] = words[low >>> 5] & ~(1 << (low & 31));
|
|
681
|
+
return fromWordsCard(words, plain.card - 1);
|
|
682
|
+
}
|
|
683
|
+
const old = lows16(plain);
|
|
684
|
+
const at = lowerBound(old, plain.off, plain.len, low, 1);
|
|
685
|
+
if (!plain.shared && plain.off === 0) {
|
|
686
|
+
old.copyWithin(at, at + 1, plain.len);
|
|
687
|
+
return arrayContainer(old, plain.card - 1);
|
|
688
|
+
}
|
|
689
|
+
const u16 = new Uint16Array(plain.len - 1);
|
|
690
|
+
u16.set(old.subarray(plain.off, plain.off + at));
|
|
691
|
+
u16.set(old.subarray(plain.off + at + 1, plain.off + plain.len), at);
|
|
692
|
+
return arrayContainer(u16, plain.card - 1);
|
|
693
|
+
}
|
|
694
|
+
function containerMax(c) {
|
|
695
|
+
if (c.kind === ARRAY2) return c.data[c.off + c.len - 1];
|
|
696
|
+
if (c.kind === RUN2) {
|
|
697
|
+
const u16 = c.data;
|
|
698
|
+
return u16[c.off + 2 * c.len - 2] + u16[c.off + 2 * c.len - 1];
|
|
699
|
+
}
|
|
700
|
+
const words = words32(c);
|
|
701
|
+
for (let i = WORDS - 1; i >= 0; i--) {
|
|
702
|
+
const w = words[i];
|
|
703
|
+
if (w !== 0) return i * 32 + 31 - Math.clz32(w);
|
|
704
|
+
}
|
|
705
|
+
throw new RangeError("a bitset container holds no bits");
|
|
706
|
+
}
|
|
707
|
+
function runCount(c, stop) {
|
|
708
|
+
if (c.kind === RUN2) return c.len;
|
|
709
|
+
let runs = 0;
|
|
710
|
+
if (c.kind === ARRAY2) {
|
|
711
|
+
const u16 = lows16(c);
|
|
712
|
+
let previous = -2;
|
|
713
|
+
for (let i = c.off; i < c.off + c.len; i++) {
|
|
714
|
+
const v = u16[i];
|
|
715
|
+
if (v !== previous + 1 && ++runs >= stop) return runs;
|
|
716
|
+
previous = v;
|
|
717
|
+
}
|
|
718
|
+
return runs;
|
|
719
|
+
}
|
|
720
|
+
const words = words32(c);
|
|
721
|
+
let carry = 0;
|
|
722
|
+
for (let i = 0; i < WORDS; i++) {
|
|
723
|
+
const w = words[i];
|
|
724
|
+
const starts = w & ~(w << 1 | carry);
|
|
725
|
+
if (starts !== 0) {
|
|
726
|
+
runs += popcount32(starts >>> 0);
|
|
727
|
+
if (runs >= stop) return runs;
|
|
728
|
+
}
|
|
729
|
+
carry = w >>> 31;
|
|
730
|
+
}
|
|
731
|
+
return runs;
|
|
732
|
+
}
|
|
733
|
+
function toRun(c, runs) {
|
|
734
|
+
const pairs = new Uint16Array(2 * runs);
|
|
735
|
+
let r = 0;
|
|
736
|
+
if (c.kind === ARRAY2) {
|
|
737
|
+
const u16 = lows16(c);
|
|
738
|
+
const end = c.off + c.len;
|
|
739
|
+
let i = c.off;
|
|
740
|
+
while (i < end) {
|
|
741
|
+
let j = i;
|
|
742
|
+
while (j + 1 < end && u16[j + 1] === u16[j] + 1) j++;
|
|
743
|
+
pairs[2 * r] = u16[i];
|
|
744
|
+
pairs[2 * r + 1] = j - i;
|
|
745
|
+
r++;
|
|
746
|
+
i = j + 1;
|
|
747
|
+
}
|
|
748
|
+
return runContainer(pairs, runs, c.card);
|
|
749
|
+
}
|
|
750
|
+
const words = words32(c);
|
|
751
|
+
let start = 0;
|
|
752
|
+
let carry = 0;
|
|
753
|
+
for (let i = 0; i < WORDS; i++) {
|
|
754
|
+
const w = words[i];
|
|
755
|
+
const next = i + 1 < WORDS ? words[i + 1] : 0;
|
|
756
|
+
let starts = (w & ~(w << 1 | carry)) >>> 0;
|
|
757
|
+
let ends = (w & ~(w >>> 1 | next << 31)) >>> 0;
|
|
758
|
+
carry = w >>> 31;
|
|
759
|
+
let both = starts | ends;
|
|
760
|
+
while (both !== 0) {
|
|
761
|
+
const bit = both & -both;
|
|
762
|
+
both ^= bit;
|
|
763
|
+
const at = i * 32 + 31 - Math.clz32(bit);
|
|
764
|
+
if ((starts & bit) !== 0) {
|
|
765
|
+
starts ^= bit;
|
|
766
|
+
start = at;
|
|
767
|
+
}
|
|
768
|
+
if ((ends & bit) !== 0) {
|
|
769
|
+
ends ^= bit;
|
|
770
|
+
pairs[2 * r] = start;
|
|
771
|
+
pairs[2 * r + 1] = at - start;
|
|
772
|
+
r++;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
return runContainer(pairs, runs, c.card);
|
|
777
|
+
}
|
|
778
|
+
function optimize(c) {
|
|
779
|
+
const plain = c.card <= ARRAY_MAX_CARDINALITY ? 2 * c.card : BITSET_BYTES;
|
|
780
|
+
const runs = runCount(c, (plain - 2) / 4);
|
|
781
|
+
if (2 + 4 * runs < plain) return c.kind === RUN2 ? c : toRun(c, runs);
|
|
782
|
+
return demote(c);
|
|
783
|
+
}
|
|
784
|
+
function skewed(shorter, longer) {
|
|
785
|
+
return longer >= GALLOP_SKEW * shorter;
|
|
786
|
+
}
|
|
787
|
+
var PROBE_MIN = 32;
|
|
788
|
+
var probeWords = new Uint32Array(WORDS);
|
|
789
|
+
function andArrays(a, b) {
|
|
790
|
+
const small = a.len <= b.len ? a : b;
|
|
791
|
+
const big = small === a ? b : a;
|
|
792
|
+
const s = lows16(small);
|
|
793
|
+
const l = lows16(big);
|
|
794
|
+
const out = values16(small.len);
|
|
795
|
+
let n = 0;
|
|
796
|
+
if (small.len >= PROBE_MIN && !skewed(small.len, big.len)) {
|
|
797
|
+
const se = small.off + small.len;
|
|
798
|
+
const first = s[small.off];
|
|
799
|
+
const last = s[se - 1];
|
|
800
|
+
const le = big.off + big.len;
|
|
801
|
+
if (last < l[big.off] || first > l[le - 1]) return null;
|
|
802
|
+
const from = gallop(l, 0, big.off, le, first);
|
|
803
|
+
const to = gallop(l, 0, from, le, last + 1);
|
|
804
|
+
const words = probeWords;
|
|
805
|
+
for (let i2 = small.off; i2 < se; i2++) {
|
|
806
|
+
const v = s[i2];
|
|
807
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
808
|
+
}
|
|
809
|
+
for (let j2 = from; j2 < to; j2++) {
|
|
810
|
+
const v = l[j2];
|
|
811
|
+
out[n] = v;
|
|
812
|
+
n += words[v >>> 5] >>> (v & 31) & 1;
|
|
813
|
+
}
|
|
814
|
+
for (let i2 = small.off; i2 < se; i2++) words[s[i2] >>> 5] = 0;
|
|
815
|
+
return arrayOfScratch(out, n);
|
|
816
|
+
}
|
|
817
|
+
if (skewed(small.len, big.len)) {
|
|
818
|
+
let at = 0;
|
|
819
|
+
const end = big.off + big.len;
|
|
820
|
+
for (let i2 = small.off; i2 < small.off + small.len; i2++) {
|
|
821
|
+
const v = s[i2];
|
|
822
|
+
at = gallop(l, 0, Math.max(at, big.off), end, v);
|
|
823
|
+
if (at === end) break;
|
|
824
|
+
if (l[at] === v) out[n++] = v;
|
|
825
|
+
}
|
|
826
|
+
return arrayOfScratch(out, n);
|
|
827
|
+
}
|
|
828
|
+
let i = small.off;
|
|
829
|
+
let j = big.off;
|
|
830
|
+
const ie = small.off + small.len;
|
|
831
|
+
const je = big.off + big.len;
|
|
832
|
+
while (i < ie && j < je) {
|
|
833
|
+
const x = s[i];
|
|
834
|
+
const y = l[j];
|
|
835
|
+
out[n] = x;
|
|
836
|
+
n += x === y ? 1 : 0;
|
|
837
|
+
i += x <= y ? 1 : 0;
|
|
838
|
+
j += x >= y ? 1 : 0;
|
|
839
|
+
}
|
|
840
|
+
return arrayOfScratch(out, n);
|
|
841
|
+
}
|
|
842
|
+
function filterByBits(a, b, keep) {
|
|
843
|
+
const u16 = lows16(a);
|
|
844
|
+
const words = words32(b);
|
|
845
|
+
const out = values16(a.len);
|
|
846
|
+
const flip = keep ? 0 : 1;
|
|
847
|
+
let n = 0;
|
|
848
|
+
for (let i = a.off; i < a.off + a.len; i++) {
|
|
849
|
+
const v = u16[i];
|
|
850
|
+
out[n] = v;
|
|
851
|
+
n += words[v >>> 5] >>> (v & 31) & 1 ^ flip;
|
|
852
|
+
}
|
|
853
|
+
return arrayOfScratch(out, n);
|
|
854
|
+
}
|
|
855
|
+
function filterByRuns(a, b, keep) {
|
|
856
|
+
const u16 = lows16(a);
|
|
857
|
+
const pairs = lows16(b);
|
|
858
|
+
const out = values16(a.len);
|
|
859
|
+
let n = 0;
|
|
860
|
+
if (keep && a.len >= b.len) {
|
|
861
|
+
const ae = a.off + a.len;
|
|
862
|
+
let i = a.off;
|
|
863
|
+
for (let r2 = 0; r2 < b.len && i < ae; r2++) {
|
|
864
|
+
const start = pairs[b.off + 2 * r2];
|
|
865
|
+
const end = start + pairs[b.off + 2 * r2 + 1];
|
|
866
|
+
while (i < ae && u16[i] < start) i++;
|
|
867
|
+
while (i < ae && u16[i] <= end) out[n++] = u16[i++];
|
|
868
|
+
}
|
|
869
|
+
return arrayOfScratch(out, n);
|
|
870
|
+
}
|
|
871
|
+
let r = 0;
|
|
872
|
+
for (let i = a.off; i < a.off + a.len; i++) {
|
|
873
|
+
const v = u16[i];
|
|
874
|
+
while (r < b.len && pairs[b.off + 2 * r] + pairs[b.off + 2 * r + 1] < v) {
|
|
875
|
+
r++;
|
|
876
|
+
}
|
|
877
|
+
const inside = r < b.len && pairs[b.off + 2 * r] <= v;
|
|
878
|
+
if (inside === keep) out[n++] = v;
|
|
879
|
+
}
|
|
880
|
+
return arrayOfScratch(out, n);
|
|
881
|
+
}
|
|
882
|
+
function andNotArrays(a, b) {
|
|
883
|
+
const x = lows16(a);
|
|
884
|
+
const y = lows16(b);
|
|
885
|
+
const out = values16(a.len);
|
|
886
|
+
let n = 0;
|
|
887
|
+
const je = b.off + b.len;
|
|
888
|
+
if (skewed(a.len, b.len)) {
|
|
889
|
+
let at = b.off;
|
|
890
|
+
for (let i2 = a.off; i2 < a.off + a.len; i2++) {
|
|
891
|
+
const v = x[i2];
|
|
892
|
+
at = gallop(y, 0, at, je, v);
|
|
893
|
+
if (at === je || y[at] !== v) out[n++] = v;
|
|
894
|
+
}
|
|
895
|
+
return arrayOfScratch(out, n);
|
|
896
|
+
}
|
|
897
|
+
if (b.len >= PROBE_MIN) {
|
|
898
|
+
const ae = a.off + a.len;
|
|
899
|
+
const first = x[a.off];
|
|
900
|
+
const last = x[ae - 1];
|
|
901
|
+
const from = gallop(y, 0, b.off, je, first);
|
|
902
|
+
const to = gallop(y, 0, from, je, last + 1);
|
|
903
|
+
const words = probeWords;
|
|
904
|
+
for (let j2 = from; j2 < to; j2++) {
|
|
905
|
+
const v = y[j2];
|
|
906
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
907
|
+
}
|
|
908
|
+
for (let i2 = a.off; i2 < ae; i2++) {
|
|
909
|
+
const v = x[i2];
|
|
910
|
+
out[n] = v;
|
|
911
|
+
n += words[v >>> 5] >>> (v & 31) & 1 ^ 1;
|
|
912
|
+
}
|
|
913
|
+
for (let j2 = from; j2 < to; j2++) words[y[j2] >>> 5] = 0;
|
|
914
|
+
return arrayOfScratch(out, n);
|
|
915
|
+
}
|
|
916
|
+
let i = a.off;
|
|
917
|
+
let j = b.off;
|
|
918
|
+
const ie = a.off + a.len;
|
|
919
|
+
while (i < ie && j < je) {
|
|
920
|
+
const p = x[i];
|
|
921
|
+
const q = y[j];
|
|
922
|
+
out[n] = p;
|
|
923
|
+
n += p < q ? 1 : 0;
|
|
924
|
+
i += p <= q ? 1 : 0;
|
|
925
|
+
j += p >= q ? 1 : 0;
|
|
926
|
+
}
|
|
927
|
+
while (i < ie) out[n++] = x[i++];
|
|
928
|
+
return arrayOfScratch(out, n);
|
|
929
|
+
}
|
|
930
|
+
function orArrays(a, b) {
|
|
931
|
+
const x = lows16(a);
|
|
932
|
+
const y = lows16(b);
|
|
933
|
+
const ie = a.off + a.len;
|
|
934
|
+
const je = b.off + b.len;
|
|
935
|
+
if (a.len + b.len > ARRAY_MAX_CARDINALITY) {
|
|
936
|
+
const words = new Uint32Array(WORDS);
|
|
937
|
+
for (let i2 = a.off; i2 < ie; i2++) {
|
|
938
|
+
const v = x[i2];
|
|
939
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
940
|
+
}
|
|
941
|
+
for (let j2 = b.off; j2 < je; j2++) {
|
|
942
|
+
const v = y[j2];
|
|
943
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
944
|
+
}
|
|
945
|
+
return fromWordsCard(words, popcountWords(words));
|
|
946
|
+
}
|
|
947
|
+
const out = values16(a.len + b.len);
|
|
948
|
+
let n = 0;
|
|
949
|
+
let i = a.off;
|
|
950
|
+
let j = b.off;
|
|
951
|
+
while (i < ie && j < je) {
|
|
952
|
+
const p = x[i];
|
|
953
|
+
const q = y[j];
|
|
954
|
+
out[n++] = p < q ? p : q;
|
|
955
|
+
i += p <= q ? 1 : 0;
|
|
956
|
+
j += p >= q ? 1 : 0;
|
|
957
|
+
}
|
|
958
|
+
while (i < ie) out[n++] = x[i++];
|
|
959
|
+
while (j < je) out[n++] = y[j++];
|
|
960
|
+
return arrayOfScratch(out, n);
|
|
961
|
+
}
|
|
962
|
+
function orIntoBitset(dest, other, inPlace) {
|
|
963
|
+
const words = inPlace ? words32(dest) : words32(dest).slice();
|
|
964
|
+
if (other.kind === ARRAY2) {
|
|
965
|
+
const u16 = lows16(other);
|
|
966
|
+
let card = dest.card;
|
|
967
|
+
for (let i = other.off; i < other.off + other.len; i++) {
|
|
968
|
+
const v = u16[i];
|
|
969
|
+
const w = words[v >>> 5];
|
|
970
|
+
const bit = 1 << (v & 31);
|
|
971
|
+
if ((w & bit) === 0) {
|
|
972
|
+
words[v >>> 5] = w | bit;
|
|
973
|
+
card++;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return bitsetContainer(words, card);
|
|
977
|
+
}
|
|
978
|
+
if (other.kind === BITSET) {
|
|
979
|
+
const o = words32(other);
|
|
980
|
+
for (let i = 0; i < WORDS; i++) words[i] = words[i] | o[i];
|
|
981
|
+
} else {
|
|
982
|
+
const pairs = lows16(other);
|
|
983
|
+
for (let r = 0; r < other.len; r++) {
|
|
984
|
+
const start = pairs[other.off + 2 * r];
|
|
985
|
+
setRange(words, start, start + pairs[other.off + 2 * r + 1]);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
return bitsetContainer(words, popcountWords(words));
|
|
989
|
+
}
|
|
990
|
+
function fromRunPairs(pairs, runs, card) {
|
|
991
|
+
if (card === 0) return null;
|
|
992
|
+
if (card <= ARRAY_MAX_CARDINALITY) {
|
|
993
|
+
const out = new Uint16Array(card);
|
|
994
|
+
let k = 0;
|
|
995
|
+
for (let r = 0; r < runs; r++) {
|
|
996
|
+
const start = pairs[2 * r];
|
|
997
|
+
const end = start + pairs[2 * r + 1];
|
|
998
|
+
for (let v = start; v <= end; v++) out[k++] = v;
|
|
999
|
+
}
|
|
1000
|
+
return arrayContainer(out, card);
|
|
1001
|
+
}
|
|
1002
|
+
const words = new Uint32Array(WORDS);
|
|
1003
|
+
for (let r = 0; r < runs; r++) {
|
|
1004
|
+
const start = pairs[2 * r];
|
|
1005
|
+
setRange(words, start, start + pairs[2 * r + 1]);
|
|
1006
|
+
}
|
|
1007
|
+
return bitsetContainer(words, card);
|
|
1008
|
+
}
|
|
1009
|
+
function andRuns(a, b) {
|
|
1010
|
+
const x = lows16(a);
|
|
1011
|
+
const y = lows16(b);
|
|
1012
|
+
const out = pairs16(2 * (a.len + b.len));
|
|
1013
|
+
let runs = 0;
|
|
1014
|
+
let card = 0;
|
|
1015
|
+
let i = 0;
|
|
1016
|
+
let j = 0;
|
|
1017
|
+
while (i < a.len && j < b.len) {
|
|
1018
|
+
const as = x[a.off + 2 * i];
|
|
1019
|
+
const ae = as + x[a.off + 2 * i + 1];
|
|
1020
|
+
const bs = y[b.off + 2 * j];
|
|
1021
|
+
const be = bs + y[b.off + 2 * j + 1];
|
|
1022
|
+
const lo = as > bs ? as : bs;
|
|
1023
|
+
const hi = ae < be ? ae : be;
|
|
1024
|
+
if (lo <= hi) {
|
|
1025
|
+
out[2 * runs] = lo;
|
|
1026
|
+
out[2 * runs + 1] = hi - lo;
|
|
1027
|
+
runs++;
|
|
1028
|
+
card += hi - lo + 1;
|
|
1029
|
+
}
|
|
1030
|
+
if (ae < be) i++;
|
|
1031
|
+
else j++;
|
|
1032
|
+
}
|
|
1033
|
+
return fromRunPairs(out, runs, card);
|
|
1034
|
+
}
|
|
1035
|
+
function orRuns(a, b) {
|
|
1036
|
+
const x = lows16(a);
|
|
1037
|
+
const y = lows16(b);
|
|
1038
|
+
const out = pairs16(2 * (a.len + b.len));
|
|
1039
|
+
let runs = 0;
|
|
1040
|
+
let card = 0;
|
|
1041
|
+
let i = 0;
|
|
1042
|
+
let j = 0;
|
|
1043
|
+
let start = -1;
|
|
1044
|
+
let end = -1;
|
|
1045
|
+
const flush = () => {
|
|
1046
|
+
if (start < 0) return;
|
|
1047
|
+
out[2 * runs] = start;
|
|
1048
|
+
out[2 * runs + 1] = end - start;
|
|
1049
|
+
runs++;
|
|
1050
|
+
card += end - start + 1;
|
|
1051
|
+
};
|
|
1052
|
+
while (i < a.len || j < b.len) {
|
|
1053
|
+
let s;
|
|
1054
|
+
let e;
|
|
1055
|
+
if (j >= b.len || i < a.len && x[a.off + 2 * i] <= y[b.off + 2 * j]) {
|
|
1056
|
+
s = x[a.off + 2 * i];
|
|
1057
|
+
e = s + x[a.off + 2 * i + 1];
|
|
1058
|
+
i++;
|
|
1059
|
+
} else {
|
|
1060
|
+
s = y[b.off + 2 * j];
|
|
1061
|
+
e = s + y[b.off + 2 * j + 1];
|
|
1062
|
+
j++;
|
|
1063
|
+
}
|
|
1064
|
+
if (start >= 0 && s <= end + 1) {
|
|
1065
|
+
if (e > end) end = e;
|
|
1066
|
+
} else {
|
|
1067
|
+
flush();
|
|
1068
|
+
start = s;
|
|
1069
|
+
end = e;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
flush();
|
|
1073
|
+
return fromRunPairs(out, runs, card);
|
|
1074
|
+
}
|
|
1075
|
+
function andNotRuns(a, b) {
|
|
1076
|
+
const x = lows16(a);
|
|
1077
|
+
const y = lows16(b);
|
|
1078
|
+
const out = pairs16(2 * (a.len + b.len + 1));
|
|
1079
|
+
let runs = 0;
|
|
1080
|
+
let card = 0;
|
|
1081
|
+
let j = 0;
|
|
1082
|
+
for (let i = 0; i < a.len; i++) {
|
|
1083
|
+
let s = x[a.off + 2 * i];
|
|
1084
|
+
const e = s + x[a.off + 2 * i + 1];
|
|
1085
|
+
while (j < b.len && y[b.off + 2 * j] + y[b.off + 2 * j + 1] < s) j++;
|
|
1086
|
+
let k = j;
|
|
1087
|
+
while (s <= e) {
|
|
1088
|
+
if (k >= b.len || y[b.off + 2 * k] > e) {
|
|
1089
|
+
out[2 * runs] = s;
|
|
1090
|
+
out[2 * runs + 1] = e - s;
|
|
1091
|
+
runs++;
|
|
1092
|
+
card += e - s + 1;
|
|
1093
|
+
break;
|
|
1094
|
+
}
|
|
1095
|
+
const bs = y[b.off + 2 * k];
|
|
1096
|
+
const be = bs + y[b.off + 2 * k + 1];
|
|
1097
|
+
if (bs > s) {
|
|
1098
|
+
out[2 * runs] = s;
|
|
1099
|
+
out[2 * runs + 1] = bs - 1 - s;
|
|
1100
|
+
runs++;
|
|
1101
|
+
card += bs - s;
|
|
1102
|
+
}
|
|
1103
|
+
s = be + 1;
|
|
1104
|
+
k++;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
return fromRunPairs(out, runs, card);
|
|
1108
|
+
}
|
|
1109
|
+
function bitsetInRuns(a, b) {
|
|
1110
|
+
const words = words32(a);
|
|
1111
|
+
const pairs = lows16(b);
|
|
1112
|
+
if (b.card <= ARRAY_MAX_CARDINALITY) {
|
|
1113
|
+
const out2 = values16(b.card);
|
|
1114
|
+
let n = 0;
|
|
1115
|
+
for (let r = 0; r < b.len; r++) {
|
|
1116
|
+
const start = pairs[b.off + 2 * r];
|
|
1117
|
+
const end = start + pairs[b.off + 2 * r + 1];
|
|
1118
|
+
const first = start >>> 5;
|
|
1119
|
+
const last = end >>> 5;
|
|
1120
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
1121
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
1122
|
+
for (let i = first; i <= last; i++) {
|
|
1123
|
+
let w = words[i];
|
|
1124
|
+
if (i === first) w &= head;
|
|
1125
|
+
if (i === last) w &= tail;
|
|
1126
|
+
const top = i * 32 + 31;
|
|
1127
|
+
while (w !== 0) {
|
|
1128
|
+
const lowest = w & -w;
|
|
1129
|
+
out2[n++] = top - Math.clz32(lowest);
|
|
1130
|
+
w ^= lowest;
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
return arrayOfScratch(out2, n);
|
|
1135
|
+
}
|
|
1136
|
+
const out = new Uint32Array(WORDS);
|
|
1137
|
+
for (let r = 0; r < b.len; r++) {
|
|
1138
|
+
const start = pairs[b.off + 2 * r];
|
|
1139
|
+
const end = start + pairs[b.off + 2 * r + 1];
|
|
1140
|
+
const first = start >>> 5;
|
|
1141
|
+
const last = end >>> 5;
|
|
1142
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
1143
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
1144
|
+
if (first === last) {
|
|
1145
|
+
out[first] = out[first] | words[first] & head & tail;
|
|
1146
|
+
} else {
|
|
1147
|
+
out[first] = out[first] | words[first] & head;
|
|
1148
|
+
for (let i = first + 1; i < last; i++) out[i] = words[i];
|
|
1149
|
+
out[last] = out[last] | words[last] & tail;
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
return fromWordsCard(out, popcountWords(out));
|
|
1153
|
+
}
|
|
1154
|
+
function containerAnd(a, b, inPlace = false) {
|
|
1155
|
+
if (a.kind === BITSET && b.kind === BITSET) {
|
|
1156
|
+
const x = words32(a);
|
|
1157
|
+
const y = words32(b);
|
|
1158
|
+
const out = inPlace ? x : new Uint32Array(WORDS);
|
|
1159
|
+
for (let i = 0; i < WORDS; i++) out[i] = x[i] & y[i];
|
|
1160
|
+
return fromWordsCard(out, popcountWords(out));
|
|
1161
|
+
}
|
|
1162
|
+
if (a.kind > b.kind) return containerAnd(b, a);
|
|
1163
|
+
if (a.kind === ARRAY2) {
|
|
1164
|
+
if (b.kind === ARRAY2) return andArrays(a, b);
|
|
1165
|
+
return b.kind === BITSET ? filterByBits(a, b, true) : filterByRuns(a, b, true);
|
|
1166
|
+
}
|
|
1167
|
+
return b.kind === RUN2 && a.kind === RUN2 ? andRuns(a, b) : bitsetInRuns(a, b);
|
|
1168
|
+
}
|
|
1169
|
+
function containerOr(a, b, inPlace = false) {
|
|
1170
|
+
if (a.kind === BITSET) return orIntoBitset(a, b, inPlace);
|
|
1171
|
+
if (b.kind === BITSET) return orIntoBitset(b, a, false);
|
|
1172
|
+
if (a.kind === ARRAY2 && b.kind === ARRAY2) return orArrays(a, b);
|
|
1173
|
+
if (a.kind === RUN2 && b.kind === RUN2) return orRuns(a, b);
|
|
1174
|
+
const words = toWords(a.kind === RUN2 ? a : b);
|
|
1175
|
+
return orIntoBitsetWords(words, a.kind === RUN2 ? b : a);
|
|
1176
|
+
}
|
|
1177
|
+
function orIntoBitsetWords(words, array) {
|
|
1178
|
+
const u16 = lows16(array);
|
|
1179
|
+
for (let i = array.off; i < array.off + array.len; i++) {
|
|
1180
|
+
const v = u16[i];
|
|
1181
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
1182
|
+
}
|
|
1183
|
+
return fromWordsCard(words, popcountWords(words));
|
|
1184
|
+
}
|
|
1185
|
+
function containerAndNot(a, b, inPlace = false) {
|
|
1186
|
+
if (a.kind === ARRAY2) {
|
|
1187
|
+
if (b.kind === ARRAY2) return andNotArrays(a, b);
|
|
1188
|
+
return b.kind === BITSET ? filterByBits(a, b, false) : filterByRuns(a, b, false);
|
|
1189
|
+
}
|
|
1190
|
+
if (a.kind === RUN2 && b.kind === RUN2) return andNotRuns(a, b);
|
|
1191
|
+
const words = a.kind === BITSET ? inPlace ? words32(a) : words32(a).slice() : toWords(a);
|
|
1192
|
+
if (b.kind === ARRAY2) {
|
|
1193
|
+
const u16 = lows16(b);
|
|
1194
|
+
for (let i = b.off; i < b.off + b.len; i++) {
|
|
1195
|
+
const v = u16[i];
|
|
1196
|
+
words[v >>> 5] = words[v >>> 5] & ~(1 << (v & 31));
|
|
1197
|
+
}
|
|
1198
|
+
} else if (b.kind === BITSET) {
|
|
1199
|
+
const y = words32(b);
|
|
1200
|
+
for (let i = 0; i < WORDS; i++) words[i] = words[i] & ~y[i];
|
|
1201
|
+
} else {
|
|
1202
|
+
const pairs = lows16(b);
|
|
1203
|
+
for (let r = 0; r < b.len; r++) {
|
|
1204
|
+
const start = pairs[b.off + 2 * r];
|
|
1205
|
+
clearRange(words, start, start + pairs[b.off + 2 * r + 1]);
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
return fromWordsCard(words, popcountWords(words));
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
// src/ts-roaring/iterator.ts
|
|
1212
|
+
var DONE = { done: true, value: void 0 };
|
|
1213
|
+
var BitmapIterator = class {
|
|
1214
|
+
#keys;
|
|
1215
|
+
#conts;
|
|
1216
|
+
/** The container being walked, as an index into `#conts`, and its kind, storage and offset. */
|
|
1217
|
+
#at = -1;
|
|
1218
|
+
#kind = ARRAY2;
|
|
1219
|
+
#u16 = new Uint16Array(0);
|
|
1220
|
+
#words = new Uint32Array(0);
|
|
1221
|
+
#off = 0;
|
|
1222
|
+
/** Added to every remainder: the container's key times 65,536. */
|
|
1223
|
+
#base = 0;
|
|
1224
|
+
/** An array: the next index and one past the last. A run container: the next run's index and the run count. A bitset: the word index. */
|
|
1225
|
+
#pos = 0;
|
|
1226
|
+
#end = 0;
|
|
1227
|
+
/** A run container: the next value and the last of the current run. A bitset: what is left of the current word. */
|
|
1228
|
+
#next = 0;
|
|
1229
|
+
#last = -1;
|
|
1230
|
+
constructor(keys, conts) {
|
|
1231
|
+
this.#keys = keys;
|
|
1232
|
+
this.#conts = conts;
|
|
1233
|
+
}
|
|
1234
|
+
[Symbol.iterator]() {
|
|
1235
|
+
return this;
|
|
1236
|
+
}
|
|
1237
|
+
next() {
|
|
1238
|
+
for (; ; ) {
|
|
1239
|
+
if (this.#kind === ARRAY2) {
|
|
1240
|
+
if (this.#pos < this.#end) {
|
|
1241
|
+
return { done: false, value: this.#base + this.#u16[this.#pos++] };
|
|
1242
|
+
}
|
|
1243
|
+
} else if (this.#kind === BITSET) {
|
|
1244
|
+
let w = this.#next;
|
|
1245
|
+
while (w === 0 && ++this.#pos < WORDS) w = this.#words[this.#pos];
|
|
1246
|
+
if (w !== 0) {
|
|
1247
|
+
const lowest = w & -w;
|
|
1248
|
+
this.#next = w ^ lowest;
|
|
1249
|
+
return {
|
|
1250
|
+
done: false,
|
|
1251
|
+
value: this.#base + this.#pos * 32 + 31 - Math.clz32(lowest)
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
} else {
|
|
1255
|
+
if (this.#next <= this.#last) return { done: false, value: this.#base + this.#next++ };
|
|
1256
|
+
if (this.#pos < this.#end) {
|
|
1257
|
+
const r = this.#off + 2 * this.#pos++;
|
|
1258
|
+
this.#next = this.#u16[r];
|
|
1259
|
+
this.#last = this.#next + this.#u16[r + 1];
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
if (!this.#advance()) return DONE;
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
/** Move to the next container; `false` when there is none. */
|
|
1267
|
+
#advance() {
|
|
1268
|
+
if (++this.#at >= this.#conts.length) {
|
|
1269
|
+
this.#at = this.#conts.length;
|
|
1270
|
+
this.#kind = ARRAY2;
|
|
1271
|
+
this.#pos = 0;
|
|
1272
|
+
this.#end = 0;
|
|
1273
|
+
return false;
|
|
1274
|
+
}
|
|
1275
|
+
const c = this.#conts[this.#at];
|
|
1276
|
+
this.#kind = c.kind;
|
|
1277
|
+
this.#base = this.#keys[this.#at] * 65536;
|
|
1278
|
+
this.#off = c.off;
|
|
1279
|
+
if (c.kind === ARRAY2) {
|
|
1280
|
+
this.#u16 = c.data;
|
|
1281
|
+
this.#pos = c.off;
|
|
1282
|
+
this.#end = c.off + c.len;
|
|
1283
|
+
} else if (c.kind === BITSET) {
|
|
1284
|
+
this.#words = c.data;
|
|
1285
|
+
this.#pos = 0;
|
|
1286
|
+
this.#next = this.#words[0];
|
|
1287
|
+
} else {
|
|
1288
|
+
this.#u16 = c.data;
|
|
1289
|
+
this.#pos = 0;
|
|
1290
|
+
this.#end = c.len;
|
|
1291
|
+
this.#next = 0;
|
|
1292
|
+
this.#last = -1;
|
|
1293
|
+
}
|
|
1294
|
+
return true;
|
|
1295
|
+
}
|
|
1296
|
+
};
|
|
1297
|
+
|
|
1298
|
+
// src/ts-roaring/wire.ts
|
|
1299
|
+
import { IntegrityError as IntegrityError2 } from "@cloudbitmaps/core";
|
|
1300
|
+
function bodySize(c) {
|
|
1301
|
+
if (c.kind === ARRAY2) return 2 * c.len;
|
|
1302
|
+
if (c.kind === BITSET) return BITSET_BYTES;
|
|
1303
|
+
return 2 + 4 * c.len;
|
|
1304
|
+
}
|
|
1305
|
+
function putU16s(out, at, src, from, count) {
|
|
1306
|
+
if (LITTLE_ENDIAN) {
|
|
1307
|
+
out.set(new Uint8Array(src.buffer, src.byteOffset + 2 * from, 2 * count), at);
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1310
|
+
const view = new DataView(out.buffer, out.byteOffset, out.byteLength);
|
|
1311
|
+
for (let i = 0; i < count; i++) view.setUint16(at + 2 * i, src[from + i], true);
|
|
1312
|
+
}
|
|
1313
|
+
function writeBody(c, out, view, at) {
|
|
1314
|
+
if (c.kind === ARRAY2) {
|
|
1315
|
+
putU16s(out, at, c.data, c.off, c.len);
|
|
1316
|
+
} else if (c.kind === BITSET) {
|
|
1317
|
+
const words = c.data;
|
|
1318
|
+
if (LITTLE_ENDIAN) out.set(new Uint8Array(words.buffer, words.byteOffset, BITSET_BYTES), at);
|
|
1319
|
+
else for (let i = 0; i < WORDS; i++) view.setUint32(at + 4 * i, words[i], true);
|
|
1320
|
+
} else {
|
|
1321
|
+
view.setUint16(at, c.len, true);
|
|
1322
|
+
putU16s(out, at + 2, c.data, c.off, 2 * c.len);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
function serializeContainers(keys, conts) {
|
|
1326
|
+
const n = conts.length;
|
|
1327
|
+
let hasRun = false;
|
|
1328
|
+
for (let i = 0; i < n; i++) if (conts[i].kind === RUN2) hasRun = true;
|
|
1329
|
+
const offsets = !hasRun || n >= NO_OFFSET_THRESHOLD;
|
|
1330
|
+
let at = hasRun ? 4 + (n + 7 >>> 3) : 8;
|
|
1331
|
+
const describeAt = at;
|
|
1332
|
+
at += 4 * n;
|
|
1333
|
+
const offsetsAt = at;
|
|
1334
|
+
if (offsets) at += 4 * n;
|
|
1335
|
+
let total = at;
|
|
1336
|
+
for (let i = 0; i < n; i++) total += bodySize(conts[i]);
|
|
1337
|
+
const out = new Uint8Array(total);
|
|
1338
|
+
const view = new DataView(out.buffer);
|
|
1339
|
+
if (hasRun) {
|
|
1340
|
+
view.setUint32(0, SERIAL_COOKIE | n - 1 << 16, true);
|
|
1341
|
+
for (let i = 0; i < n; i++) {
|
|
1342
|
+
if (conts[i].kind === RUN2) {
|
|
1343
|
+
out[4 + (i >>> 3)] = out[4 + (i >>> 3)] | 1 << (i & 7);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
} else {
|
|
1347
|
+
view.setUint32(0, SERIAL_COOKIE_NO_RUNCONTAINER, true);
|
|
1348
|
+
view.setUint32(4, n, true);
|
|
1349
|
+
}
|
|
1350
|
+
for (let i = 0; i < n; i++) {
|
|
1351
|
+
const c = conts[i];
|
|
1352
|
+
view.setUint16(describeAt + 4 * i, keys[i], true);
|
|
1353
|
+
view.setUint16(describeAt + 4 * i + 2, c.card - 1, true);
|
|
1354
|
+
if (offsets) view.setUint32(offsetsAt + 4 * i, at, true);
|
|
1355
|
+
writeBody(c, out, view, at);
|
|
1356
|
+
at += bodySize(c);
|
|
1357
|
+
}
|
|
1358
|
+
return out;
|
|
1359
|
+
}
|
|
1360
|
+
function decodeContainers(input) {
|
|
1361
|
+
const bytes = Uint8Array.prototype.slice.call(input);
|
|
1362
|
+
const layout = parsePortableLayout(bytes);
|
|
1363
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1364
|
+
const keys = new Array(layout.length);
|
|
1365
|
+
const conts = new Array(layout.length);
|
|
1366
|
+
let n = 0;
|
|
1367
|
+
let end = bytes.byteLength === 0 ? 0 : 8;
|
|
1368
|
+
let slabLength = 0;
|
|
1369
|
+
let small = 0;
|
|
1370
|
+
for (const { cardinality, kind, offset } of layout) {
|
|
1371
|
+
if (kind === ARRAY) {
|
|
1372
|
+
slabLength += cardinality;
|
|
1373
|
+
small++;
|
|
1374
|
+
} else if (kind !== BITMAP) {
|
|
1375
|
+
slabLength += 2 * view.getUint16(offset, true);
|
|
1376
|
+
small++;
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
const own = LITTLE_ENDIAN && bytes.byteOffset === 0 && bytes.buffer instanceof ArrayBuffer && bytes.buffer.byteLength === bytes.byteLength;
|
|
1380
|
+
const alias = own && small === 1;
|
|
1381
|
+
const slab = alias ? NO_U16 : new Uint16Array(slabLength);
|
|
1382
|
+
let used = 0;
|
|
1383
|
+
for (const { key, cardinality, kind, offset } of layout) {
|
|
1384
|
+
keys[n] = key;
|
|
1385
|
+
if (kind === ARRAY) {
|
|
1386
|
+
if (alias && offset % 2 === 0) {
|
|
1387
|
+
const values = new Uint16Array(bytes.buffer, offset, cardinality);
|
|
1388
|
+
conts[n++] = new Container(ARRAY2, values, 0, cardinality, cardinality, false);
|
|
1389
|
+
} else {
|
|
1390
|
+
const into = alias ? new Uint16Array(cardinality) : slab;
|
|
1391
|
+
getU16s(bytes, view, offset, into, alias ? 0 : used, cardinality);
|
|
1392
|
+
conts[n++] = new Container(
|
|
1393
|
+
ARRAY2,
|
|
1394
|
+
into,
|
|
1395
|
+
alias ? 0 : used,
|
|
1396
|
+
cardinality,
|
|
1397
|
+
cardinality,
|
|
1398
|
+
small > 1
|
|
1399
|
+
);
|
|
1400
|
+
used += cardinality;
|
|
1401
|
+
}
|
|
1402
|
+
end = offset + 2 * cardinality;
|
|
1403
|
+
} else if (kind === BITMAP) {
|
|
1404
|
+
let words;
|
|
1405
|
+
if (own && offset % 4 === 0) {
|
|
1406
|
+
words = new Uint32Array(bytes.buffer, offset, WORDS);
|
|
1407
|
+
} else {
|
|
1408
|
+
words = new Uint32Array(WORDS);
|
|
1409
|
+
if (LITTLE_ENDIAN) {
|
|
1410
|
+
new Uint8Array(words.buffer).set(bytes.subarray(offset, offset + BITSET_BYTES));
|
|
1411
|
+
} else {
|
|
1412
|
+
for (let i = 0; i < WORDS; i++) words[i] = view.getUint32(offset + 4 * i, true);
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
conts[n++] = bitsetContainer(words, cardinality);
|
|
1416
|
+
end = offset + BITSET_BYTES;
|
|
1417
|
+
} else {
|
|
1418
|
+
const runs = view.getUint16(offset, true);
|
|
1419
|
+
if (alias && (offset + 2) % 2 === 0) {
|
|
1420
|
+
const pairs = new Uint16Array(bytes.buffer, offset + 2, 2 * runs);
|
|
1421
|
+
conts[n++] = new Container(RUN2, pairs, 0, runs, cardinality, false);
|
|
1422
|
+
} else {
|
|
1423
|
+
const into = alias ? new Uint16Array(2 * runs) : slab;
|
|
1424
|
+
getU16s(bytes, view, offset + 2, into, alias ? 0 : used, 2 * runs);
|
|
1425
|
+
conts[n++] = new Container(RUN2, into, alias ? 0 : used, runs, cardinality, small > 1);
|
|
1426
|
+
used += 2 * runs;
|
|
1427
|
+
}
|
|
1428
|
+
end = offset + 2 + 4 * runs;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
return { keys, conts, end };
|
|
1432
|
+
}
|
|
1433
|
+
var NO_U16 = new Uint16Array(0);
|
|
1434
|
+
function getU16s(bytes, view, at, slab, to, count) {
|
|
1435
|
+
if (LITTLE_ENDIAN) {
|
|
1436
|
+
new Uint8Array(slab.buffer, 2 * to, 2 * count).set(bytes.subarray(at, at + 2 * count));
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
for (let i = 0; i < count; i++) slab[to + i] = view.getUint16(at + 2 * i, true);
|
|
1440
|
+
}
|
|
1441
|
+
function trailingBytes(length, end) {
|
|
1442
|
+
return new IntegrityError2(
|
|
1443
|
+
`portable roaring: ${length - end} bytes follow the bitmap's last container, which ends at byte ${end}. One buffer holds one bitmap: to load several as one segment, combine them first and serialize the result.`
|
|
1444
|
+
);
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
// src/ts-roaring/bitmap.ts
|
|
1448
|
+
var MAX_ID = 4294967295;
|
|
1449
|
+
function isId(value) {
|
|
1450
|
+
return Number.isInteger(value) && value >= 0 && value <= MAX_ID;
|
|
1451
|
+
}
|
|
1452
|
+
function badId(value) {
|
|
1453
|
+
return new ValidationError2(`${String(value)} is not an integer from 0 to ${MAX_ID}`);
|
|
1454
|
+
}
|
|
1455
|
+
function checkedId(value) {
|
|
1456
|
+
if (!isId(value)) throw badId(value);
|
|
1457
|
+
return value;
|
|
1458
|
+
}
|
|
1459
|
+
function collect(values) {
|
|
1460
|
+
if (values instanceof Uint32Array) return values;
|
|
1461
|
+
if (Array.isArray(values)) {
|
|
1462
|
+
const list = values;
|
|
1463
|
+
const out2 = new Uint32Array(list.length);
|
|
1464
|
+
for (let i = 0; i < list.length; i++) {
|
|
1465
|
+
const v = list[i];
|
|
1466
|
+
if (v >>> 0 !== v) throw badId(v);
|
|
1467
|
+
out2[i] = v;
|
|
1468
|
+
}
|
|
1469
|
+
return out2;
|
|
1470
|
+
}
|
|
1471
|
+
let out = new Uint32Array(1024);
|
|
1472
|
+
let n = 0;
|
|
1473
|
+
for (const v of values) {
|
|
1474
|
+
if (n === out.length) {
|
|
1475
|
+
const grown = new Uint32Array(2 * n);
|
|
1476
|
+
grown.set(out);
|
|
1477
|
+
out = grown;
|
|
1478
|
+
}
|
|
1479
|
+
out[n++] = checkedId(v);
|
|
1480
|
+
}
|
|
1481
|
+
return out.subarray(0, n);
|
|
1482
|
+
}
|
|
1483
|
+
function collectLenient(values) {
|
|
1484
|
+
const list = Array.isArray(values) ? values : Array.from(values);
|
|
1485
|
+
const out = new Uint32Array(list.length);
|
|
1486
|
+
let n = 0;
|
|
1487
|
+
for (let i = 0; i < list.length; i++) {
|
|
1488
|
+
const v = list[i];
|
|
1489
|
+
if (v >>> 0 === v) out[n++] = v;
|
|
1490
|
+
}
|
|
1491
|
+
return out.subarray(0, n);
|
|
1492
|
+
}
|
|
1493
|
+
function sortedIds(ids, owned) {
|
|
1494
|
+
for (let i = 1; i < ids.length; i++) {
|
|
1495
|
+
if (ids[i - 1] > ids[i]) return (owned ? ids : ids.slice()).sort();
|
|
1496
|
+
}
|
|
1497
|
+
return ids;
|
|
1498
|
+
}
|
|
1499
|
+
function lowerBound2(keys, key, from = 0) {
|
|
1500
|
+
let lo = from;
|
|
1501
|
+
let hi = keys.length;
|
|
1502
|
+
while (lo < hi) {
|
|
1503
|
+
const mid = lo + hi >>> 1;
|
|
1504
|
+
if (keys[mid] < key) lo = mid + 1;
|
|
1505
|
+
else hi = mid;
|
|
1506
|
+
}
|
|
1507
|
+
return lo;
|
|
1508
|
+
}
|
|
1509
|
+
function gallopKeys(keys, key, from) {
|
|
1510
|
+
let step = 1;
|
|
1511
|
+
let lo = from;
|
|
1512
|
+
let hi = from;
|
|
1513
|
+
while (hi < keys.length && keys[hi] < key) {
|
|
1514
|
+
lo = hi + 1;
|
|
1515
|
+
hi += step;
|
|
1516
|
+
step *= 2;
|
|
1517
|
+
}
|
|
1518
|
+
return lowerBoundIn(keys, key, lo, Math.min(hi, keys.length));
|
|
1519
|
+
}
|
|
1520
|
+
function lowerBoundIn(keys, key, from, to) {
|
|
1521
|
+
let lo = from;
|
|
1522
|
+
let hi = to;
|
|
1523
|
+
while (lo < hi) {
|
|
1524
|
+
const mid = lo + hi >>> 1;
|
|
1525
|
+
if (keys[mid] < key) lo = mid + 1;
|
|
1526
|
+
else hi = mid;
|
|
1527
|
+
}
|
|
1528
|
+
return lo;
|
|
1529
|
+
}
|
|
1530
|
+
var OR = 0;
|
|
1531
|
+
var AND = 1;
|
|
1532
|
+
var AND_NOT = 2;
|
|
1533
|
+
function combineContainers(op, a, b) {
|
|
1534
|
+
if (op === OR) return containerOr(a, b, !a.shared);
|
|
1535
|
+
if (op === AND) return containerAnd(a, b, !a.shared);
|
|
1536
|
+
return containerAndNot(a, b, !a.shared);
|
|
1537
|
+
}
|
|
1538
|
+
var TsBitmap = class _TsBitmap {
|
|
1539
|
+
#keys;
|
|
1540
|
+
#conts;
|
|
1541
|
+
#size;
|
|
1542
|
+
constructor(keys, conts, size) {
|
|
1543
|
+
this.#keys = keys;
|
|
1544
|
+
this.#conts = conts;
|
|
1545
|
+
this.#size = size;
|
|
327
1546
|
}
|
|
328
1547
|
static empty() {
|
|
329
|
-
return new
|
|
1548
|
+
return new _TsBitmap([], [], 0);
|
|
330
1549
|
}
|
|
331
1550
|
static fromValues(values) {
|
|
332
|
-
|
|
1551
|
+
const bitmap = _TsBitmap.empty();
|
|
1552
|
+
bitmap.addMany(values);
|
|
1553
|
+
return bitmap;
|
|
333
1554
|
}
|
|
334
1555
|
/**
|
|
335
|
-
* Cap the size, check the structure, then
|
|
336
|
-
*
|
|
337
|
-
* decode — the native addon is never handed unbounded, malformed or unsafe-format input. With `whole`, bytes
|
|
338
|
-
* after the bitmap's last container are refused too; without it they are ignored, as the native decoder ignores
|
|
339
|
-
* them.
|
|
1556
|
+
* Cap the size, check the structure, then build the containers. With `whole`, bytes after the last container are
|
|
1557
|
+
* refused too; without it they are ignored.
|
|
340
1558
|
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
* values out of order, overlapping runs, a cardinality that disagrees with the bits) decode into a bitmap
|
|
344
|
-
* whose `maximum()`, `has()` and `size` are wrong, and some of those shapes crash the process when used.
|
|
1559
|
+
* @throws {IntegrityError} if the input exceeds `maxBytes`, is not a well-formed portable bitmap, or (with `whole`)
|
|
1560
|
+
* has bytes after it.
|
|
345
1561
|
*/
|
|
346
1562
|
static safeDeserialize(bytes, maxBytes, options = {}) {
|
|
347
1563
|
if (bytes.length > maxBytes) {
|
|
348
|
-
throw new
|
|
1564
|
+
throw new IntegrityError3(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
349
1565
|
}
|
|
350
|
-
const end =
|
|
1566
|
+
const { keys, conts, end } = decodeContainers(bytes);
|
|
351
1567
|
if (options.whole === true && end !== bytes.byteLength) {
|
|
352
|
-
throw
|
|
353
|
-
`portable roaring: ${bytes.byteLength - end} bytes follow the bitmap's last container, which ends at byte ${end}. One buffer holds one bitmap: to load several as one segment, combine them first (RoaringBitmap32.orMany) and serialize the result.`
|
|
354
|
-
);
|
|
355
|
-
}
|
|
356
|
-
try {
|
|
357
|
-
return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
|
|
358
|
-
} catch (err) {
|
|
359
|
-
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
1568
|
+
throw trailingBytes(bytes.byteLength, end);
|
|
360
1569
|
}
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
return this.bitmap;
|
|
1570
|
+
let size = 0;
|
|
1571
|
+
for (let i = 0; i < conts.length; i++) size += conts[i].card;
|
|
1572
|
+
return new _TsBitmap(keys, conts, size);
|
|
365
1573
|
}
|
|
366
1574
|
serialize() {
|
|
367
|
-
return this
|
|
1575
|
+
return serializeContainers(this.#keys, this.#conts);
|
|
368
1576
|
}
|
|
369
|
-
/**
|
|
370
|
-
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
371
|
-
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
372
|
-
*
|
|
373
|
-
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
374
|
-
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
375
|
-
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
376
|
-
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
377
|
-
* whichever encoding is smaller per container.
|
|
378
|
-
*
|
|
379
|
-
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
380
|
-
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
381
|
-
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
382
|
-
* columns; don't carry a figure between them without checking which denominator it had.
|
|
383
|
-
*
|
|
384
|
-
* Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
|
|
385
|
-
* path deliberately does not.
|
|
386
|
-
*
|
|
387
|
-
* CANONICAL, which `runOptimize()` alone is not. Where a container's run and array encodings are the same size
|
|
388
|
-
* (cardinality = 2 × runs + 1: three values in one run, five in two), CRoaring keeps whichever kind the
|
|
389
|
-
* container already has. A chunk built from ids starts as an array and stays one; the same set reached through
|
|
390
|
-
* `addRange`, an earlier `runOptimize()` or an operation over run containers is a run, and serializes to other
|
|
391
|
-
* bytes. Undoing run compression first starts every container where one built from ids starts, so the bytes
|
|
392
|
-
* depend on membership alone. On a bitmap built from ids there is nothing to undo.
|
|
393
|
-
*/
|
|
1577
|
+
/** Re-encode each container canonically; membership is untouched. */
|
|
394
1578
|
optimize() {
|
|
395
|
-
this
|
|
396
|
-
|
|
1579
|
+
const conts = this.#conts;
|
|
1580
|
+
for (let i = 0; i < conts.length; i++) conts[i] = optimize(conts[i]);
|
|
397
1581
|
}
|
|
398
1582
|
/**
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
*
|
|
1583
|
+
* The set as storage chunks: each payload is the portable bytes of a one-container bitmap holding that chunk's low 16
|
|
1584
|
+
* bits, optimized the way {@link optimize} does. Nothing is built until a chunk is asked for, so a writer that yields
|
|
1585
|
+
* between chunks runs this in slices. The call snapshots the containers and marks each shared, so a change to the
|
|
1586
|
+
* bitmap while the chunks are read copies a container instead of altering what the chunks will say. The bitmap's own
|
|
1587
|
+
* containers are not optimized by it.
|
|
404
1588
|
*/
|
|
405
1589
|
encodeChunks() {
|
|
406
|
-
this.
|
|
407
|
-
|
|
1590
|
+
const keys = this.#keys.slice();
|
|
1591
|
+
const conts = this.#conts.slice();
|
|
1592
|
+
for (let i = 0; i < conts.length; i++) conts[i].shared = true;
|
|
1593
|
+
return {
|
|
1594
|
+
*[Symbol.iterator]() {
|
|
1595
|
+
for (let i = 0; i < conts.length; i++) {
|
|
1596
|
+
const c = optimize(conts[i]);
|
|
1597
|
+
yield {
|
|
1598
|
+
chunkKey: keys[i],
|
|
1599
|
+
payload: serializeContainers([0], [c]),
|
|
1600
|
+
cardinality: c.card
|
|
1601
|
+
};
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
};
|
|
408
1605
|
}
|
|
409
|
-
/**
|
|
410
|
-
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the last container, so the engine's
|
|
411
|
-
* per-chunk range assertion costs one call per chunk rather than a walk per id. The last container holds the
|
|
412
|
-
* largest value only because {@link SafeBitmap.safeDeserialize} refuses bytes whose containers or values are
|
|
413
|
-
* out of order.
|
|
414
|
-
*/
|
|
415
1606
|
maximum() {
|
|
416
|
-
|
|
1607
|
+
const last = this.#conts.length - 1;
|
|
1608
|
+
if (last < 0) return void 0;
|
|
1609
|
+
return this.#keys[last] * 65536 + containerMax(this.#conts[last]);
|
|
417
1610
|
}
|
|
418
1611
|
add(value) {
|
|
419
|
-
|
|
1612
|
+
checkedId(value);
|
|
1613
|
+
const key = value >>> 16;
|
|
1614
|
+
const low = value & 65535;
|
|
1615
|
+
const at = lowerBound2(this.#keys, key);
|
|
1616
|
+
const found = this.#keys[at] === key;
|
|
1617
|
+
if (!found) {
|
|
1618
|
+
this.#insert(at, key, arrayContainer(Uint16Array.of(low), 1));
|
|
1619
|
+
this.#size++;
|
|
1620
|
+
return;
|
|
1621
|
+
}
|
|
1622
|
+
const before = this.#conts[at];
|
|
1623
|
+
const after = containerAdd(before, low);
|
|
1624
|
+
this.#conts[at] = after;
|
|
1625
|
+
this.#size += after.card - before.card;
|
|
420
1626
|
}
|
|
1627
|
+
/**
|
|
1628
|
+
* Add the ids. They are grouped by key in ascending order, so each container is touched once, a key already held is
|
|
1629
|
+
* merged where it is, and the new keys are put in with one pass over the key list however many there are.
|
|
1630
|
+
*/
|
|
421
1631
|
addMany(values) {
|
|
422
|
-
|
|
1632
|
+
const ids = sortedIds(collect(values), !(values instanceof Uint32Array));
|
|
1633
|
+
const fresh = [];
|
|
1634
|
+
let at = 0;
|
|
1635
|
+
for (let i = 0; i < ids.length; ) {
|
|
1636
|
+
const key = ids[i] >>> 16;
|
|
1637
|
+
let end = i + 1;
|
|
1638
|
+
while (end < ids.length && ids[end] >>> 16 === key) end++;
|
|
1639
|
+
const group = fromSortedIds(ids, i, end);
|
|
1640
|
+
i = end;
|
|
1641
|
+
at = gallopKeys(this.#keys, key, at);
|
|
1642
|
+
if (this.#keys[at] === key) {
|
|
1643
|
+
const before = this.#conts[at];
|
|
1644
|
+
const after = containerOr(before, group, !before.shared);
|
|
1645
|
+
this.#conts[at] = after;
|
|
1646
|
+
this.#size += after.card - before.card;
|
|
1647
|
+
} else {
|
|
1648
|
+
fresh.push({ key, group });
|
|
1649
|
+
this.#size += group.card;
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
if (fresh.length > 0) this.#insertAll(fresh);
|
|
423
1653
|
}
|
|
424
1654
|
remove(value) {
|
|
425
|
-
|
|
1655
|
+
if (!isId(value)) return;
|
|
1656
|
+
const key = value >>> 16;
|
|
1657
|
+
const at = lowerBound2(this.#keys, key);
|
|
1658
|
+
if (this.#keys[at] !== key) return;
|
|
1659
|
+
const before = this.#conts[at];
|
|
1660
|
+
const after = containerRemove(before, value & 65535);
|
|
1661
|
+
if (after === null) {
|
|
1662
|
+
this.#keys.splice(at, 1);
|
|
1663
|
+
this.#conts.splice(at, 1);
|
|
1664
|
+
this.#size -= before.card;
|
|
1665
|
+
return;
|
|
1666
|
+
}
|
|
1667
|
+
this.#conts[at] = after;
|
|
1668
|
+
this.#size += after.card - before.card;
|
|
426
1669
|
}
|
|
1670
|
+
/** Remove the ids (anything that is not an id is ignored), grouped by key; the keys left empty go in one pass. */
|
|
427
1671
|
removeMany(values) {
|
|
428
|
-
|
|
1672
|
+
const ids = sortedIds(collectLenient(values), true);
|
|
1673
|
+
const keys = this.#keys;
|
|
1674
|
+
const conts = this.#conts;
|
|
1675
|
+
let at = 0;
|
|
1676
|
+
let emptied = 0;
|
|
1677
|
+
for (let i = 0; i < ids.length; ) {
|
|
1678
|
+
const key = ids[i] >>> 16;
|
|
1679
|
+
let end = i + 1;
|
|
1680
|
+
while (end < ids.length && ids[end] >>> 16 === key) end++;
|
|
1681
|
+
const from = i;
|
|
1682
|
+
i = end;
|
|
1683
|
+
at = gallopKeys(keys, key, at);
|
|
1684
|
+
if (keys[at] !== key) continue;
|
|
1685
|
+
const before = conts[at];
|
|
1686
|
+
if (before.card === 0) continue;
|
|
1687
|
+
const after = containerAndNot(before, fromSortedIds(ids, from, end), !before.shared);
|
|
1688
|
+
this.#size -= before.card - (after === null ? 0 : after.card);
|
|
1689
|
+
if (after === null) {
|
|
1690
|
+
conts[at] = EMPTIED;
|
|
1691
|
+
emptied++;
|
|
1692
|
+
} else {
|
|
1693
|
+
conts[at] = after;
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
if (emptied > 0) this.#dropEmptied();
|
|
429
1697
|
}
|
|
430
1698
|
has(value) {
|
|
431
|
-
|
|
1699
|
+
if (!isId(value)) return false;
|
|
1700
|
+
const key = value >>> 16;
|
|
1701
|
+
const at = lowerBound2(this.#keys, key);
|
|
1702
|
+
return this.#keys[at] === key && containerHas(this.#conts[at], value & 65535);
|
|
432
1703
|
}
|
|
433
1704
|
get size() {
|
|
434
|
-
return this
|
|
1705
|
+
return this.#size;
|
|
435
1706
|
}
|
|
436
1707
|
get isEmpty() {
|
|
437
|
-
return this
|
|
1708
|
+
return this.#size === 0;
|
|
438
1709
|
}
|
|
439
1710
|
clone() {
|
|
440
|
-
|
|
1711
|
+
const conts = new Array(this.#conts.length);
|
|
1712
|
+
for (let i = 0; i < conts.length; i++) {
|
|
1713
|
+
const c = this.#conts[i];
|
|
1714
|
+
c.shared = true;
|
|
1715
|
+
conts[i] = c;
|
|
1716
|
+
}
|
|
1717
|
+
return new _TsBitmap(this.#keys.slice(), conts, this.#size);
|
|
441
1718
|
}
|
|
442
|
-
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
443
|
-
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
444
|
-
// `bitmap` (accessible on same-class instances) for the native op.
|
|
445
|
-
/** In-place union: `this = this ∪ other`. */
|
|
446
1719
|
orInPlace(other) {
|
|
447
|
-
this
|
|
1720
|
+
this.#combine(_TsBitmap.#of(other), OR);
|
|
448
1721
|
}
|
|
449
|
-
/** In-place difference: `this = this \ other`. */
|
|
450
|
-
andNotInPlace(other) {
|
|
451
|
-
this.bitmap.andNotInPlace(other.bitmap);
|
|
452
|
-
}
|
|
453
|
-
/** In-place intersection: `this = this ∩ other`. */
|
|
454
1722
|
andInPlace(other) {
|
|
455
|
-
this
|
|
1723
|
+
this.#combine(_TsBitmap.#of(other), AND);
|
|
1724
|
+
}
|
|
1725
|
+
andNotInPlace(other) {
|
|
1726
|
+
this.#combine(_TsBitmap.#of(other), AND_NOT);
|
|
456
1727
|
}
|
|
457
|
-
/** Ascending iterator over the set values. */
|
|
458
1728
|
[Symbol.iterator]() {
|
|
459
|
-
return this
|
|
1729
|
+
return new BitmapIterator(this.#keys, this.#conts);
|
|
460
1730
|
}
|
|
461
1731
|
toArray() {
|
|
462
|
-
return this.
|
|
1732
|
+
return Array.from(this.toUint32Array());
|
|
463
1733
|
}
|
|
464
|
-
/** The
|
|
1734
|
+
/** The ids as a new ascending `Uint32Array`. */
|
|
465
1735
|
toUint32Array() {
|
|
466
|
-
|
|
1736
|
+
const out = new Uint32Array(this.#size);
|
|
1737
|
+
let at = 0;
|
|
1738
|
+
for (let i = 0; i < this.#conts.length; i++) {
|
|
1739
|
+
at = appendIds(this.#conts[i], out, at, this.#keys[i] * 65536);
|
|
1740
|
+
}
|
|
1741
|
+
return out;
|
|
1742
|
+
}
|
|
1743
|
+
static #of(other) {
|
|
1744
|
+
if (!(other instanceof _TsBitmap)) {
|
|
1745
|
+
throw new ValidationError2("a TypeScript roaring bitmap combines only with another one");
|
|
1746
|
+
}
|
|
1747
|
+
return other;
|
|
467
1748
|
}
|
|
1749
|
+
#insert(at, key, container) {
|
|
1750
|
+
if (at === this.#keys.length) {
|
|
1751
|
+
this.#keys.push(key);
|
|
1752
|
+
this.#conts.push(container);
|
|
1753
|
+
} else {
|
|
1754
|
+
this.#keys.splice(at, 0, key);
|
|
1755
|
+
this.#conts.splice(at, 0, container);
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
/** Put the new keys, ascending and none held, in: after the last key by pushing, else in one merge pass. */
|
|
1759
|
+
#insertAll(fresh) {
|
|
1760
|
+
const keys = this.#keys;
|
|
1761
|
+
const conts = this.#conts;
|
|
1762
|
+
if (keys.length === 0 || keys[keys.length - 1] < fresh[0].key) {
|
|
1763
|
+
for (const { key, group } of fresh) {
|
|
1764
|
+
keys.push(key);
|
|
1765
|
+
conts.push(group);
|
|
1766
|
+
}
|
|
1767
|
+
return;
|
|
1768
|
+
}
|
|
1769
|
+
const outKeys = [];
|
|
1770
|
+
const outConts = [];
|
|
1771
|
+
let i = 0;
|
|
1772
|
+
let j = 0;
|
|
1773
|
+
while (i < keys.length || j < fresh.length) {
|
|
1774
|
+
if (j === fresh.length || i < keys.length && keys[i] < fresh[j].key) {
|
|
1775
|
+
outKeys.push(keys[i]);
|
|
1776
|
+
outConts.push(conts[i++]);
|
|
1777
|
+
} else {
|
|
1778
|
+
const next = fresh[j++];
|
|
1779
|
+
outKeys.push(next.key);
|
|
1780
|
+
outConts.push(next.group);
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
this.#keys = outKeys;
|
|
1784
|
+
this.#conts = outConts;
|
|
1785
|
+
}
|
|
1786
|
+
/** Remove the keys whose container was emptied by {@link removeMany}, in one pass. */
|
|
1787
|
+
#dropEmptied() {
|
|
1788
|
+
let kept = 0;
|
|
1789
|
+
for (let i = 0; i < this.#conts.length; i++) {
|
|
1790
|
+
if (this.#conts[i] === EMPTIED) continue;
|
|
1791
|
+
this.#keys[kept] = this.#keys[i];
|
|
1792
|
+
this.#conts[kept++] = this.#conts[i];
|
|
1793
|
+
}
|
|
1794
|
+
this.#keys.length = kept;
|
|
1795
|
+
this.#conts.length = kept;
|
|
1796
|
+
}
|
|
1797
|
+
/**
|
|
1798
|
+
* Walk both key lists and rebuild this one: where both hold a key, the combine of the two containers; where only this
|
|
1799
|
+
* does, its container, unless the operation is an intersection; where only `that` does, a copy of its container for a
|
|
1800
|
+
* union and nothing otherwise. A combine that leaves no value drops the key. An intersection jumps over a stretch of
|
|
1801
|
+
* keys the other side lacks by doubling steps, so a few keys against many cost the few.
|
|
1802
|
+
*/
|
|
1803
|
+
#combine(that, op) {
|
|
1804
|
+
const a = this.#keys;
|
|
1805
|
+
const b = that.#keys;
|
|
1806
|
+
const ca = this.#conts;
|
|
1807
|
+
const cb = that.#conts;
|
|
1808
|
+
const keys = [];
|
|
1809
|
+
const conts = [];
|
|
1810
|
+
let size = 0;
|
|
1811
|
+
let i = 0;
|
|
1812
|
+
let j = 0;
|
|
1813
|
+
while (i < a.length && j < b.length) {
|
|
1814
|
+
const ka = a[i];
|
|
1815
|
+
const kb = b[j];
|
|
1816
|
+
if (ka === kb) {
|
|
1817
|
+
const c = combineContainers(op, ca[i++], cb[j++]);
|
|
1818
|
+
if (c !== null) {
|
|
1819
|
+
keys.push(ka);
|
|
1820
|
+
conts.push(c);
|
|
1821
|
+
size += c.card;
|
|
1822
|
+
}
|
|
1823
|
+
} else if (ka < kb) {
|
|
1824
|
+
if (op === AND) {
|
|
1825
|
+
i = gallopKeys(a, kb, i + 1);
|
|
1826
|
+
} else {
|
|
1827
|
+
keys.push(ka);
|
|
1828
|
+
conts.push(ca[i]);
|
|
1829
|
+
size += ca[i++].card;
|
|
1830
|
+
}
|
|
1831
|
+
} else if (op === AND) {
|
|
1832
|
+
j = gallopKeys(b, ka, j + 1);
|
|
1833
|
+
} else {
|
|
1834
|
+
if (op === OR) {
|
|
1835
|
+
keys.push(kb);
|
|
1836
|
+
conts.push(cloneContainer(cb[j]));
|
|
1837
|
+
size += cb[j].card;
|
|
1838
|
+
}
|
|
1839
|
+
j++;
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
if (op !== AND) {
|
|
1843
|
+
for (; i < a.length; i++) {
|
|
1844
|
+
keys.push(a[i]);
|
|
1845
|
+
conts.push(ca[i]);
|
|
1846
|
+
size += ca[i].card;
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
if (op === OR) {
|
|
1850
|
+
for (; j < b.length; j++) {
|
|
1851
|
+
keys.push(b[j]);
|
|
1852
|
+
conts.push(cloneContainer(cb[j]));
|
|
1853
|
+
size += cb[j].card;
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
this.#keys = keys;
|
|
1857
|
+
this.#conts = conts;
|
|
1858
|
+
this.#size = size;
|
|
1859
|
+
}
|
|
1860
|
+
};
|
|
1861
|
+
var EMPTIED = arrayContainer(new Uint16Array(0), 0);
|
|
1862
|
+
var tsRoaringCodec = {
|
|
1863
|
+
empty: () => TsBitmap.empty(),
|
|
1864
|
+
fromValues: (values) => TsBitmap.fromValues(values),
|
|
1865
|
+
safeDeserialize: (bytes, maxBytes, options) => TsBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
1866
|
+
owns: (bitmap) => bitmap instanceof TsBitmap,
|
|
1867
|
+
intersectSmallestFirst: true
|
|
468
1868
|
};
|
|
1869
|
+
|
|
1870
|
+
// src/roaring-codec.ts
|
|
469
1871
|
function bitmapAsLoadInput(input) {
|
|
470
|
-
|
|
1872
|
+
if (typeof input !== "object" || input === null) return input;
|
|
1873
|
+
const tag = Object.prototype.toString.call(input).slice(8, -1);
|
|
1874
|
+
if (tag === "Uint8Array" || tag === "Uint8ClampedArray") return input;
|
|
1875
|
+
return Symbol.iterator in input && typeof input.serialize === "function" ? { bitmap: input } : input;
|
|
1876
|
+
}
|
|
1877
|
+
var activeCodec = tsRoaringCodec;
|
|
1878
|
+
function flagshipCodec() {
|
|
1879
|
+
return activeCodec;
|
|
471
1880
|
}
|
|
472
|
-
var roaringCodec = {
|
|
473
|
-
empty: () => SafeBitmap.empty(),
|
|
474
|
-
fromValues: (values) => SafeBitmap.fromValues(values),
|
|
475
|
-
safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
476
|
-
owns: (bitmap) => bitmap instanceof SafeBitmap
|
|
477
|
-
};
|
|
478
1881
|
|
|
479
1882
|
// src/system-clock.ts
|
|
480
1883
|
var SystemClock = class {
|
|
@@ -614,17 +2017,18 @@ import {
|
|
|
614
2017
|
DEFAULT_RETRY_POLICY,
|
|
615
2018
|
NOOP_METRICS,
|
|
616
2019
|
RetryingStorageChunkSource,
|
|
617
|
-
ValidationError as
|
|
2020
|
+
ValidationError as ValidationError4,
|
|
618
2021
|
isStorageBackend
|
|
619
2022
|
} from "@cloudbitmaps/core";
|
|
620
2023
|
function resolveRetryPolicy(partial) {
|
|
621
|
-
const
|
|
2024
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
2025
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
622
2026
|
return {
|
|
623
|
-
maxAttempts:
|
|
2027
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
624
2028
|
baseDelayMs,
|
|
625
|
-
maxDelayMs: partial.maxDelayMs
|
|
626
|
-
backoffFactor:
|
|
627
|
-
jitter:
|
|
2029
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
2030
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
2031
|
+
jitter: orDefault("jitter")
|
|
628
2032
|
};
|
|
629
2033
|
}
|
|
630
2034
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -637,7 +2041,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
637
2041
|
if (options.retry !== false) {
|
|
638
2042
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
639
2043
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
640
|
-
throw new
|
|
2044
|
+
throw new ValidationError4("retry.onRetry must be a function");
|
|
641
2045
|
}
|
|
642
2046
|
retryOptions = {
|
|
643
2047
|
clock,
|
|
@@ -661,13 +2065,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
661
2065
|
source: resolved.source,
|
|
662
2066
|
driver: resolved.driver,
|
|
663
2067
|
registry: resolved.registry,
|
|
2068
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
664
2069
|
retryOptions
|
|
665
2070
|
};
|
|
666
2071
|
}
|
|
667
2072
|
function resolveStorageSource(options, clock) {
|
|
668
2073
|
const storage = options.storage;
|
|
669
2074
|
if (storage === null || typeof storage !== "object") {
|
|
670
|
-
throw new
|
|
2075
|
+
throw new ValidationError4(
|
|
671
2076
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
672
2077
|
);
|
|
673
2078
|
}
|
|
@@ -677,14 +2082,14 @@ function resolveStorageSource(options, clock) {
|
|
|
677
2082
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
678
2083
|
const isBackend = isStorageBackend(storage);
|
|
679
2084
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
680
|
-
throw new
|
|
2085
|
+
throw new ValidationError4(
|
|
681
2086
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
682
2087
|
);
|
|
683
2088
|
}
|
|
684
2089
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
685
2090
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
686
2091
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
687
|
-
throw new
|
|
2092
|
+
throw new ValidationError4(
|
|
688
2093
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
689
2094
|
);
|
|
690
2095
|
}
|
|
@@ -693,11 +2098,11 @@ function resolveStorageSource(options, clock) {
|
|
|
693
2098
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
694
2099
|
if (!storageOk || !registryOk) {
|
|
695
2100
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
696
|
-
throw new
|
|
2101
|
+
throw new ValidationError4(
|
|
697
2102
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
698
2103
|
);
|
|
699
2104
|
}
|
|
700
|
-
throw new
|
|
2105
|
+
throw new ValidationError4(
|
|
701
2106
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
702
2107
|
);
|
|
703
2108
|
}
|
|
@@ -718,18 +2123,18 @@ function resolveStorageSource(options, clock) {
|
|
|
718
2123
|
};
|
|
719
2124
|
}
|
|
720
2125
|
if (hasGetChunk && hasPutImmutable) {
|
|
721
|
-
throw new
|
|
2126
|
+
throw new ValidationError4(
|
|
722
2127
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
723
2128
|
);
|
|
724
2129
|
}
|
|
725
2130
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
726
|
-
throw new
|
|
2131
|
+
throw new ValidationError4(
|
|
727
2132
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
728
2133
|
);
|
|
729
2134
|
}
|
|
730
2135
|
if (hasGetChunk) {
|
|
731
2136
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
732
|
-
throw new
|
|
2137
|
+
throw new ValidationError4(
|
|
733
2138
|
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
734
2139
|
);
|
|
735
2140
|
}
|
|
@@ -751,32 +2156,32 @@ function resolveStorageSource(options, clock) {
|
|
|
751
2156
|
}
|
|
752
2157
|
|
|
753
2158
|
// src/options.ts
|
|
754
|
-
import { ValidationError as
|
|
2159
|
+
import { ValidationError as ValidationError5 } from "@cloudbitmaps/core";
|
|
755
2160
|
var keysOf = () => (keys) => keys;
|
|
756
2161
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
757
2162
|
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
758
2163
|
if (options === void 0 || options === null) return void 0;
|
|
759
2164
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
760
|
-
return new
|
|
2165
|
+
return new ValidationError5(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
761
2166
|
}
|
|
762
2167
|
const bag = options;
|
|
763
2168
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
764
2169
|
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
765
2170
|
if (unknown.length > 0) {
|
|
766
|
-
return new
|
|
2171
|
+
return new ValidationError5(
|
|
767
2172
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
768
2173
|
);
|
|
769
2174
|
}
|
|
770
2175
|
const audit = bag.audit;
|
|
771
2176
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
772
|
-
return new
|
|
2177
|
+
return new ValidationError5(
|
|
773
2178
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
774
2179
|
);
|
|
775
2180
|
}
|
|
776
2181
|
for (const name of booleans) {
|
|
777
2182
|
const v = bag[name];
|
|
778
2183
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
779
|
-
return new
|
|
2184
|
+
return new ValidationError5(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
780
2185
|
}
|
|
781
2186
|
}
|
|
782
2187
|
return void 0;
|
|
@@ -796,28 +2201,24 @@ function checkOptions(options, op, known, booleans, misplaced) {
|
|
|
796
2201
|
}
|
|
797
2202
|
|
|
798
2203
|
// src/index.ts
|
|
2204
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
799
2205
|
import {
|
|
800
|
-
MemoryStorage,
|
|
801
|
-
LocalFsStorage,
|
|
802
|
-
destroySegment,
|
|
803
|
-
eraseNamespace,
|
|
804
2206
|
excludingReservedRows as excludingReservedRows2,
|
|
805
2207
|
readRetentionPolicy,
|
|
806
|
-
InProcessKeystore,
|
|
807
2208
|
CloudRoaringError,
|
|
808
|
-
ValidationError as
|
|
2209
|
+
ValidationError as ValidationError7,
|
|
809
2210
|
WriteConflictError as WriteConflictError2,
|
|
810
|
-
IntegrityError as
|
|
2211
|
+
IntegrityError as IntegrityError4,
|
|
811
2212
|
NotFoundError as NotFoundError2,
|
|
812
2213
|
UnsupportedError as UnsupportedError3,
|
|
813
2214
|
CapabilityError,
|
|
814
2215
|
TransientError,
|
|
815
2216
|
KeyUnavailableError,
|
|
816
|
-
BudgetExceededError,
|
|
2217
|
+
BudgetExceededError as BudgetExceededError2,
|
|
817
2218
|
LeaseExpiredError as LeaseExpiredError2,
|
|
818
2219
|
LeaseLimitError,
|
|
819
2220
|
StaleOperandError,
|
|
820
|
-
errorKind,
|
|
2221
|
+
errorKind as errorKind2,
|
|
821
2222
|
CountingMetricsSink,
|
|
822
2223
|
RecordingAuditSink
|
|
823
2224
|
} from "@cloudbitmaps/core";
|
|
@@ -827,13 +2228,15 @@ var SystemRng = class {
|
|
|
827
2228
|
return Math.random();
|
|
828
2229
|
}
|
|
829
2230
|
};
|
|
830
|
-
function validateRef(ref) {
|
|
2231
|
+
function validateRef(ref, verb) {
|
|
831
2232
|
if (typeof ref !== "object" || ref === null) {
|
|
832
|
-
throw new
|
|
2233
|
+
throw new ValidationError6(
|
|
833
2234
|
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
834
2235
|
);
|
|
835
2236
|
}
|
|
2237
|
+
refuseUnknownRefKeys(ref, verb);
|
|
836
2238
|
validateSegmentRef(ref);
|
|
2239
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
837
2240
|
}
|
|
838
2241
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
839
2242
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
@@ -848,15 +2251,20 @@ function validateConcurrency(concurrency) {
|
|
|
848
2251
|
function requireScope(options, op) {
|
|
849
2252
|
if (options.namespace !== void 0) {
|
|
850
2253
|
if (typeof options.namespace !== "string") {
|
|
851
|
-
throw new
|
|
2254
|
+
throw new ValidationError6(
|
|
852
2255
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
853
2256
|
);
|
|
854
2257
|
}
|
|
855
2258
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
856
2259
|
}
|
|
857
2260
|
refuseReservedNamespace(options.namespace);
|
|
2261
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
2262
|
+
throw new ValidationError6(
|
|
2263
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
2264
|
+
);
|
|
2265
|
+
}
|
|
858
2266
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
859
|
-
throw new
|
|
2267
|
+
throw new ValidationError6(
|
|
860
2268
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
861
2269
|
);
|
|
862
2270
|
}
|
|
@@ -903,7 +2311,7 @@ var CALL_KEYS = {
|
|
|
903
2311
|
"metadata",
|
|
904
2312
|
"keep"
|
|
905
2313
|
]),
|
|
906
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
2314
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
907
2315
|
segments: keysOf()(["namespace"]),
|
|
908
2316
|
subjectReport: keysOf()([
|
|
909
2317
|
"namespace",
|
|
@@ -964,16 +2372,16 @@ var originOf;
|
|
|
964
2372
|
function assertOneStore(verb, role, h, origin) {
|
|
965
2373
|
if (h instanceof Segment) {
|
|
966
2374
|
if (originOf(h) === origin) return;
|
|
967
|
-
throw new
|
|
2375
|
+
throw new ValidationError6(
|
|
968
2376
|
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
969
2377
|
);
|
|
970
2378
|
}
|
|
971
2379
|
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
972
|
-
throw new
|
|
2380
|
+
throw new ValidationError6(
|
|
973
2381
|
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
974
2382
|
);
|
|
975
2383
|
}
|
|
976
|
-
throw new
|
|
2384
|
+
throw new ValidationError6(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
977
2385
|
}
|
|
978
2386
|
var MATERIALIZE_MANY_KEYS = [
|
|
979
2387
|
"operands",
|
|
@@ -1042,6 +2450,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1042
2450
|
// no underlying driver to write through).
|
|
1043
2451
|
storageDriver;
|
|
1044
2452
|
registry;
|
|
2453
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
2454
|
+
backend;
|
|
1045
2455
|
keystore;
|
|
1046
2456
|
requireEncryption;
|
|
1047
2457
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1061,7 +2471,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1061
2471
|
*/
|
|
1062
2472
|
static rejectUnknownOptions(options) {
|
|
1063
2473
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1064
|
-
throw new
|
|
2474
|
+
throw new ValidationError6(
|
|
1065
2475
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1066
2476
|
);
|
|
1067
2477
|
}
|
|
@@ -1079,7 +2489,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1079
2489
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1080
2490
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1081
2491
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1082
|
-
throw new
|
|
2492
|
+
throw new ValidationError6(
|
|
1083
2493
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1084
2494
|
);
|
|
1085
2495
|
}
|
|
@@ -1089,7 +2499,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1089
2499
|
if (unknown.length === 0) return;
|
|
1090
2500
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1091
2501
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1092
|
-
throw new
|
|
2502
|
+
throw new ValidationError6(
|
|
1093
2503
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1094
2504
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1095
2505
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1107,23 +2517,51 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1107
2517
|
];
|
|
1108
2518
|
const entropy = options.seams?.entropy;
|
|
1109
2519
|
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1110
|
-
throw new
|
|
2520
|
+
throw new ValidationError6(
|
|
1111
2521
|
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1112
2522
|
);
|
|
1113
2523
|
}
|
|
1114
2524
|
for (const [seam, name, needs] of seams) {
|
|
1115
|
-
if (seam === void 0
|
|
1116
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1117
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1118
|
-
throw new
|
|
2525
|
+
if (seam === void 0) continue;
|
|
2526
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
2527
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
2528
|
+
throw new ValidationError6(
|
|
1119
2529
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1120
2530
|
);
|
|
1121
2531
|
}
|
|
1122
2532
|
}
|
|
1123
2533
|
}
|
|
2534
|
+
/**
|
|
2535
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
2536
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
2537
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
2538
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
2539
|
+
*/
|
|
2540
|
+
static rejectBadValues(options) {
|
|
2541
|
+
const cache = options.cache;
|
|
2542
|
+
if (cache !== void 0) {
|
|
2543
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
2544
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
2545
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
2546
|
+
const ttl = cache.ttlMs;
|
|
2547
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
2548
|
+
throw new ValidationError6(
|
|
2549
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
2550
|
+
);
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
2554
|
+
}
|
|
2555
|
+
/**
|
|
2556
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
2557
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
2558
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
2559
|
+
* calls, throws `ValidationError`.
|
|
2560
|
+
*/
|
|
1124
2561
|
constructor(options) {
|
|
1125
2562
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1126
2563
|
_CloudRoaring.rejectHollowSeams(options);
|
|
2564
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1127
2565
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1128
2566
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1129
2567
|
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
@@ -1139,7 +2577,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1139
2577
|
const deps = {
|
|
1140
2578
|
storage: wired.storage,
|
|
1141
2579
|
cache,
|
|
1142
|
-
codec:
|
|
2580
|
+
codec: flagshipCodec(),
|
|
1143
2581
|
// the facade injects the flagship codec; core stays codec-agnostic
|
|
1144
2582
|
clock,
|
|
1145
2583
|
metrics,
|
|
@@ -1154,6 +2592,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1154
2592
|
this.metrics = metrics;
|
|
1155
2593
|
this.storageDriver = wired.driver;
|
|
1156
2594
|
this.registry = wired.registry;
|
|
2595
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
2596
|
+
this.backend = wired.backend;
|
|
1157
2597
|
this.keystore = options.encryption?.keystore;
|
|
1158
2598
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1159
2599
|
}
|
|
@@ -1180,7 +2620,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1180
2620
|
clock: this.clock,
|
|
1181
2621
|
rng: this.rng,
|
|
1182
2622
|
entropy: this.entropy,
|
|
1183
|
-
codec:
|
|
2623
|
+
codec: flagshipCodec(),
|
|
1184
2624
|
// facade injects the flagship codec
|
|
1185
2625
|
keystore: this.keystore,
|
|
1186
2626
|
requireEncryption: this.requireEncryption,
|
|
@@ -1197,17 +2637,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1197
2637
|
*/
|
|
1198
2638
|
segment(name, options) {
|
|
1199
2639
|
if (typeof name === "object" && name !== null) {
|
|
1200
|
-
throw new
|
|
2640
|
+
throw new ValidationError6(
|
|
1201
2641
|
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1202
2642
|
);
|
|
1203
2643
|
}
|
|
1204
2644
|
if (options !== void 0 && options !== null) {
|
|
1205
2645
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1206
|
-
throw new
|
|
2646
|
+
throw new ValidationError6("segment: options must be an object such as { namespace }");
|
|
1207
2647
|
}
|
|
1208
2648
|
const proto = Object.getPrototypeOf(options);
|
|
1209
2649
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1210
|
-
throw new
|
|
2650
|
+
throw new ValidationError6(
|
|
1211
2651
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1212
2652
|
);
|
|
1213
2653
|
}
|
|
@@ -1215,7 +2655,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1215
2655
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1216
2656
|
);
|
|
1217
2657
|
if (unknown.length > 0) {
|
|
1218
|
-
throw new
|
|
2658
|
+
throw new ValidationError6(
|
|
1219
2659
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1220
2660
|
);
|
|
1221
2661
|
}
|
|
@@ -1297,7 +2737,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1297
2737
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1298
2738
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1299
2739
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1300
|
-
keep: options?.keep
|
|
2740
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1301
2741
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1302
2742
|
};
|
|
1303
2743
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1325,20 +2765,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1325
2765
|
/** The one body of the forms above. */
|
|
1326
2766
|
async materializeMany(options) {
|
|
1327
2767
|
if (typeof options !== "object" || options === null) {
|
|
1328
|
-
throw new
|
|
2768
|
+
throw new ValidationError6(
|
|
1329
2769
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1330
2770
|
);
|
|
1331
2771
|
}
|
|
1332
2772
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1333
2773
|
const unknown = unknownManyKeys(options);
|
|
1334
2774
|
if (unknown.length > 0) {
|
|
1335
|
-
throw new
|
|
2775
|
+
throw new ValidationError6(
|
|
1336
2776
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1337
2777
|
);
|
|
1338
2778
|
}
|
|
1339
2779
|
const dryRunIn = options.dryRun;
|
|
1340
2780
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1341
|
-
throw new
|
|
2781
|
+
throw new ValidationError6("materializeMany: dryRun must be a boolean");
|
|
1342
2782
|
}
|
|
1343
2783
|
const dryRun = dryRunIn === true;
|
|
1344
2784
|
const operandsIn = options.operands;
|
|
@@ -1350,7 +2790,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1350
2790
|
const feedIn = options.feed;
|
|
1351
2791
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1352
2792
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1353
|
-
throw new
|
|
2793
|
+
throw new ValidationError6(
|
|
1354
2794
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1355
2795
|
);
|
|
1356
2796
|
}
|
|
@@ -1360,15 +2800,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1360
2800
|
const budgetOption = options.budget;
|
|
1361
2801
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1362
2802
|
if (typeof pinOption !== "boolean")
|
|
1363
|
-
throw new
|
|
2803
|
+
throw new ValidationError6("materializeMany: pin must be a boolean");
|
|
1364
2804
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1365
|
-
throw new
|
|
2805
|
+
throw new ValidationError6("materializeMany: allowAbsentOperands must be a boolean");
|
|
1366
2806
|
}
|
|
1367
2807
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1368
|
-
throw new
|
|
2808
|
+
throw new ValidationError6("materializeMany: operands must be an object of name to segment");
|
|
1369
2809
|
}
|
|
1370
2810
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1371
|
-
throw new
|
|
2811
|
+
throw new ValidationError6("materializeMany: outputs must be a non-empty array");
|
|
1372
2812
|
}
|
|
1373
2813
|
const names = Object.keys(operandsIn);
|
|
1374
2814
|
const handles = /* @__PURE__ */ new Map();
|
|
@@ -1379,14 +2819,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1379
2819
|
}
|
|
1380
2820
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1381
2821
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1382
|
-
throw new
|
|
2822
|
+
throw new ValidationError6(
|
|
1383
2823
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1384
2824
|
);
|
|
1385
2825
|
}
|
|
1386
2826
|
if (mayBeEmptyIn.length > 0) {
|
|
1387
2827
|
const first = mayBeEmptyIn[0];
|
|
1388
2828
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1389
|
-
throw new
|
|
2829
|
+
throw new ValidationError6(
|
|
1390
2830
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1391
2831
|
);
|
|
1392
2832
|
}
|
|
@@ -1396,13 +2836,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1396
2836
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1397
2837
|
outputsIn.forEach((raw, i) => {
|
|
1398
2838
|
if (typeof raw !== "object" || raw === null) {
|
|
1399
|
-
throw new
|
|
2839
|
+
throw new ValidationError6(`outputs[${i}] must be an object`);
|
|
1400
2840
|
}
|
|
1401
2841
|
const bad = Object.keys(raw).filter(
|
|
1402
2842
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1403
2843
|
);
|
|
1404
2844
|
if (bad.length > 0) {
|
|
1405
|
-
throw new
|
|
2845
|
+
throw new ValidationError6(
|
|
1406
2846
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1407
2847
|
);
|
|
1408
2848
|
}
|
|
@@ -1410,12 +2850,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1410
2850
|
const dest = spec.dest;
|
|
1411
2851
|
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1412
2852
|
if (operandKeys.has(dest.key())) {
|
|
1413
|
-
throw new
|
|
2853
|
+
throw new ValidationError6(
|
|
1414
2854
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1415
2855
|
);
|
|
1416
2856
|
}
|
|
1417
2857
|
if (destKeys.has(dest.key())) {
|
|
1418
|
-
throw new
|
|
2858
|
+
throw new ValidationError6(
|
|
1419
2859
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1420
2860
|
);
|
|
1421
2861
|
}
|
|
@@ -1488,13 +2928,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1488
2928
|
held.set(key, { at, handle });
|
|
1489
2929
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1490
2930
|
if (!samePin(prior.at, at)) {
|
|
1491
|
-
throw new
|
|
2931
|
+
throw new ValidationError6(
|
|
1492
2932
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1493
2933
|
);
|
|
1494
2934
|
}
|
|
1495
2935
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1496
2936
|
if (!pinOption) {
|
|
1497
|
-
throw new
|
|
2937
|
+
throw new ValidationError6(
|
|
1498
2938
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1499
2939
|
);
|
|
1500
2940
|
}
|
|
@@ -1522,7 +2962,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1522
2962
|
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1523
2963
|
return runCombineMany(final, {
|
|
1524
2964
|
source,
|
|
1525
|
-
codec:
|
|
2965
|
+
codec: flagshipCodec(),
|
|
1526
2966
|
clock: this.clock,
|
|
1527
2967
|
...metricsOn ? { metrics: this.metrics } : {}
|
|
1528
2968
|
});
|
|
@@ -1582,7 +3022,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1582
3022
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1583
3023
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1584
3024
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1585
|
-
throw new
|
|
3025
|
+
throw new ValidationError6(
|
|
1586
3026
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1587
3027
|
);
|
|
1588
3028
|
}
|
|
@@ -1632,13 +3072,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1632
3072
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1633
3073
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1634
3074
|
*
|
|
3075
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
3076
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
3077
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
3078
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
3079
|
+
*
|
|
1635
3080
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1636
3081
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1637
3082
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1638
3083
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1639
3084
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
3085
|
+
*
|
|
3086
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
3087
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
3088
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
3089
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
3090
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
3091
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
3092
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
3093
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
3094
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
3095
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
3096
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
3097
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
3098
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1640
3099
|
*/
|
|
1641
|
-
async subjectReport(id, options
|
|
3100
|
+
async subjectReport(id, options) {
|
|
1642
3101
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1643
3102
|
options ??= {};
|
|
1644
3103
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1653,26 +3112,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1653
3112
|
budget,
|
|
1654
3113
|
"subjectReport"
|
|
1655
3114
|
);
|
|
3115
|
+
let opens = 0;
|
|
3116
|
+
const charge = () => {
|
|
3117
|
+
opens += 1;
|
|
3118
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
3119
|
+
throw new BudgetExceededError(
|
|
3120
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
3121
|
+
{
|
|
3122
|
+
option: "budget.maxRequests",
|
|
3123
|
+
limit: budget.maxRequests,
|
|
3124
|
+
spent: budget.maxRequests + 1
|
|
3125
|
+
}
|
|
3126
|
+
);
|
|
3127
|
+
}
|
|
3128
|
+
};
|
|
1656
3129
|
const membership = await mapWithConcurrency(
|
|
1657
3130
|
recs,
|
|
1658
3131
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1659
3132
|
async (rec) => {
|
|
1660
|
-
if (rec.status === "destroyed") return null;
|
|
1661
3133
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
3134
|
+
if (rec.status === "destroyed") {
|
|
3135
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
3136
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
3137
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
3138
|
+
return held === null ? null : { tombstone: true, ref };
|
|
3139
|
+
}
|
|
1662
3140
|
if (this.crbmSource !== void 0) {
|
|
1663
3141
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1664
|
-
const
|
|
1665
|
-
if (rowVersionOfLive(held) !==
|
|
3142
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
3143
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1666
3144
|
}
|
|
1667
|
-
return await this.engine.has(ref, id) ? {
|
|
3145
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1668
3146
|
}
|
|
1669
3147
|
);
|
|
1670
|
-
const
|
|
1671
|
-
|
|
3148
|
+
const listed = membership.filter((m) => m !== null);
|
|
3149
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
3150
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
3151
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1672
3152
|
}
|
|
1673
3153
|
/**
|
|
1674
3154
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1675
|
-
* from the bucket on return
|
|
3155
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
3156
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
3157
|
+
*
|
|
3158
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
3159
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
3160
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
3161
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1676
3162
|
*
|
|
1677
3163
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1678
3164
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1737,15 +3223,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1737
3223
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1738
3224
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1739
3225
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1740
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1741
|
-
*
|
|
3226
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
3227
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
3228
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
3229
|
+
* that exhausts the budget is
|
|
1742
3230
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1743
3231
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1744
3232
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1745
3233
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1746
3234
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1747
3235
|
*/
|
|
1748
|
-
async eraseSubject(id, options
|
|
3236
|
+
async eraseSubject(id, options) {
|
|
1749
3237
|
this.erasureEpoch++;
|
|
1750
3238
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1751
3239
|
options ??= {};
|
|
@@ -1801,7 +3289,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1801
3289
|
segment: rec.segment,
|
|
1802
3290
|
namespace: rec.namespace,
|
|
1803
3291
|
erased: false,
|
|
1804
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
3292
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
3293
|
+
// the class lets a note be matched to the guide's table
|
|
1805
3294
|
};
|
|
1806
3295
|
} finally {
|
|
1807
3296
|
try {
|
|
@@ -1822,11 +3311,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1822
3311
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1823
3312
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1824
3313
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
3314
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1825
3315
|
*
|
|
1826
3316
|
* Needs a backend.
|
|
1827
3317
|
*/
|
|
1828
3318
|
async generations(ref) {
|
|
1829
|
-
validateRef(ref);
|
|
3319
|
+
ref = validateRef(ref, "generations");
|
|
1830
3320
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1831
3321
|
}
|
|
1832
3322
|
/**
|
|
@@ -1856,7 +3346,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1856
3346
|
* ```
|
|
1857
3347
|
*/
|
|
1858
3348
|
async exists(ref) {
|
|
1859
|
-
validateRef(ref);
|
|
3349
|
+
ref = validateRef(ref, "exists");
|
|
1860
3350
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1861
3351
|
}
|
|
1862
3352
|
/**
|
|
@@ -1913,8 +3403,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1913
3403
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1914
3404
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1915
3405
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1916
|
-
* crypto-shredded
|
|
1917
|
-
*
|
|
3406
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
3407
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
3408
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1918
3409
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1919
3410
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1920
3411
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1925,8 +3416,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1925
3416
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1926
3417
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1927
3418
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1928
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1929
|
-
*
|
|
3419
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
3420
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
3421
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
3422
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1930
3423
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1931
3424
|
*
|
|
1932
3425
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1956,7 +3449,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1956
3449
|
* Needs a backend.
|
|
1957
3450
|
*/
|
|
1958
3451
|
async rollback(ref, toGeneration, options = {}) {
|
|
1959
|
-
validateRef(ref);
|
|
3452
|
+
ref = validateRef(ref, "rollback");
|
|
1960
3453
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1961
3454
|
options ??= {};
|
|
1962
3455
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1972,9 +3465,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1972
3465
|
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
1973
3466
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1974
3467
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1975
|
-
* containers with no per-id work, and gives the generation chunk for chunk what its ids would.
|
|
1976
|
-
* `
|
|
1977
|
-
* `
|
|
3468
|
+
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. An
|
|
3469
|
+
* iterable with a `serialize` method (a `RoaringBitmap32`, another copy's bitmap, or a bitmap `deserializePortable`
|
|
3470
|
+
* returned) passed as ids loads as `{ bitmap }` and is not iterated. A `Uint8Array` or `Buffer` passed as ids is
|
|
3471
|
+
* refused with `ValidationError`, since each byte would load as an id, even when it has a `serialize` of its own:
|
|
3472
|
+
* pass bytes as `{ serialized }`.
|
|
3473
|
+
*
|
|
3474
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
3475
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
3476
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
3477
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1978
3478
|
*
|
|
1979
3479
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1980
3480
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -2033,7 +3533,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2033
3533
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
2034
3534
|
* what the pass deleted by name, and one name may have been gone already.
|
|
2035
3535
|
*
|
|
2036
|
-
*
|
|
3536
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
3537
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
3538
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
3539
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
3540
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
3541
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
3542
|
+
*
|
|
3543
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
2037
3544
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
2038
3545
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
2039
3546
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2056,8 +3563,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2056
3563
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2057
3564
|
*/
|
|
2058
3565
|
async load(ref, input, options = {}) {
|
|
2059
|
-
validateRef(ref);
|
|
2060
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
3566
|
+
ref = validateRef(ref, "load");
|
|
3567
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
3568
|
+
if (typeof input === "object" && input !== null) {
|
|
3569
|
+
const reads = STREAM_SOURCES.get(input);
|
|
3570
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
3571
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
3572
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
3573
|
+
throw new ValidationError6(
|
|
3574
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
3575
|
+
);
|
|
3576
|
+
}
|
|
3577
|
+
}
|
|
2061
3578
|
options ??= {};
|
|
2062
3579
|
const deps = this.lifecycleDeps("load");
|
|
2063
3580
|
try {
|
|
@@ -2067,11 +3584,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2067
3584
|
}
|
|
2068
3585
|
}
|
|
2069
3586
|
/**
|
|
2070
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
3587
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
3588
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2071
3589
|
*
|
|
2072
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2073
|
-
*
|
|
2074
|
-
*
|
|
3590
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
3591
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
3592
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
3593
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
3594
|
+
* encrypted segment it does both.
|
|
2075
3595
|
*
|
|
2076
3596
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2077
3597
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2112,9 +3632,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2112
3632
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2113
3633
|
*/
|
|
2114
3634
|
async dropSegment(ref, options) {
|
|
2115
|
-
validateRef(ref);
|
|
3635
|
+
ref = validateRef(ref, "dropSegment");
|
|
2116
3636
|
if (options === void 0 || options === null) {
|
|
2117
|
-
throw new
|
|
3637
|
+
throw new ValidationError6("dropSegment: options { confirmSegment } are required");
|
|
2118
3638
|
}
|
|
2119
3639
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2120
3640
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2129,8 +3649,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2129
3649
|
}
|
|
2130
3650
|
}
|
|
2131
3651
|
/**
|
|
2132
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2133
|
-
*
|
|
3652
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
3653
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
3654
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2134
3655
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2135
3656
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2136
3657
|
*
|
|
@@ -2152,10 +3673,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2152
3673
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2153
3674
|
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2154
3675
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2155
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
3676
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2156
3677
|
*/
|
|
2157
3678
|
async setRetention(ref, policy) {
|
|
2158
|
-
validateRef(ref);
|
|
3679
|
+
ref = validateRef(ref, "setRetention");
|
|
2159
3680
|
return setSegmentRetention(
|
|
2160
3681
|
ref,
|
|
2161
3682
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2169,16 +3690,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2169
3690
|
* "never expires" on a segment someone believes is expiring.
|
|
2170
3691
|
*/
|
|
2171
3692
|
async getRetention(ref) {
|
|
2172
|
-
validateRef(ref);
|
|
3693
|
+
ref = validateRef(ref, "getRetention");
|
|
2173
3694
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2174
3695
|
}
|
|
2175
3696
|
/**
|
|
2176
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2177
|
-
* when there was none
|
|
3697
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
3698
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
3699
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
3700
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2178
3701
|
* to the setter is how a typo becomes a deletion.
|
|
2179
3702
|
*/
|
|
2180
3703
|
async clearRetention(ref) {
|
|
2181
|
-
validateRef(ref);
|
|
3704
|
+
ref = validateRef(ref, "clearRetention");
|
|
2182
3705
|
return clearSegmentRetention(ref, {
|
|
2183
3706
|
registry: this.requireRegistry("clearRetention"),
|
|
2184
3707
|
clock: this.clock,
|
|
@@ -2203,20 +3726,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2203
3726
|
*
|
|
2204
3727
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2205
3728
|
* for (const e of swept.entries) {
|
|
2206
|
-
*
|
|
3729
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
3730
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2207
3731
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2208
3732
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2209
3733
|
* }
|
|
2210
3734
|
* }
|
|
2211
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
3735
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2212
3736
|
* ```
|
|
2213
3737
|
*
|
|
2214
3738
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2215
3739
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2216
3740
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2217
|
-
* `limit` (default 100) cut the pass short and more
|
|
2218
|
-
*
|
|
2219
|
-
*
|
|
3741
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
3742
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
3743
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
3744
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2220
3745
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2221
3746
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2222
3747
|
*
|
|
@@ -2226,9 +3751,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2226
3751
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2227
3752
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2228
3753
|
*
|
|
2229
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
3754
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2230
3755
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2231
|
-
* first, since nothing else
|
|
3756
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
3757
|
+
* only
|
|
3758
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2232
3759
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2233
3760
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2234
3761
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2249,7 +3776,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2249
3776
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2250
3777
|
const result = await retireExpired(
|
|
2251
3778
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2252
|
-
{ ...opts, now: opts.now
|
|
3779
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2253
3780
|
);
|
|
2254
3781
|
if (opts.dryRun !== true) {
|
|
2255
3782
|
for (const entry of result.entries) {
|
|
@@ -2296,7 +3823,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2296
3823
|
* ```
|
|
2297
3824
|
*/
|
|
2298
3825
|
invalidate(ref) {
|
|
2299
|
-
validateRef(ref);
|
|
3826
|
+
ref = validateRef(ref, "invalidate");
|
|
2300
3827
|
refuseReservedNamespace(ref.namespace);
|
|
2301
3828
|
this.engine.invalidate(ref);
|
|
2302
3829
|
}
|
|
@@ -2356,11 +3883,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2356
3883
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2357
3884
|
throw err;
|
|
2358
3885
|
}
|
|
2359
|
-
const pinnedAt = {
|
|
2360
|
-
generation: at?.generation ?? null,
|
|
2361
|
-
version: at?.version ?? null,
|
|
2362
|
-
fingerprint: at?.fingerprint ?? null
|
|
2363
|
-
};
|
|
3886
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2364
3887
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2365
3888
|
return makeSegment({
|
|
2366
3889
|
engine: this.engineWithPins(pins),
|
|
@@ -2400,7 +3923,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2400
3923
|
return new SegmentEngine({
|
|
2401
3924
|
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource2(pinned, this.retryOptions),
|
|
2402
3925
|
cache: this.cache,
|
|
2403
|
-
codec:
|
|
3926
|
+
codec: flagshipCodec(),
|
|
2404
3927
|
clock: this.clock,
|
|
2405
3928
|
metrics: this.metrics,
|
|
2406
3929
|
budget: this.budget,
|
|
@@ -2430,8 +3953,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2430
3953
|
if (held.has(key)) {
|
|
2431
3954
|
const before = held.get(key);
|
|
2432
3955
|
if (!samePin(before, at)) {
|
|
2433
|
-
throw new
|
|
2434
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
3956
|
+
throw new ValidationError6(
|
|
3957
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2435
3958
|
);
|
|
2436
3959
|
}
|
|
2437
3960
|
} else {
|
|
@@ -2462,8 +3985,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2462
3985
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2463
3986
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2464
3987
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2465
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2466
|
-
*
|
|
3988
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
3989
|
+
* dropped) are skipped. Pair
|
|
3990
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2467
3991
|
*
|
|
2468
3992
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2469
3993
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2496,6 +4020,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2496
4020
|
options
|
|
2497
4021
|
);
|
|
2498
4022
|
}
|
|
4023
|
+
/**
|
|
4024
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
4025
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
4026
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
4027
|
+
*
|
|
4028
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
4029
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
4030
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
4031
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
4032
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
4033
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
4034
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
4035
|
+
*
|
|
4036
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
4037
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
4038
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
4039
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
4040
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
4041
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
4042
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
4043
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
4044
|
+
* timing.
|
|
4045
|
+
*
|
|
4046
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
4047
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
4048
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
4049
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
4050
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
4051
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
4052
|
+
*
|
|
4053
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
4054
|
+
*/
|
|
4055
|
+
async preflight() {
|
|
4056
|
+
const deps = this.lifecycleDeps("preflight");
|
|
4057
|
+
const backend = this.backend;
|
|
4058
|
+
let checkLifecycle;
|
|
4059
|
+
try {
|
|
4060
|
+
const method = backend?.checkLifecycle;
|
|
4061
|
+
if (typeof method === "function")
|
|
4062
|
+
checkLifecycle = method.bind(backend);
|
|
4063
|
+
} catch {
|
|
4064
|
+
}
|
|
4065
|
+
return runPreflight({
|
|
4066
|
+
storage: deps.storage,
|
|
4067
|
+
registry: deps.registry,
|
|
4068
|
+
clock: this.clock,
|
|
4069
|
+
entropy: this.entropy,
|
|
4070
|
+
checkLifecycle
|
|
4071
|
+
});
|
|
4072
|
+
}
|
|
2499
4073
|
/**
|
|
2500
4074
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2501
4075
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2504,15 +4078,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2504
4078
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2505
4079
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2506
4080
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2507
|
-
*
|
|
2508
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
4081
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
4082
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
4083
|
+
* check `failed`. The `export-segments`
|
|
2509
4084
|
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2510
4085
|
*/
|
|
2511
4086
|
async exportSegments(sink, options = {}) {
|
|
2512
4087
|
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments, [], MISPLACED.exportSegments);
|
|
2513
4088
|
const opts = picked(options, CALL_KEYS.exportSegments);
|
|
2514
4089
|
const registry = this.requireRegistry("exportSegments");
|
|
2515
|
-
return runExport(this, registry, sink, { ...opts, codec:
|
|
4090
|
+
return runExport(this, registry, sink, { ...opts, codec: flagshipCodec() });
|
|
2516
4091
|
}
|
|
2517
4092
|
};
|
|
2518
4093
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
@@ -2540,12 +4115,23 @@ var readOptions = (options) => ({
|
|
|
2540
4115
|
});
|
|
2541
4116
|
var NO_SEGMENTS = [];
|
|
2542
4117
|
function segmentList(list, what) {
|
|
2543
|
-
if (!Array.isArray(list)) throw new
|
|
4118
|
+
if (!Array.isArray(list)) throw new ValidationError6(`${what} must be an array of segments`);
|
|
2544
4119
|
return list;
|
|
2545
4120
|
}
|
|
4121
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
4122
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
4123
|
+
function readsFrom(stream, origin, keys) {
|
|
4124
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
4125
|
+
return stream;
|
|
4126
|
+
}
|
|
2546
4127
|
function operandsLabel(verb) {
|
|
2547
4128
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2548
4129
|
}
|
|
4130
|
+
function refuseNoOperands(verb) {
|
|
4131
|
+
throw new ValidationError6(
|
|
4132
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
4133
|
+
);
|
|
4134
|
+
}
|
|
2549
4135
|
var AS_IDS = {
|
|
2550
4136
|
failing,
|
|
2551
4137
|
guard: guardIds,
|
|
@@ -2562,9 +4148,9 @@ var CombineChunks = class {
|
|
|
2562
4148
|
var AS_CHUNKS = {
|
|
2563
4149
|
guard: guardItems,
|
|
2564
4150
|
failing: (err) => refusing(err)(),
|
|
2565
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2566
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2567
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
4151
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
4152
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
4153
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2568
4154
|
};
|
|
2569
4155
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2570
4156
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2623,7 +4209,7 @@ var Segment = class _Segment {
|
|
|
2623
4209
|
*/
|
|
2624
4210
|
constructor(parts) {
|
|
2625
4211
|
if (!minting) {
|
|
2626
|
-
throw new
|
|
4212
|
+
throw new ValidationError6(
|
|
2627
4213
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2628
4214
|
);
|
|
2629
4215
|
}
|
|
@@ -2662,7 +4248,7 @@ var Segment = class _Segment {
|
|
|
2662
4248
|
*
|
|
2663
4249
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2664
4250
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2665
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
4251
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2666
4252
|
*
|
|
2667
4253
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2668
4254
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2737,11 +4323,11 @@ var Segment = class _Segment {
|
|
|
2737
4323
|
leaseUntilOf(op, options) {
|
|
2738
4324
|
if (options === void 0 || options === null) return void 0;
|
|
2739
4325
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2740
|
-
throw new
|
|
4326
|
+
throw new ValidationError6(`${op}: options must be an object such as { leaseUntil }`);
|
|
2741
4327
|
}
|
|
2742
4328
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2743
4329
|
if (unknown.length > 0) {
|
|
2744
|
-
throw new
|
|
4330
|
+
throw new ValidationError6(
|
|
2745
4331
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2746
4332
|
);
|
|
2747
4333
|
}
|
|
@@ -2749,17 +4335,17 @@ var Segment = class _Segment {
|
|
|
2749
4335
|
if (until === void 0) return void 0;
|
|
2750
4336
|
const now = this.clock.now();
|
|
2751
4337
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2752
|
-
throw new
|
|
4338
|
+
throw new ValidationError6(
|
|
2753
4339
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2754
4340
|
);
|
|
2755
4341
|
}
|
|
2756
4342
|
if (until <= now) {
|
|
2757
|
-
throw new
|
|
4343
|
+
throw new ValidationError6(
|
|
2758
4344
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2759
4345
|
);
|
|
2760
4346
|
}
|
|
2761
4347
|
if (until > now + MAX_LEASE_MS) {
|
|
2762
|
-
throw new
|
|
4348
|
+
throw new ValidationError6(
|
|
2763
4349
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2764
4350
|
);
|
|
2765
4351
|
}
|
|
@@ -2837,10 +4423,10 @@ var Segment = class _Segment {
|
|
|
2837
4423
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2838
4424
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2839
4425
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2840
|
-
* than `generation` and `
|
|
4426
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2841
4427
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2842
4428
|
*
|
|
2843
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
4429
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2844
4430
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2845
4431
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2846
4432
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2853,18 +4439,32 @@ var Segment = class _Segment {
|
|
|
2853
4439
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2854
4440
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2855
4441
|
* is in `options`, the second.
|
|
4442
|
+
*
|
|
4443
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
4444
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
4445
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2856
4446
|
*/
|
|
2857
4447
|
async pinAt(at, options) {
|
|
2858
4448
|
this.assertLease();
|
|
2859
|
-
const { generation, fingerprint } = at ?? {};
|
|
2860
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
4449
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
4450
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2861
4451
|
if (unknown.length > 0) {
|
|
2862
|
-
throw new
|
|
2863
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
4452
|
+
throw new ValidationError6(
|
|
4453
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
4454
|
+
);
|
|
4455
|
+
}
|
|
4456
|
+
if (generation === null) {
|
|
4457
|
+
throw new ValidationError6(
|
|
4458
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
4459
|
+
);
|
|
4460
|
+
}
|
|
4461
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
4462
|
+
throw new ValidationError6(
|
|
4463
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2864
4464
|
);
|
|
2865
4465
|
}
|
|
2866
4466
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2867
|
-
throw new
|
|
4467
|
+
throw new ValidationError6(
|
|
2868
4468
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2869
4469
|
);
|
|
2870
4470
|
}
|
|
@@ -2881,9 +4481,11 @@ var Segment = class _Segment {
|
|
|
2881
4481
|
return segmentKey(this.ref);
|
|
2882
4482
|
}
|
|
2883
4483
|
/**
|
|
2884
|
-
* Refuse an `*Into`
|
|
2885
|
-
* segment of another store
|
|
2886
|
-
*
|
|
4484
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
4485
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
4486
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
4487
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
4488
|
+
* first. Called by the three `*Into` verbs only.
|
|
2887
4489
|
*
|
|
2888
4490
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2889
4491
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
@@ -2893,8 +4495,28 @@ var Segment = class _Segment {
|
|
|
2893
4495
|
this.assertSegments(verb, "the destination", [dest]);
|
|
2894
4496
|
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
2895
4497
|
this.assertSegments(verb, "an exclude", excludes);
|
|
4498
|
+
const base = verb.slice(0, -"Into".length);
|
|
4499
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
4500
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
2896
4501
|
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
2897
4502
|
}
|
|
4503
|
+
/**
|
|
4504
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
4505
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
4506
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
4507
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
4508
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
4509
|
+
*/
|
|
4510
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
4511
|
+
const key = dest.key();
|
|
4512
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
4513
|
+
if (role === void 0) return;
|
|
4514
|
+
const { namespace, segment } = dest.ref;
|
|
4515
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
4516
|
+
throw new ValidationError6(
|
|
4517
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
4518
|
+
);
|
|
4519
|
+
}
|
|
2898
4520
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2899
4521
|
guarded(out, handles, result) {
|
|
2900
4522
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
@@ -3006,7 +4628,11 @@ var Segment = class _Segment {
|
|
|
3006
4628
|
engine.iterate(ref, bounds),
|
|
3007
4629
|
() => engine.iterateBatches(ref, bounds)
|
|
3008
4630
|
);
|
|
3009
|
-
return
|
|
4631
|
+
return readsFrom(
|
|
4632
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
4633
|
+
this.origin,
|
|
4634
|
+
[this.key()]
|
|
4635
|
+
);
|
|
3010
4636
|
}
|
|
3011
4637
|
/**
|
|
3012
4638
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -3055,7 +4681,11 @@ var Segment = class _Segment {
|
|
|
3055
4681
|
n,
|
|
3056
4682
|
options == null ? void 0 : rangeOf(options)
|
|
3057
4683
|
);
|
|
3058
|
-
return
|
|
4684
|
+
return readsFrom(
|
|
4685
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
4686
|
+
this.origin,
|
|
4687
|
+
[this.key()]
|
|
4688
|
+
);
|
|
3059
4689
|
}
|
|
3060
4690
|
/**
|
|
3061
4691
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3094,7 +4724,8 @@ var Segment = class _Segment {
|
|
|
3094
4724
|
*
|
|
3095
4725
|
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3096
4726
|
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3097
|
-
* first read, before any request.
|
|
4727
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
4728
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3098
4729
|
*/
|
|
3099
4730
|
intersect(others, options) {
|
|
3100
4731
|
const bad = optionsError(
|
|
@@ -3126,6 +4757,15 @@ var Segment = class _Segment {
|
|
|
3126
4757
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3127
4758
|
*
|
|
3128
4759
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
4760
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
4761
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
4762
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
4763
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
4764
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
4765
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
4766
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
4767
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
4768
|
+
* once, at one instant.
|
|
3129
4769
|
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3130
4770
|
*/
|
|
3131
4771
|
async intersectInto(dest, others, options) {
|
|
@@ -3140,7 +4780,8 @@ var Segment = class _Segment {
|
|
|
3140
4780
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3141
4781
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3142
4782
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3143
|
-
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes
|
|
4783
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
4784
|
+
* empty `others` reads `this` itself.
|
|
3144
4785
|
*/
|
|
3145
4786
|
union(others, options) {
|
|
3146
4787
|
const bad = optionsError(
|
|
@@ -3208,6 +4849,7 @@ var Segment = class _Segment {
|
|
|
3208
4849
|
try {
|
|
3209
4850
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3210
4851
|
const others = segmentList(operands, operandsLabel(verb));
|
|
4852
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3211
4853
|
handles = [this, ...others, ...exclude];
|
|
3212
4854
|
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3213
4855
|
this.assertSegments(verb, "an exclude", exclude);
|
|
@@ -3221,17 +4863,21 @@ var Segment = class _Segment {
|
|
|
3221
4863
|
} catch (err) {
|
|
3222
4864
|
return out.failing(err);
|
|
3223
4865
|
}
|
|
4866
|
+
const keys = handles.map((h) => h.key());
|
|
4867
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3224
4868
|
if (verb === "andNot") {
|
|
3225
4869
|
const refs2 = operands.map((o) => o.ref);
|
|
3226
4870
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3227
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
4871
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3228
4872
|
}
|
|
3229
4873
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3230
4874
|
const opts = this.refsIn(options, exclude);
|
|
3231
|
-
return
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
4875
|
+
return known(
|
|
4876
|
+
this.guarded(
|
|
4877
|
+
out,
|
|
4878
|
+
handles,
|
|
4879
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
4880
|
+
)
|
|
3235
4881
|
);
|
|
3236
4882
|
}
|
|
3237
4883
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
@@ -3285,22 +4931,37 @@ ${USAGE}`,
|
|
|
3285
4931
|
code: 2
|
|
3286
4932
|
};
|
|
3287
4933
|
}
|
|
4934
|
+
var EXPORT_VARIABLES = [
|
|
4935
|
+
"CR_EXPORT_ROOT",
|
|
4936
|
+
"CR_EXPORT_OUT",
|
|
4937
|
+
"CR_EXPORT_FORMAT",
|
|
4938
|
+
"CR_EXPORT_NAMESPACE"
|
|
4939
|
+
];
|
|
3288
4940
|
function parseConfig(env) {
|
|
4941
|
+
const unknown = Object.keys(env).filter(
|
|
4942
|
+
(name) => name.startsWith("CR_EXPORT_") && !EXPORT_VARIABLES.includes(name)
|
|
4943
|
+
);
|
|
4944
|
+
if (unknown.length > 0) {
|
|
4945
|
+
const which = unknown.length > 1 ? `${unknown.join(", ")} are not settings export-segments takes.` : `${unknown[0]} is not a setting export-segments takes.`;
|
|
4946
|
+
throw new ValidationError7(
|
|
4947
|
+
`${which} It takes ${EXPORT_VARIABLES.join(", ")}; run export-segments --help`
|
|
4948
|
+
);
|
|
4949
|
+
}
|
|
3289
4950
|
const root = env.CR_EXPORT_ROOT;
|
|
3290
4951
|
if (root === void 0 || root === "") {
|
|
3291
|
-
throw new
|
|
4952
|
+
throw new ValidationError7(
|
|
3292
4953
|
"CR_EXPORT_ROOT is not set: the directory that holds the store's storage/ and registry/. export-segments also needs CR_EXPORT_OUT; run export-segments --help"
|
|
3293
4954
|
);
|
|
3294
4955
|
}
|
|
3295
4956
|
const out = env.CR_EXPORT_OUT;
|
|
3296
4957
|
if (out === void 0 || out === "") {
|
|
3297
|
-
throw new
|
|
4958
|
+
throw new ValidationError7(
|
|
3298
4959
|
"CR_EXPORT_OUT is not set: the directory to write the dump to. export-segments also needs CR_EXPORT_ROOT; run export-segments --help"
|
|
3299
4960
|
);
|
|
3300
4961
|
}
|
|
3301
4962
|
const format = env.CR_EXPORT_FORMAT || "roaring";
|
|
3302
4963
|
if (format !== "roaring" && format !== "ndjson") {
|
|
3303
|
-
throw new
|
|
4964
|
+
throw new ValidationError7(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
|
|
3304
4965
|
}
|
|
3305
4966
|
const namespace = env.CR_EXPORT_NAMESPACE || void 0;
|
|
3306
4967
|
return { root, out, format, namespace };
|
|
@@ -3315,7 +4976,7 @@ async function ensureOwnDirectory(out, dir) {
|
|
|
3315
4976
|
}
|
|
3316
4977
|
const found = await lstat(dir);
|
|
3317
4978
|
const refuse = (why) => {
|
|
3318
|
-
throw new
|
|
4979
|
+
throw new ValidationError7(`the export directory is not one this tool made: ${why}`);
|
|
3319
4980
|
};
|
|
3320
4981
|
if (found.isSymbolicLink()) refuse("a namespace directory is a symbolic link");
|
|
3321
4982
|
if (!found.isDirectory()) refuse("a namespace path is not a directory");
|
|
@@ -3330,7 +4991,7 @@ function fsSink(out) {
|
|
|
3330
4991
|
async open(ref, ext) {
|
|
3331
4992
|
const part = namespacePathPart(ref.namespace);
|
|
3332
4993
|
if (part.toLowerCase() === MANIFEST_NAME) {
|
|
3333
|
-
throw new
|
|
4994
|
+
throw new ValidationError7(
|
|
3334
4995
|
`a namespace named ${MANIFEST_NAME} (in any letter case) cannot be exported to a directory: its folder would be the manifest's path`
|
|
3335
4996
|
);
|
|
3336
4997
|
}
|
|
@@ -3375,10 +5036,10 @@ async function requireStoreRoot(root) {
|
|
|
3375
5036
|
try {
|
|
3376
5037
|
entries = await readdir(root);
|
|
3377
5038
|
} catch {
|
|
3378
|
-
throw new
|
|
5039
|
+
throw new ValidationError7("CR_EXPORT_ROOT does not name a directory that can be read");
|
|
3379
5040
|
}
|
|
3380
5041
|
if (entries.length === 0) {
|
|
3381
|
-
throw new
|
|
5042
|
+
throw new ValidationError7(
|
|
3382
5043
|
"CR_EXPORT_ROOT is empty: it holds no store (an unmounted volume, or the wrong directory)"
|
|
3383
5044
|
);
|
|
3384
5045
|
}
|
|
@@ -3394,7 +5055,7 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3394
5055
|
const store = new CloudRoaring({ storage });
|
|
3395
5056
|
const priorManifest = await lstat(join(config.out, MANIFEST_NAME)).catch(() => null);
|
|
3396
5057
|
if (priorManifest?.isDirectory() === true) {
|
|
3397
|
-
throw new
|
|
5058
|
+
throw new ValidationError7(
|
|
3398
5059
|
`${join(config.out, MANIFEST_NAME)} is a directory, so the manifest cannot be written there: remove or rename it, or export to a fresh directory`
|
|
3399
5060
|
);
|
|
3400
5061
|
}
|