@cloudbitmaps/roaring 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +2 -2
- package/PRIVACY.md +24 -18
- package/README.md +28 -11
- package/dist/bin/export-segments.js +1980 -319
- package/dist/bin/export-segments.js.map +4 -4
- package/dist/index.d.ts +263 -61
- package/dist/index.js +1996 -315
- package/dist/index.js.map +4 -4
- package/dist/roaring-codec.d.ts +36 -105
- package/package.json +4 -5
package/dist/index.js
CHANGED
|
@@ -11,17 +11,20 @@ import {
|
|
|
11
11
|
RetryingStorageChunkSource as RetryingStorageChunkSource2,
|
|
12
12
|
SegmentEngine,
|
|
13
13
|
LEASE_SKEW_MS,
|
|
14
|
+
BudgetExceededError,
|
|
14
15
|
LeaseExpiredError,
|
|
15
16
|
MAX_LEASE_MS,
|
|
16
17
|
NotFoundError,
|
|
17
18
|
UnsupportedError as UnsupportedError2,
|
|
18
|
-
ValidationError as
|
|
19
|
+
ValidationError as ValidationError6,
|
|
20
|
+
errorKind,
|
|
19
21
|
WriteConflictError,
|
|
20
22
|
checkBudget,
|
|
21
23
|
collectWithinBudget,
|
|
22
24
|
excludingReservedRows,
|
|
23
25
|
dropSegment,
|
|
24
26
|
eraseIdFromSegment,
|
|
27
|
+
findIdUnderTombstone,
|
|
25
28
|
judgeLoad,
|
|
26
29
|
loadSegment,
|
|
27
30
|
loadSegmentChunks,
|
|
@@ -42,7 +45,12 @@ import {
|
|
|
42
45
|
splitId,
|
|
43
46
|
takeLease
|
|
44
47
|
} from "@cloudbitmaps/core";
|
|
45
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
pointerIdOf,
|
|
50
|
+
requireBool,
|
|
51
|
+
requirePositiveInt,
|
|
52
|
+
validateSegmentRef
|
|
53
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
46
54
|
|
|
47
55
|
// src/open-charging-storage.ts
|
|
48
56
|
var OpenChargingStorage = class {
|
|
@@ -58,6 +66,7 @@ var OpenChargingStorage = class {
|
|
|
58
66
|
}
|
|
59
67
|
inner;
|
|
60
68
|
charge;
|
|
69
|
+
/** Generations an open of costs nothing more: the scan's current one, this call's writes, and each one charged. */
|
|
61
70
|
free = /* @__PURE__ */ new Set();
|
|
62
71
|
capabilities() {
|
|
63
72
|
return this.inner.capabilities();
|
|
@@ -70,9 +79,15 @@ var OpenChargingStorage = class {
|
|
|
70
79
|
getRange(key, offset, length) {
|
|
71
80
|
return this.inner.getRange(key, offset, length);
|
|
72
81
|
}
|
|
73
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* One `getTail` is one open: a reader takes the footer and the index from it, and reads chunks with `getRange`. A
|
|
84
|
+
* generation is charged at its first open only.
|
|
85
|
+
*/
|
|
74
86
|
async getTail(key, maxBytes) {
|
|
75
|
-
if (!this.free.has(key.generation))
|
|
87
|
+
if (!this.free.has(key.generation)) {
|
|
88
|
+
this.charge();
|
|
89
|
+
this.free.add(key.generation);
|
|
90
|
+
}
|
|
76
91
|
return this.inner.getTail(key, maxBytes);
|
|
77
92
|
}
|
|
78
93
|
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
@@ -86,7 +101,12 @@ var OpenChargingStorage = class {
|
|
|
86
101
|
|
|
87
102
|
// src/index.ts
|
|
88
103
|
import { listGenerations, rollbackSegment } from "@cloudbitmaps/core";
|
|
89
|
-
import {
|
|
104
|
+
import {
|
|
105
|
+
destroySegment as coreDestroySegment,
|
|
106
|
+
eraseNamespace as coreEraseNamespace
|
|
107
|
+
} from "@cloudbitmaps/core";
|
|
108
|
+
import { listSegments, refuseUnknownRefKeys, segmentExists } from "@cloudbitmaps/core";
|
|
109
|
+
import { runPreflight } from "@cloudbitmaps/core";
|
|
90
110
|
|
|
91
111
|
// src/reserved-namespace.ts
|
|
92
112
|
import { ValidationError } from "@cloudbitmaps/core";
|
|
@@ -100,8 +120,10 @@ function refuseReservedNamespace(namespace) {
|
|
|
100
120
|
}
|
|
101
121
|
|
|
102
122
|
// src/roaring-codec.ts
|
|
103
|
-
import
|
|
104
|
-
|
|
123
|
+
import { ValidationError as ValidationError3, decodeSerialized } from "@cloudbitmaps/core";
|
|
124
|
+
|
|
125
|
+
// src/ts-roaring/bitmap.ts
|
|
126
|
+
import { IntegrityError as IntegrityError3, ValidationError as ValidationError2 } from "@cloudbitmaps/core";
|
|
105
127
|
|
|
106
128
|
// src/portable/layout.ts
|
|
107
129
|
import { IntegrityError } from "@cloudbitmaps/core";
|
|
@@ -119,45 +141,10 @@ function overrun(bytes, offset, length, what) {
|
|
|
119
141
|
`portable roaring: ${what} needs bytes [${offset}, ${offset + length}) of a ${bytes.byteLength}-byte buffer`
|
|
120
142
|
);
|
|
121
143
|
}
|
|
122
|
-
function
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
var SINGLE_RUN_HEADER_BYTES = 9;
|
|
127
|
-
function* containerPayloads(bytes) {
|
|
128
|
-
if (bytes.byteLength === 0) return;
|
|
129
|
-
const { view, count, runFlagsAt, descriptiveAt, bodiesAt } = header(bytes);
|
|
130
|
-
let pos = bodiesAt;
|
|
131
|
-
let previousKey = -1;
|
|
132
|
-
for (let i = 0; i < count; i++) {
|
|
133
|
-
const key = view.getUint16(descriptiveAt + i * 4, true);
|
|
134
|
-
if (key <= previousKey) {
|
|
135
|
-
throw new IntegrityError(
|
|
136
|
-
`portable roaring: container keys are not strictly ascending (${previousKey} then ${key} at index ${i})`
|
|
137
|
-
);
|
|
138
|
-
}
|
|
139
|
-
previousKey = key;
|
|
140
|
-
const cardinality = view.getUint16(descriptiveAt + i * 4 + 2, true) + 1;
|
|
141
|
-
const isRun = runFlagsAt >= 0 && (bytes[runFlagsAt + (i >>> 3)] & 1 << (i & 7)) !== 0;
|
|
142
|
-
const kind = isRun ? RUN : cardinality > ARRAY_MAX_CARDINALITY ? BITMAP : ARRAY;
|
|
143
|
-
const size = checkContainer(bytes, view, pos, kind, cardinality, i);
|
|
144
|
-
const headerBytes = isRun ? SINGLE_RUN_HEADER_BYTES : SINGLE_HEADER_BYTES;
|
|
145
|
-
const payload = new Uint8Array(headerBytes + size);
|
|
146
|
-
const out = new DataView(payload.buffer);
|
|
147
|
-
if (isRun) {
|
|
148
|
-
out.setUint32(0, SERIAL_COOKIE, true);
|
|
149
|
-
payload[4] = 1;
|
|
150
|
-
out.setUint16(7, cardinality - 1, true);
|
|
151
|
-
} else {
|
|
152
|
-
out.setUint32(0, SERIAL_COOKIE_NO_RUNCONTAINER, true);
|
|
153
|
-
out.setUint32(4, 1, true);
|
|
154
|
-
out.setUint16(10, cardinality - 1, true);
|
|
155
|
-
out.setUint32(12, SINGLE_HEADER_BYTES, true);
|
|
156
|
-
}
|
|
157
|
-
payload.set(bytes.subarray(pos, pos + size), headerBytes);
|
|
158
|
-
pos += size;
|
|
159
|
-
yield { chunkKey: key, payload, cardinality };
|
|
160
|
-
}
|
|
144
|
+
function parsePortableLayout(bytes) {
|
|
145
|
+
const containers = [];
|
|
146
|
+
walk(bytes, containers);
|
|
147
|
+
return containers;
|
|
161
148
|
}
|
|
162
149
|
function header(bytes) {
|
|
163
150
|
const length = bytes.byteLength;
|
|
@@ -308,165 +295,1606 @@ function popcount(view, at) {
|
|
|
308
295
|
return bits;
|
|
309
296
|
}
|
|
310
297
|
|
|
311
|
-
// src/roaring
|
|
312
|
-
var
|
|
313
|
-
var
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
298
|
+
// src/ts-roaring/containers.ts
|
|
299
|
+
var ARRAY2 = 0;
|
|
300
|
+
var BITSET = 1;
|
|
301
|
+
var RUN2 = 2;
|
|
302
|
+
var WORDS = 2048;
|
|
303
|
+
var BITSET_BYTES = 8192;
|
|
304
|
+
var GALLOP_SKEW = 16;
|
|
305
|
+
var LITTLE_ENDIAN = new Uint8Array(Uint16Array.of(1).buffer)[0] === 1;
|
|
306
|
+
var Container = class {
|
|
307
|
+
constructor(kind, data, off, len, card, shared) {
|
|
308
|
+
this.kind = kind;
|
|
309
|
+
this.data = data;
|
|
310
|
+
this.off = off;
|
|
311
|
+
this.len = len;
|
|
312
|
+
this.card = card;
|
|
313
|
+
this.shared = shared;
|
|
314
|
+
}
|
|
315
|
+
kind;
|
|
316
|
+
data;
|
|
317
|
+
off;
|
|
318
|
+
len;
|
|
319
|
+
card;
|
|
320
|
+
shared;
|
|
321
|
+
};
|
|
322
|
+
function lows16(c) {
|
|
323
|
+
return c.data;
|
|
324
|
+
}
|
|
325
|
+
function words32(c) {
|
|
326
|
+
return c.data;
|
|
327
|
+
}
|
|
328
|
+
function arrayContainer(values, card) {
|
|
329
|
+
return new Container(ARRAY2, values, 0, card, card, false);
|
|
330
|
+
}
|
|
331
|
+
function bitsetContainer(words, card) {
|
|
332
|
+
return new Container(BITSET, words, 0, 0, card, false);
|
|
333
|
+
}
|
|
334
|
+
function runContainer(pairs, runs, card) {
|
|
335
|
+
return new Container(RUN2, pairs, 0, runs, card, false);
|
|
336
|
+
}
|
|
337
|
+
var valueScratch = new Uint16Array(8192);
|
|
338
|
+
var pairScratch = new Uint16Array(8192);
|
|
339
|
+
var SCRATCH_KEEP = 65536;
|
|
340
|
+
function values16(n) {
|
|
341
|
+
if (valueScratch.length >= n) return valueScratch;
|
|
342
|
+
if (n > SCRATCH_KEEP) return new Uint16Array(n);
|
|
343
|
+
valueScratch = new Uint16Array(Math.max(n, Math.min(2 * valueScratch.length, SCRATCH_KEEP)));
|
|
344
|
+
return valueScratch;
|
|
345
|
+
}
|
|
346
|
+
function pairs16(n) {
|
|
347
|
+
if (pairScratch.length >= n) return pairScratch;
|
|
348
|
+
if (n > SCRATCH_KEEP) return new Uint16Array(n);
|
|
349
|
+
pairScratch = new Uint16Array(Math.max(n, Math.min(2 * pairScratch.length, SCRATCH_KEEP)));
|
|
350
|
+
return pairScratch;
|
|
351
|
+
}
|
|
352
|
+
function arrayOfScratch(scratch, n) {
|
|
353
|
+
return n === 0 ? null : arrayContainer(scratch.slice(0, n), n);
|
|
354
|
+
}
|
|
355
|
+
function popcount32(word) {
|
|
356
|
+
let w = word - (word >>> 1 & 1431655765);
|
|
357
|
+
w = (w & 858993459) + (w >>> 2 & 858993459);
|
|
358
|
+
return Math.imul(w + (w >>> 4) & 252645135, 16843009) >>> 24;
|
|
359
|
+
}
|
|
360
|
+
function popcountWords(d) {
|
|
361
|
+
let total = 0;
|
|
362
|
+
let ones = 0;
|
|
363
|
+
let twos = 0;
|
|
364
|
+
let fours = 0;
|
|
365
|
+
let eights = 0;
|
|
366
|
+
for (let i = 0; i < WORDS; i += 16) {
|
|
367
|
+
let u;
|
|
368
|
+
let twosA;
|
|
369
|
+
let twosB;
|
|
370
|
+
let foursA;
|
|
371
|
+
let foursB;
|
|
372
|
+
let x = d[i];
|
|
373
|
+
let y = d[i + 1];
|
|
374
|
+
u = ones ^ x;
|
|
375
|
+
twosA = ones & x | u & y;
|
|
376
|
+
ones = u ^ y;
|
|
377
|
+
x = d[i + 2];
|
|
378
|
+
y = d[i + 3];
|
|
379
|
+
u = ones ^ x;
|
|
380
|
+
twosB = ones & x | u & y;
|
|
381
|
+
ones = u ^ y;
|
|
382
|
+
u = twos ^ twosA;
|
|
383
|
+
foursA = twos & twosA | u & twosB;
|
|
384
|
+
twos = u ^ twosB;
|
|
385
|
+
x = d[i + 4];
|
|
386
|
+
y = d[i + 5];
|
|
387
|
+
u = ones ^ x;
|
|
388
|
+
twosA = ones & x | u & y;
|
|
389
|
+
ones = u ^ y;
|
|
390
|
+
x = d[i + 6];
|
|
391
|
+
y = d[i + 7];
|
|
392
|
+
u = ones ^ x;
|
|
393
|
+
twosB = ones & x | u & y;
|
|
394
|
+
ones = u ^ y;
|
|
395
|
+
u = twos ^ twosA;
|
|
396
|
+
foursB = twos & twosA | u & twosB;
|
|
397
|
+
twos = u ^ twosB;
|
|
398
|
+
u = fours ^ foursA;
|
|
399
|
+
const eightsA = fours & foursA | u & foursB;
|
|
400
|
+
fours = u ^ foursB;
|
|
401
|
+
x = d[i + 8];
|
|
402
|
+
y = d[i + 9];
|
|
403
|
+
u = ones ^ x;
|
|
404
|
+
twosA = ones & x | u & y;
|
|
405
|
+
ones = u ^ y;
|
|
406
|
+
x = d[i + 10];
|
|
407
|
+
y = d[i + 11];
|
|
408
|
+
u = ones ^ x;
|
|
409
|
+
twosB = ones & x | u & y;
|
|
410
|
+
ones = u ^ y;
|
|
411
|
+
u = twos ^ twosA;
|
|
412
|
+
foursA = twos & twosA | u & twosB;
|
|
413
|
+
twos = u ^ twosB;
|
|
414
|
+
x = d[i + 12];
|
|
415
|
+
y = d[i + 13];
|
|
416
|
+
u = ones ^ x;
|
|
417
|
+
twosA = ones & x | u & y;
|
|
418
|
+
ones = u ^ y;
|
|
419
|
+
x = d[i + 14];
|
|
420
|
+
y = d[i + 15];
|
|
421
|
+
u = ones ^ x;
|
|
422
|
+
twosB = ones & x | u & y;
|
|
423
|
+
ones = u ^ y;
|
|
424
|
+
u = twos ^ twosA;
|
|
425
|
+
foursB = twos & twosA | u & twosB;
|
|
426
|
+
twos = u ^ twosB;
|
|
427
|
+
u = fours ^ foursA;
|
|
428
|
+
const eightsB = fours & foursA | u & foursB;
|
|
429
|
+
fours = u ^ foursB;
|
|
430
|
+
u = eights ^ eightsA;
|
|
431
|
+
const sixteens = eights & eightsA | u & eightsB;
|
|
432
|
+
eights = u ^ eightsB;
|
|
433
|
+
total += popcount32(sixteens);
|
|
434
|
+
}
|
|
435
|
+
return 16 * total + 8 * popcount32(eights) + 4 * popcount32(fours) + 2 * popcount32(twos) + popcount32(ones);
|
|
436
|
+
}
|
|
437
|
+
function setRange(words, start, end) {
|
|
438
|
+
const first = start >>> 5;
|
|
439
|
+
const last = end >>> 5;
|
|
440
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
441
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
442
|
+
if (first === last) {
|
|
443
|
+
words[first] = words[first] | head & tail;
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
words[first] = words[first] | head;
|
|
447
|
+
for (let i = first + 1; i < last; i++) words[i] = 4294967295;
|
|
448
|
+
words[last] = words[last] | tail;
|
|
449
|
+
}
|
|
450
|
+
function clearRange(words, start, end) {
|
|
451
|
+
const first = start >>> 5;
|
|
452
|
+
const last = end >>> 5;
|
|
453
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
454
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
455
|
+
if (first === last) {
|
|
456
|
+
words[first] = words[first] & ~(head & tail);
|
|
457
|
+
return;
|
|
458
|
+
}
|
|
459
|
+
words[first] = words[first] & ~head;
|
|
460
|
+
for (let i = first + 1; i < last; i++) words[i] = 0;
|
|
461
|
+
words[last] = words[last] & ~tail;
|
|
462
|
+
}
|
|
463
|
+
function fromSortedIds(ids, from, to) {
|
|
464
|
+
const n = to - from;
|
|
465
|
+
if (n <= ARRAY_MAX_CARDINALITY) {
|
|
466
|
+
const lows = new Uint16Array(n);
|
|
467
|
+
let count = 0;
|
|
468
|
+
let previous = -1;
|
|
469
|
+
for (let i = from; i < to; i++) {
|
|
470
|
+
const low = ids[i] & 65535;
|
|
471
|
+
if (low !== previous) lows[count++] = low;
|
|
472
|
+
previous = low;
|
|
473
|
+
}
|
|
474
|
+
return arrayContainer(count === n ? lows : lows.slice(0, count), count);
|
|
475
|
+
}
|
|
476
|
+
const words = new Uint32Array(WORDS);
|
|
477
|
+
for (let i = from; i < to; i++) {
|
|
478
|
+
const v = ids[i] & 65535;
|
|
479
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
480
|
+
}
|
|
481
|
+
return fromWordsCard(words, popcountWords(words));
|
|
482
|
+
}
|
|
483
|
+
function fromWordsCard(words, card) {
|
|
484
|
+
if (card === 0) return null;
|
|
485
|
+
if (card > ARRAY_MAX_CARDINALITY) return bitsetContainer(words, card);
|
|
486
|
+
const out = new Uint16Array(card);
|
|
487
|
+
wordsToLows(words, out);
|
|
488
|
+
return arrayContainer(out, card);
|
|
489
|
+
}
|
|
490
|
+
function wordsToLows(words, out) {
|
|
491
|
+
let k = 0;
|
|
492
|
+
for (let i = 0; i < WORDS; i++) {
|
|
493
|
+
let w = words[i];
|
|
494
|
+
const base = i * 32 + 31;
|
|
495
|
+
while (w !== 0) {
|
|
496
|
+
const lowest = w & -w;
|
|
497
|
+
out[k++] = base - Math.clz32(lowest);
|
|
498
|
+
w ^= lowest;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
return k;
|
|
502
|
+
}
|
|
503
|
+
function writeLows(c, out) {
|
|
504
|
+
if (c.kind === ARRAY2) {
|
|
505
|
+
out.set(lows16(c).subarray(c.off, c.off + c.len));
|
|
506
|
+
} else if (c.kind === BITSET) {
|
|
507
|
+
wordsToLows(words32(c), out);
|
|
508
|
+
} else {
|
|
509
|
+
const u16 = lows16(c);
|
|
510
|
+
let k = 0;
|
|
511
|
+
for (let r = 0; r < c.len; r++) {
|
|
512
|
+
const start = u16[c.off + 2 * r];
|
|
513
|
+
const end = start + u16[c.off + 2 * r + 1];
|
|
514
|
+
for (let v = start; v <= end; v++) out[k++] = v;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
function appendIds(c, out, at, base) {
|
|
519
|
+
if (c.kind === ARRAY2) {
|
|
520
|
+
const u162 = lows16(c);
|
|
521
|
+
const end = c.off + c.len;
|
|
522
|
+
if (base === 0) {
|
|
523
|
+
out.set(u162.subarray(c.off, end), at);
|
|
524
|
+
return at + c.len;
|
|
525
|
+
}
|
|
526
|
+
for (let i = c.off; i < end; i++) out[at++] = base + u162[i];
|
|
527
|
+
return at;
|
|
528
|
+
}
|
|
529
|
+
if (c.kind === BITSET) {
|
|
530
|
+
const words = words32(c);
|
|
531
|
+
for (let i = 0; i < WORDS; i++) {
|
|
532
|
+
let w = words[i];
|
|
533
|
+
const top = base + i * 32 + 31;
|
|
534
|
+
while (w !== 0) {
|
|
535
|
+
const lowest = w & -w;
|
|
536
|
+
out[at++] = top - Math.clz32(lowest);
|
|
537
|
+
w ^= lowest;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
return at;
|
|
541
|
+
}
|
|
542
|
+
const u16 = lows16(c);
|
|
543
|
+
for (let r = 0; r < c.len; r++) {
|
|
544
|
+
const start = u16[c.off + 2 * r];
|
|
545
|
+
const last = start + u16[c.off + 2 * r + 1];
|
|
546
|
+
if (last - start < LONG_RUN) {
|
|
547
|
+
for (let v = base + start; v <= base + last; v++) out[at++] = v;
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
const from = at;
|
|
551
|
+
out.set(iota().subarray(start, last + 1), at);
|
|
552
|
+
at += last - start + 1;
|
|
553
|
+
if (base !== 0) for (let i = from; i < at; i++) out[i] = out[i] + base;
|
|
554
|
+
}
|
|
555
|
+
return at;
|
|
556
|
+
}
|
|
557
|
+
var LONG_RUN = 48;
|
|
558
|
+
var iotaTable;
|
|
559
|
+
function iota() {
|
|
560
|
+
if (iotaTable === void 0) {
|
|
561
|
+
iotaTable = new Uint16Array(65536);
|
|
562
|
+
for (let i = 0; i < 65536; i++) iotaTable[i] = i;
|
|
563
|
+
}
|
|
564
|
+
return iotaTable;
|
|
565
|
+
}
|
|
566
|
+
function toWords(c) {
|
|
567
|
+
if (c.kind === BITSET) return words32(c);
|
|
568
|
+
const words = new Uint32Array(WORDS);
|
|
569
|
+
const u16 = lows16(c);
|
|
570
|
+
if (c.kind === ARRAY2) {
|
|
571
|
+
for (let i = c.off; i < c.off + c.len; i++) {
|
|
572
|
+
const v = u16[i];
|
|
573
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
574
|
+
}
|
|
575
|
+
} else {
|
|
576
|
+
for (let r = 0; r < c.len; r++) {
|
|
577
|
+
const start = u16[c.off + 2 * r];
|
|
578
|
+
setRange(words, start, start + u16[c.off + 2 * r + 1]);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
return words;
|
|
582
|
+
}
|
|
583
|
+
function demote(c) {
|
|
584
|
+
if (c.kind !== RUN2) return c;
|
|
585
|
+
if (c.card > ARRAY_MAX_CARDINALITY) return bitsetContainer(toWords(c), c.card);
|
|
586
|
+
const out = new Uint16Array(c.card);
|
|
587
|
+
writeLows(c, out);
|
|
588
|
+
return arrayContainer(out, c.card);
|
|
589
|
+
}
|
|
590
|
+
function cloneContainer(c) {
|
|
591
|
+
if (c.kind === BITSET) return bitsetContainer(words32(c).slice(), c.card);
|
|
592
|
+
const copy = lows16(c).slice(c.off, c.off + (c.kind === ARRAY2 ? c.len : 2 * c.len));
|
|
593
|
+
return new Container(c.kind, copy, 0, c.len, c.card, false);
|
|
594
|
+
}
|
|
595
|
+
function writableWords(c) {
|
|
596
|
+
return c.shared ? words32(c).slice() : words32(c);
|
|
597
|
+
}
|
|
598
|
+
function lowerBound(data, off, len, low, stride) {
|
|
599
|
+
let lo = 0;
|
|
600
|
+
let hi = len;
|
|
601
|
+
while (lo < hi) {
|
|
602
|
+
const mid = lo + hi >>> 1;
|
|
603
|
+
if (data[off + mid * stride] < low) lo = mid + 1;
|
|
604
|
+
else hi = mid;
|
|
605
|
+
}
|
|
606
|
+
return lo;
|
|
607
|
+
}
|
|
608
|
+
function gallop(data, off, from, end, target) {
|
|
609
|
+
let step = 1;
|
|
610
|
+
let lo = from;
|
|
611
|
+
let hi = from;
|
|
612
|
+
while (hi < end && data[off + hi] < target) {
|
|
613
|
+
lo = hi + 1;
|
|
614
|
+
hi += step;
|
|
615
|
+
step *= 2;
|
|
616
|
+
}
|
|
617
|
+
if (hi > end) hi = end;
|
|
618
|
+
while (lo < hi) {
|
|
619
|
+
const mid = lo + hi >>> 1;
|
|
620
|
+
if (data[off + mid] < target) lo = mid + 1;
|
|
621
|
+
else hi = mid;
|
|
622
|
+
}
|
|
623
|
+
return lo;
|
|
624
|
+
}
|
|
625
|
+
function containerHas(c, low) {
|
|
626
|
+
if (c.kind === BITSET)
|
|
627
|
+
return (c.data[low >>> 5] >>> (low & 31) & 1) === 1;
|
|
628
|
+
const u16 = c.data;
|
|
629
|
+
if (c.kind === ARRAY2) {
|
|
630
|
+
const at = lowerBound(u16, c.off, c.len, low, 1);
|
|
631
|
+
return at < c.len && u16[c.off + at] === low;
|
|
632
|
+
}
|
|
633
|
+
const after = lowerBound(u16, c.off, c.len, low + 1, 2);
|
|
634
|
+
if (after === 0) return false;
|
|
635
|
+
const r = c.off + 2 * (after - 1);
|
|
636
|
+
return low <= u16[r] + u16[r + 1];
|
|
637
|
+
}
|
|
638
|
+
function containerAdd(c, low) {
|
|
639
|
+
if (containerHas(c, low)) return c;
|
|
640
|
+
const plain = demote(c);
|
|
641
|
+
if (plain.kind === BITSET) {
|
|
642
|
+
const words = writableWords(plain);
|
|
643
|
+
words[low >>> 5] = words[low >>> 5] | 1 << (low & 31);
|
|
644
|
+
return bitsetContainer(words, plain.card + 1);
|
|
645
|
+
}
|
|
646
|
+
if (plain.card === ARRAY_MAX_CARDINALITY) {
|
|
647
|
+
const words = toWords(plain);
|
|
648
|
+
words[low >>> 5] = words[low >>> 5] | 1 << (low & 31);
|
|
649
|
+
return bitsetContainer(words, plain.card + 1);
|
|
650
|
+
}
|
|
651
|
+
const old = lows16(plain);
|
|
652
|
+
const at = lowerBound(old, plain.off, plain.len, low, 1);
|
|
653
|
+
let u16 = old;
|
|
654
|
+
if (!plain.shared && plain.off === 0 && old.length > plain.len) {
|
|
655
|
+
u16.copyWithin(at + 1, at, plain.len);
|
|
656
|
+
} else {
|
|
657
|
+
u16 = new Uint16Array(Math.min(ARRAY_MAX_CARDINALITY, Math.max(8, plain.len * 2)));
|
|
658
|
+
u16.set(old.subarray(plain.off, plain.off + at));
|
|
659
|
+
u16.set(old.subarray(plain.off + at, plain.off + plain.len), at + 1);
|
|
660
|
+
}
|
|
661
|
+
u16[at] = low;
|
|
662
|
+
return arrayContainer(u16, plain.card + 1);
|
|
663
|
+
}
|
|
664
|
+
function containerRemove(c, low) {
|
|
665
|
+
if (!containerHas(c, low)) return c;
|
|
666
|
+
if (c.card === 1) return null;
|
|
667
|
+
const plain = demote(c);
|
|
668
|
+
if (plain.kind === BITSET) {
|
|
669
|
+
const words = writableWords(plain);
|
|
670
|
+
words[low >>> 5] = words[low >>> 5] & ~(1 << (low & 31));
|
|
671
|
+
return fromWordsCard(words, plain.card - 1);
|
|
672
|
+
}
|
|
673
|
+
const old = lows16(plain);
|
|
674
|
+
const at = lowerBound(old, plain.off, plain.len, low, 1);
|
|
675
|
+
if (!plain.shared && plain.off === 0) {
|
|
676
|
+
old.copyWithin(at, at + 1, plain.len);
|
|
677
|
+
return arrayContainer(old, plain.card - 1);
|
|
678
|
+
}
|
|
679
|
+
const u16 = new Uint16Array(plain.len - 1);
|
|
680
|
+
u16.set(old.subarray(plain.off, plain.off + at));
|
|
681
|
+
u16.set(old.subarray(plain.off + at + 1, plain.off + plain.len), at);
|
|
682
|
+
return arrayContainer(u16, plain.card - 1);
|
|
683
|
+
}
|
|
684
|
+
function containerMax(c) {
|
|
685
|
+
if (c.kind === ARRAY2) return c.data[c.off + c.len - 1];
|
|
686
|
+
if (c.kind === RUN2) {
|
|
687
|
+
const u16 = c.data;
|
|
688
|
+
return u16[c.off + 2 * c.len - 2] + u16[c.off + 2 * c.len - 1];
|
|
689
|
+
}
|
|
690
|
+
const words = words32(c);
|
|
691
|
+
for (let i = WORDS - 1; i >= 0; i--) {
|
|
692
|
+
const w = words[i];
|
|
693
|
+
if (w !== 0) return i * 32 + 31 - Math.clz32(w);
|
|
694
|
+
}
|
|
695
|
+
throw new RangeError("a bitset container holds no bits");
|
|
696
|
+
}
|
|
697
|
+
function runCount(c, stop) {
|
|
698
|
+
if (c.kind === RUN2) return c.len;
|
|
699
|
+
let runs = 0;
|
|
700
|
+
if (c.kind === ARRAY2) {
|
|
701
|
+
const u16 = lows16(c);
|
|
702
|
+
let previous = -2;
|
|
703
|
+
for (let i = c.off; i < c.off + c.len; i++) {
|
|
704
|
+
const v = u16[i];
|
|
705
|
+
if (v !== previous + 1 && ++runs >= stop) return runs;
|
|
706
|
+
previous = v;
|
|
707
|
+
}
|
|
708
|
+
return runs;
|
|
709
|
+
}
|
|
710
|
+
const words = words32(c);
|
|
711
|
+
let carry = 0;
|
|
712
|
+
for (let i = 0; i < WORDS; i++) {
|
|
713
|
+
const w = words[i];
|
|
714
|
+
const starts = w & ~(w << 1 | carry);
|
|
715
|
+
if (starts !== 0) {
|
|
716
|
+
runs += popcount32(starts >>> 0);
|
|
717
|
+
if (runs >= stop) return runs;
|
|
718
|
+
}
|
|
719
|
+
carry = w >>> 31;
|
|
720
|
+
}
|
|
721
|
+
return runs;
|
|
722
|
+
}
|
|
723
|
+
function toRun(c, runs) {
|
|
724
|
+
const pairs = new Uint16Array(2 * runs);
|
|
725
|
+
let r = 0;
|
|
726
|
+
if (c.kind === ARRAY2) {
|
|
727
|
+
const u16 = lows16(c);
|
|
728
|
+
const end = c.off + c.len;
|
|
729
|
+
let i = c.off;
|
|
730
|
+
while (i < end) {
|
|
731
|
+
let j = i;
|
|
732
|
+
while (j + 1 < end && u16[j + 1] === u16[j] + 1) j++;
|
|
733
|
+
pairs[2 * r] = u16[i];
|
|
734
|
+
pairs[2 * r + 1] = j - i;
|
|
735
|
+
r++;
|
|
736
|
+
i = j + 1;
|
|
737
|
+
}
|
|
738
|
+
return runContainer(pairs, runs, c.card);
|
|
739
|
+
}
|
|
740
|
+
const words = words32(c);
|
|
741
|
+
let start = 0;
|
|
742
|
+
let carry = 0;
|
|
743
|
+
for (let i = 0; i < WORDS; i++) {
|
|
744
|
+
const w = words[i];
|
|
745
|
+
const next = i + 1 < WORDS ? words[i + 1] : 0;
|
|
746
|
+
let starts = (w & ~(w << 1 | carry)) >>> 0;
|
|
747
|
+
let ends = (w & ~(w >>> 1 | next << 31)) >>> 0;
|
|
748
|
+
carry = w >>> 31;
|
|
749
|
+
let both = starts | ends;
|
|
750
|
+
while (both !== 0) {
|
|
751
|
+
const bit = both & -both;
|
|
752
|
+
both ^= bit;
|
|
753
|
+
const at = i * 32 + 31 - Math.clz32(bit);
|
|
754
|
+
if ((starts & bit) !== 0) {
|
|
755
|
+
starts ^= bit;
|
|
756
|
+
start = at;
|
|
757
|
+
}
|
|
758
|
+
if ((ends & bit) !== 0) {
|
|
759
|
+
ends ^= bit;
|
|
760
|
+
pairs[2 * r] = start;
|
|
761
|
+
pairs[2 * r + 1] = at - start;
|
|
762
|
+
r++;
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
return runContainer(pairs, runs, c.card);
|
|
767
|
+
}
|
|
768
|
+
function optimize(c) {
|
|
769
|
+
const plain = c.card <= ARRAY_MAX_CARDINALITY ? 2 * c.card : BITSET_BYTES;
|
|
770
|
+
const runs = runCount(c, (plain - 2) / 4);
|
|
771
|
+
if (2 + 4 * runs < plain) return c.kind === RUN2 ? c : toRun(c, runs);
|
|
772
|
+
return demote(c);
|
|
773
|
+
}
|
|
774
|
+
function skewed(shorter, longer) {
|
|
775
|
+
return longer >= GALLOP_SKEW * shorter;
|
|
776
|
+
}
|
|
777
|
+
var PROBE_MIN = 32;
|
|
778
|
+
var probeWords = new Uint32Array(WORDS);
|
|
779
|
+
function andArrays(a, b) {
|
|
780
|
+
const small = a.len <= b.len ? a : b;
|
|
781
|
+
const big = small === a ? b : a;
|
|
782
|
+
const s = lows16(small);
|
|
783
|
+
const l = lows16(big);
|
|
784
|
+
const out = values16(small.len);
|
|
785
|
+
let n = 0;
|
|
786
|
+
if (small.len >= PROBE_MIN && !skewed(small.len, big.len)) {
|
|
787
|
+
const se = small.off + small.len;
|
|
788
|
+
const first = s[small.off];
|
|
789
|
+
const last = s[se - 1];
|
|
790
|
+
const le = big.off + big.len;
|
|
791
|
+
if (last < l[big.off] || first > l[le - 1]) return null;
|
|
792
|
+
const from = gallop(l, 0, big.off, le, first);
|
|
793
|
+
const to = gallop(l, 0, from, le, last + 1);
|
|
794
|
+
const words = probeWords;
|
|
795
|
+
for (let i2 = small.off; i2 < se; i2++) {
|
|
796
|
+
const v = s[i2];
|
|
797
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
798
|
+
}
|
|
799
|
+
for (let j2 = from; j2 < to; j2++) {
|
|
800
|
+
const v = l[j2];
|
|
801
|
+
out[n] = v;
|
|
802
|
+
n += words[v >>> 5] >>> (v & 31) & 1;
|
|
803
|
+
}
|
|
804
|
+
for (let i2 = small.off; i2 < se; i2++) words[s[i2] >>> 5] = 0;
|
|
805
|
+
return arrayOfScratch(out, n);
|
|
806
|
+
}
|
|
807
|
+
if (skewed(small.len, big.len)) {
|
|
808
|
+
let at = 0;
|
|
809
|
+
const end = big.off + big.len;
|
|
810
|
+
for (let i2 = small.off; i2 < small.off + small.len; i2++) {
|
|
811
|
+
const v = s[i2];
|
|
812
|
+
at = gallop(l, 0, Math.max(at, big.off), end, v);
|
|
813
|
+
if (at === end) break;
|
|
814
|
+
if (l[at] === v) out[n++] = v;
|
|
815
|
+
}
|
|
816
|
+
return arrayOfScratch(out, n);
|
|
817
|
+
}
|
|
818
|
+
let i = small.off;
|
|
819
|
+
let j = big.off;
|
|
820
|
+
const ie = small.off + small.len;
|
|
821
|
+
const je = big.off + big.len;
|
|
822
|
+
while (i < ie && j < je) {
|
|
823
|
+
const x = s[i];
|
|
824
|
+
const y = l[j];
|
|
825
|
+
out[n] = x;
|
|
826
|
+
n += x === y ? 1 : 0;
|
|
827
|
+
i += x <= y ? 1 : 0;
|
|
828
|
+
j += x >= y ? 1 : 0;
|
|
829
|
+
}
|
|
830
|
+
return arrayOfScratch(out, n);
|
|
831
|
+
}
|
|
832
|
+
function filterByBits(a, b, keep) {
|
|
833
|
+
const u16 = lows16(a);
|
|
834
|
+
const words = words32(b);
|
|
835
|
+
const out = values16(a.len);
|
|
836
|
+
const flip = keep ? 0 : 1;
|
|
837
|
+
let n = 0;
|
|
838
|
+
for (let i = a.off; i < a.off + a.len; i++) {
|
|
839
|
+
const v = u16[i];
|
|
840
|
+
out[n] = v;
|
|
841
|
+
n += words[v >>> 5] >>> (v & 31) & 1 ^ flip;
|
|
842
|
+
}
|
|
843
|
+
return arrayOfScratch(out, n);
|
|
844
|
+
}
|
|
845
|
+
function filterByRuns(a, b, keep) {
|
|
846
|
+
const u16 = lows16(a);
|
|
847
|
+
const pairs = lows16(b);
|
|
848
|
+
const out = values16(a.len);
|
|
849
|
+
let n = 0;
|
|
850
|
+
if (keep && a.len >= b.len) {
|
|
851
|
+
const ae = a.off + a.len;
|
|
852
|
+
let i = a.off;
|
|
853
|
+
for (let r2 = 0; r2 < b.len && i < ae; r2++) {
|
|
854
|
+
const start = pairs[b.off + 2 * r2];
|
|
855
|
+
const end = start + pairs[b.off + 2 * r2 + 1];
|
|
856
|
+
while (i < ae && u16[i] < start) i++;
|
|
857
|
+
while (i < ae && u16[i] <= end) out[n++] = u16[i++];
|
|
858
|
+
}
|
|
859
|
+
return arrayOfScratch(out, n);
|
|
860
|
+
}
|
|
861
|
+
let r = 0;
|
|
862
|
+
for (let i = a.off; i < a.off + a.len; i++) {
|
|
863
|
+
const v = u16[i];
|
|
864
|
+
while (r < b.len && pairs[b.off + 2 * r] + pairs[b.off + 2 * r + 1] < v) {
|
|
865
|
+
r++;
|
|
866
|
+
}
|
|
867
|
+
const inside = r < b.len && pairs[b.off + 2 * r] <= v;
|
|
868
|
+
if (inside === keep) out[n++] = v;
|
|
869
|
+
}
|
|
870
|
+
return arrayOfScratch(out, n);
|
|
871
|
+
}
|
|
872
|
+
function andNotArrays(a, b) {
|
|
873
|
+
const x = lows16(a);
|
|
874
|
+
const y = lows16(b);
|
|
875
|
+
const out = values16(a.len);
|
|
876
|
+
let n = 0;
|
|
877
|
+
const je = b.off + b.len;
|
|
878
|
+
if (skewed(a.len, b.len)) {
|
|
879
|
+
let at = b.off;
|
|
880
|
+
for (let i2 = a.off; i2 < a.off + a.len; i2++) {
|
|
881
|
+
const v = x[i2];
|
|
882
|
+
at = gallop(y, 0, at, je, v);
|
|
883
|
+
if (at === je || y[at] !== v) out[n++] = v;
|
|
884
|
+
}
|
|
885
|
+
return arrayOfScratch(out, n);
|
|
886
|
+
}
|
|
887
|
+
if (b.len >= PROBE_MIN) {
|
|
888
|
+
const ae = a.off + a.len;
|
|
889
|
+
const first = x[a.off];
|
|
890
|
+
const last = x[ae - 1];
|
|
891
|
+
const from = gallop(y, 0, b.off, je, first);
|
|
892
|
+
const to = gallop(y, 0, from, je, last + 1);
|
|
893
|
+
const words = probeWords;
|
|
894
|
+
for (let j2 = from; j2 < to; j2++) {
|
|
895
|
+
const v = y[j2];
|
|
896
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
897
|
+
}
|
|
898
|
+
for (let i2 = a.off; i2 < ae; i2++) {
|
|
899
|
+
const v = x[i2];
|
|
900
|
+
out[n] = v;
|
|
901
|
+
n += words[v >>> 5] >>> (v & 31) & 1 ^ 1;
|
|
902
|
+
}
|
|
903
|
+
for (let j2 = from; j2 < to; j2++) words[y[j2] >>> 5] = 0;
|
|
904
|
+
return arrayOfScratch(out, n);
|
|
905
|
+
}
|
|
906
|
+
let i = a.off;
|
|
907
|
+
let j = b.off;
|
|
908
|
+
const ie = a.off + a.len;
|
|
909
|
+
while (i < ie && j < je) {
|
|
910
|
+
const p = x[i];
|
|
911
|
+
const q = y[j];
|
|
912
|
+
out[n] = p;
|
|
913
|
+
n += p < q ? 1 : 0;
|
|
914
|
+
i += p <= q ? 1 : 0;
|
|
915
|
+
j += p >= q ? 1 : 0;
|
|
916
|
+
}
|
|
917
|
+
while (i < ie) out[n++] = x[i++];
|
|
918
|
+
return arrayOfScratch(out, n);
|
|
919
|
+
}
|
|
920
|
+
function orArrays(a, b) {
|
|
921
|
+
const x = lows16(a);
|
|
922
|
+
const y = lows16(b);
|
|
923
|
+
const ie = a.off + a.len;
|
|
924
|
+
const je = b.off + b.len;
|
|
925
|
+
if (a.len + b.len > ARRAY_MAX_CARDINALITY) {
|
|
926
|
+
const words = new Uint32Array(WORDS);
|
|
927
|
+
for (let i2 = a.off; i2 < ie; i2++) {
|
|
928
|
+
const v = x[i2];
|
|
929
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
930
|
+
}
|
|
931
|
+
for (let j2 = b.off; j2 < je; j2++) {
|
|
932
|
+
const v = y[j2];
|
|
933
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
934
|
+
}
|
|
935
|
+
return fromWordsCard(words, popcountWords(words));
|
|
936
|
+
}
|
|
937
|
+
const out = values16(a.len + b.len);
|
|
938
|
+
let n = 0;
|
|
939
|
+
let i = a.off;
|
|
940
|
+
let j = b.off;
|
|
941
|
+
while (i < ie && j < je) {
|
|
942
|
+
const p = x[i];
|
|
943
|
+
const q = y[j];
|
|
944
|
+
out[n++] = p < q ? p : q;
|
|
945
|
+
i += p <= q ? 1 : 0;
|
|
946
|
+
j += p >= q ? 1 : 0;
|
|
947
|
+
}
|
|
948
|
+
while (i < ie) out[n++] = x[i++];
|
|
949
|
+
while (j < je) out[n++] = y[j++];
|
|
950
|
+
return arrayOfScratch(out, n);
|
|
951
|
+
}
|
|
952
|
+
function orIntoBitset(dest, other, inPlace) {
|
|
953
|
+
const words = inPlace ? words32(dest) : words32(dest).slice();
|
|
954
|
+
if (other.kind === ARRAY2) {
|
|
955
|
+
const u16 = lows16(other);
|
|
956
|
+
let card = dest.card;
|
|
957
|
+
for (let i = other.off; i < other.off + other.len; i++) {
|
|
958
|
+
const v = u16[i];
|
|
959
|
+
const w = words[v >>> 5];
|
|
960
|
+
const bit = 1 << (v & 31);
|
|
961
|
+
if ((w & bit) === 0) {
|
|
962
|
+
words[v >>> 5] = w | bit;
|
|
963
|
+
card++;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return bitsetContainer(words, card);
|
|
967
|
+
}
|
|
968
|
+
if (other.kind === BITSET) {
|
|
969
|
+
const o = words32(other);
|
|
970
|
+
for (let i = 0; i < WORDS; i++) words[i] = words[i] | o[i];
|
|
971
|
+
} else {
|
|
972
|
+
const pairs = lows16(other);
|
|
973
|
+
for (let r = 0; r < other.len; r++) {
|
|
974
|
+
const start = pairs[other.off + 2 * r];
|
|
975
|
+
setRange(words, start, start + pairs[other.off + 2 * r + 1]);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
return bitsetContainer(words, popcountWords(words));
|
|
979
|
+
}
|
|
980
|
+
function fromRunPairs(pairs, runs, card) {
|
|
981
|
+
if (card === 0) return null;
|
|
982
|
+
if (card <= ARRAY_MAX_CARDINALITY) {
|
|
983
|
+
const out = new Uint16Array(card);
|
|
984
|
+
let k = 0;
|
|
985
|
+
for (let r = 0; r < runs; r++) {
|
|
986
|
+
const start = pairs[2 * r];
|
|
987
|
+
const end = start + pairs[2 * r + 1];
|
|
988
|
+
for (let v = start; v <= end; v++) out[k++] = v;
|
|
989
|
+
}
|
|
990
|
+
return arrayContainer(out, card);
|
|
991
|
+
}
|
|
992
|
+
const words = new Uint32Array(WORDS);
|
|
993
|
+
for (let r = 0; r < runs; r++) {
|
|
994
|
+
const start = pairs[2 * r];
|
|
995
|
+
setRange(words, start, start + pairs[2 * r + 1]);
|
|
996
|
+
}
|
|
997
|
+
return bitsetContainer(words, card);
|
|
998
|
+
}
|
|
999
|
+
function andRuns(a, b) {
|
|
1000
|
+
const x = lows16(a);
|
|
1001
|
+
const y = lows16(b);
|
|
1002
|
+
const out = pairs16(2 * (a.len + b.len));
|
|
1003
|
+
let runs = 0;
|
|
1004
|
+
let card = 0;
|
|
1005
|
+
let i = 0;
|
|
1006
|
+
let j = 0;
|
|
1007
|
+
while (i < a.len && j < b.len) {
|
|
1008
|
+
const as = x[a.off + 2 * i];
|
|
1009
|
+
const ae = as + x[a.off + 2 * i + 1];
|
|
1010
|
+
const bs = y[b.off + 2 * j];
|
|
1011
|
+
const be = bs + y[b.off + 2 * j + 1];
|
|
1012
|
+
const lo = as > bs ? as : bs;
|
|
1013
|
+
const hi = ae < be ? ae : be;
|
|
1014
|
+
if (lo <= hi) {
|
|
1015
|
+
out[2 * runs] = lo;
|
|
1016
|
+
out[2 * runs + 1] = hi - lo;
|
|
1017
|
+
runs++;
|
|
1018
|
+
card += hi - lo + 1;
|
|
1019
|
+
}
|
|
1020
|
+
if (ae < be) i++;
|
|
1021
|
+
else j++;
|
|
1022
|
+
}
|
|
1023
|
+
return fromRunPairs(out, runs, card);
|
|
1024
|
+
}
|
|
1025
|
+
function orRuns(a, b) {
|
|
1026
|
+
const x = lows16(a);
|
|
1027
|
+
const y = lows16(b);
|
|
1028
|
+
const out = pairs16(2 * (a.len + b.len));
|
|
1029
|
+
let runs = 0;
|
|
1030
|
+
let card = 0;
|
|
1031
|
+
let i = 0;
|
|
1032
|
+
let j = 0;
|
|
1033
|
+
let start = -1;
|
|
1034
|
+
let end = -1;
|
|
1035
|
+
const flush = () => {
|
|
1036
|
+
if (start < 0) return;
|
|
1037
|
+
out[2 * runs] = start;
|
|
1038
|
+
out[2 * runs + 1] = end - start;
|
|
1039
|
+
runs++;
|
|
1040
|
+
card += end - start + 1;
|
|
1041
|
+
};
|
|
1042
|
+
while (i < a.len || j < b.len) {
|
|
1043
|
+
let s;
|
|
1044
|
+
let e;
|
|
1045
|
+
if (j >= b.len || i < a.len && x[a.off + 2 * i] <= y[b.off + 2 * j]) {
|
|
1046
|
+
s = x[a.off + 2 * i];
|
|
1047
|
+
e = s + x[a.off + 2 * i + 1];
|
|
1048
|
+
i++;
|
|
1049
|
+
} else {
|
|
1050
|
+
s = y[b.off + 2 * j];
|
|
1051
|
+
e = s + y[b.off + 2 * j + 1];
|
|
1052
|
+
j++;
|
|
1053
|
+
}
|
|
1054
|
+
if (start >= 0 && s <= end + 1) {
|
|
1055
|
+
if (e > end) end = e;
|
|
1056
|
+
} else {
|
|
1057
|
+
flush();
|
|
1058
|
+
start = s;
|
|
1059
|
+
end = e;
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
flush();
|
|
1063
|
+
return fromRunPairs(out, runs, card);
|
|
1064
|
+
}
|
|
1065
|
+
function andNotRuns(a, b) {
|
|
1066
|
+
const x = lows16(a);
|
|
1067
|
+
const y = lows16(b);
|
|
1068
|
+
const out = pairs16(2 * (a.len + b.len + 1));
|
|
1069
|
+
let runs = 0;
|
|
1070
|
+
let card = 0;
|
|
1071
|
+
let j = 0;
|
|
1072
|
+
for (let i = 0; i < a.len; i++) {
|
|
1073
|
+
let s = x[a.off + 2 * i];
|
|
1074
|
+
const e = s + x[a.off + 2 * i + 1];
|
|
1075
|
+
while (j < b.len && y[b.off + 2 * j] + y[b.off + 2 * j + 1] < s) j++;
|
|
1076
|
+
let k = j;
|
|
1077
|
+
while (s <= e) {
|
|
1078
|
+
if (k >= b.len || y[b.off + 2 * k] > e) {
|
|
1079
|
+
out[2 * runs] = s;
|
|
1080
|
+
out[2 * runs + 1] = e - s;
|
|
1081
|
+
runs++;
|
|
1082
|
+
card += e - s + 1;
|
|
1083
|
+
break;
|
|
1084
|
+
}
|
|
1085
|
+
const bs = y[b.off + 2 * k];
|
|
1086
|
+
const be = bs + y[b.off + 2 * k + 1];
|
|
1087
|
+
if (bs > s) {
|
|
1088
|
+
out[2 * runs] = s;
|
|
1089
|
+
out[2 * runs + 1] = bs - 1 - s;
|
|
1090
|
+
runs++;
|
|
1091
|
+
card += bs - s;
|
|
1092
|
+
}
|
|
1093
|
+
s = be + 1;
|
|
1094
|
+
k++;
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
return fromRunPairs(out, runs, card);
|
|
1098
|
+
}
|
|
1099
|
+
function bitsetInRuns(a, b) {
|
|
1100
|
+
const words = words32(a);
|
|
1101
|
+
const pairs = lows16(b);
|
|
1102
|
+
if (b.card <= ARRAY_MAX_CARDINALITY) {
|
|
1103
|
+
const out2 = values16(b.card);
|
|
1104
|
+
let n = 0;
|
|
1105
|
+
for (let r = 0; r < b.len; r++) {
|
|
1106
|
+
const start = pairs[b.off + 2 * r];
|
|
1107
|
+
const end = start + pairs[b.off + 2 * r + 1];
|
|
1108
|
+
const first = start >>> 5;
|
|
1109
|
+
const last = end >>> 5;
|
|
1110
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
1111
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
1112
|
+
for (let i = first; i <= last; i++) {
|
|
1113
|
+
let w = words[i];
|
|
1114
|
+
if (i === first) w &= head;
|
|
1115
|
+
if (i === last) w &= tail;
|
|
1116
|
+
const top = i * 32 + 31;
|
|
1117
|
+
while (w !== 0) {
|
|
1118
|
+
const lowest = w & -w;
|
|
1119
|
+
out2[n++] = top - Math.clz32(lowest);
|
|
1120
|
+
w ^= lowest;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
return arrayOfScratch(out2, n);
|
|
1125
|
+
}
|
|
1126
|
+
const out = new Uint32Array(WORDS);
|
|
1127
|
+
for (let r = 0; r < b.len; r++) {
|
|
1128
|
+
const start = pairs[b.off + 2 * r];
|
|
1129
|
+
const end = start + pairs[b.off + 2 * r + 1];
|
|
1130
|
+
const first = start >>> 5;
|
|
1131
|
+
const last = end >>> 5;
|
|
1132
|
+
const head = 4294967295 << (start & 31) >>> 0;
|
|
1133
|
+
const tail = 4294967295 >>> 31 - (end & 31);
|
|
1134
|
+
if (first === last) {
|
|
1135
|
+
out[first] = out[first] | words[first] & head & tail;
|
|
1136
|
+
} else {
|
|
1137
|
+
out[first] = out[first] | words[first] & head;
|
|
1138
|
+
for (let i = first + 1; i < last; i++) out[i] = words[i];
|
|
1139
|
+
out[last] = out[last] | words[last] & tail;
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
return fromWordsCard(out, popcountWords(out));
|
|
1143
|
+
}
|
|
1144
|
+
function containerAnd(a, b, inPlace = false) {
|
|
1145
|
+
if (a.kind === BITSET && b.kind === BITSET) {
|
|
1146
|
+
const x = words32(a);
|
|
1147
|
+
const y = words32(b);
|
|
1148
|
+
const out = inPlace ? x : new Uint32Array(WORDS);
|
|
1149
|
+
for (let i = 0; i < WORDS; i++) out[i] = x[i] & y[i];
|
|
1150
|
+
return fromWordsCard(out, popcountWords(out));
|
|
1151
|
+
}
|
|
1152
|
+
if (a.kind > b.kind) return containerAnd(b, a);
|
|
1153
|
+
if (a.kind === ARRAY2) {
|
|
1154
|
+
if (b.kind === ARRAY2) return andArrays(a, b);
|
|
1155
|
+
return b.kind === BITSET ? filterByBits(a, b, true) : filterByRuns(a, b, true);
|
|
1156
|
+
}
|
|
1157
|
+
return b.kind === RUN2 && a.kind === RUN2 ? andRuns(a, b) : bitsetInRuns(a, b);
|
|
1158
|
+
}
|
|
1159
|
+
function containerOr(a, b, inPlace = false) {
|
|
1160
|
+
if (a.kind === BITSET) return orIntoBitset(a, b, inPlace);
|
|
1161
|
+
if (b.kind === BITSET) return orIntoBitset(b, a, false);
|
|
1162
|
+
if (a.kind === ARRAY2 && b.kind === ARRAY2) return orArrays(a, b);
|
|
1163
|
+
if (a.kind === RUN2 && b.kind === RUN2) return orRuns(a, b);
|
|
1164
|
+
const words = toWords(a.kind === RUN2 ? a : b);
|
|
1165
|
+
return orIntoBitsetWords(words, a.kind === RUN2 ? b : a);
|
|
1166
|
+
}
|
|
1167
|
+
function orIntoBitsetWords(words, array) {
|
|
1168
|
+
const u16 = lows16(array);
|
|
1169
|
+
for (let i = array.off; i < array.off + array.len; i++) {
|
|
1170
|
+
const v = u16[i];
|
|
1171
|
+
words[v >>> 5] = words[v >>> 5] | 1 << (v & 31);
|
|
1172
|
+
}
|
|
1173
|
+
return fromWordsCard(words, popcountWords(words));
|
|
1174
|
+
}
|
|
1175
|
+
function containerAndNot(a, b, inPlace = false) {
|
|
1176
|
+
if (a.kind === ARRAY2) {
|
|
1177
|
+
if (b.kind === ARRAY2) return andNotArrays(a, b);
|
|
1178
|
+
return b.kind === BITSET ? filterByBits(a, b, false) : filterByRuns(a, b, false);
|
|
1179
|
+
}
|
|
1180
|
+
if (a.kind === RUN2 && b.kind === RUN2) return andNotRuns(a, b);
|
|
1181
|
+
const words = a.kind === BITSET ? inPlace ? words32(a) : words32(a).slice() : toWords(a);
|
|
1182
|
+
if (b.kind === ARRAY2) {
|
|
1183
|
+
const u16 = lows16(b);
|
|
1184
|
+
for (let i = b.off; i < b.off + b.len; i++) {
|
|
1185
|
+
const v = u16[i];
|
|
1186
|
+
words[v >>> 5] = words[v >>> 5] & ~(1 << (v & 31));
|
|
1187
|
+
}
|
|
1188
|
+
} else if (b.kind === BITSET) {
|
|
1189
|
+
const y = words32(b);
|
|
1190
|
+
for (let i = 0; i < WORDS; i++) words[i] = words[i] & ~y[i];
|
|
1191
|
+
} else {
|
|
1192
|
+
const pairs = lows16(b);
|
|
1193
|
+
for (let r = 0; r < b.len; r++) {
|
|
1194
|
+
const start = pairs[b.off + 2 * r];
|
|
1195
|
+
clearRange(words, start, start + pairs[b.off + 2 * r + 1]);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
return fromWordsCard(words, popcountWords(words));
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
// src/ts-roaring/iterator.ts
|
|
1202
|
+
var DONE = { done: true, value: void 0 };
|
|
1203
|
+
var BitmapIterator = class {
|
|
1204
|
+
#keys;
|
|
1205
|
+
#conts;
|
|
1206
|
+
/** The container being walked, as an index into `#conts`, and its kind, storage and offset. */
|
|
1207
|
+
#at = -1;
|
|
1208
|
+
#kind = ARRAY2;
|
|
1209
|
+
#u16 = new Uint16Array(0);
|
|
1210
|
+
#words = new Uint32Array(0);
|
|
1211
|
+
#off = 0;
|
|
1212
|
+
/** Added to every remainder: the container's key times 65,536. */
|
|
1213
|
+
#base = 0;
|
|
1214
|
+
/** An array: the next index and one past the last. A run container: the next run's index and the run count. A bitset: the word index. */
|
|
1215
|
+
#pos = 0;
|
|
1216
|
+
#end = 0;
|
|
1217
|
+
/** A run container: the next value and the last of the current run. A bitset: what is left of the current word. */
|
|
1218
|
+
#next = 0;
|
|
1219
|
+
#last = -1;
|
|
1220
|
+
constructor(keys, conts) {
|
|
1221
|
+
this.#keys = keys;
|
|
1222
|
+
this.#conts = conts;
|
|
1223
|
+
}
|
|
1224
|
+
[Symbol.iterator]() {
|
|
1225
|
+
return this;
|
|
1226
|
+
}
|
|
1227
|
+
next() {
|
|
1228
|
+
for (; ; ) {
|
|
1229
|
+
if (this.#kind === ARRAY2) {
|
|
1230
|
+
if (this.#pos < this.#end) {
|
|
1231
|
+
return { done: false, value: this.#base + this.#u16[this.#pos++] };
|
|
1232
|
+
}
|
|
1233
|
+
} else if (this.#kind === BITSET) {
|
|
1234
|
+
let w = this.#next;
|
|
1235
|
+
while (w === 0 && ++this.#pos < WORDS) w = this.#words[this.#pos];
|
|
1236
|
+
if (w !== 0) {
|
|
1237
|
+
const lowest = w & -w;
|
|
1238
|
+
this.#next = w ^ lowest;
|
|
1239
|
+
return {
|
|
1240
|
+
done: false,
|
|
1241
|
+
value: this.#base + this.#pos * 32 + 31 - Math.clz32(lowest)
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
} else {
|
|
1245
|
+
if (this.#next <= this.#last) return { done: false, value: this.#base + this.#next++ };
|
|
1246
|
+
if (this.#pos < this.#end) {
|
|
1247
|
+
const r = this.#off + 2 * this.#pos++;
|
|
1248
|
+
this.#next = this.#u16[r];
|
|
1249
|
+
this.#last = this.#next + this.#u16[r + 1];
|
|
1250
|
+
continue;
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
if (!this.#advance()) return DONE;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
/** Move to the next container; `false` when there is none. */
|
|
1257
|
+
#advance() {
|
|
1258
|
+
if (++this.#at >= this.#conts.length) {
|
|
1259
|
+
this.#at = this.#conts.length;
|
|
1260
|
+
this.#kind = ARRAY2;
|
|
1261
|
+
this.#pos = 0;
|
|
1262
|
+
this.#end = 0;
|
|
1263
|
+
return false;
|
|
1264
|
+
}
|
|
1265
|
+
const c = this.#conts[this.#at];
|
|
1266
|
+
this.#kind = c.kind;
|
|
1267
|
+
this.#base = this.#keys[this.#at] * 65536;
|
|
1268
|
+
this.#off = c.off;
|
|
1269
|
+
if (c.kind === ARRAY2) {
|
|
1270
|
+
this.#u16 = c.data;
|
|
1271
|
+
this.#pos = c.off;
|
|
1272
|
+
this.#end = c.off + c.len;
|
|
1273
|
+
} else if (c.kind === BITSET) {
|
|
1274
|
+
this.#words = c.data;
|
|
1275
|
+
this.#pos = 0;
|
|
1276
|
+
this.#next = this.#words[0];
|
|
1277
|
+
} else {
|
|
1278
|
+
this.#u16 = c.data;
|
|
1279
|
+
this.#pos = 0;
|
|
1280
|
+
this.#end = c.len;
|
|
1281
|
+
this.#next = 0;
|
|
1282
|
+
this.#last = -1;
|
|
1283
|
+
}
|
|
1284
|
+
return true;
|
|
1285
|
+
}
|
|
1286
|
+
};
|
|
1287
|
+
|
|
1288
|
+
// src/ts-roaring/wire.ts
|
|
1289
|
+
import { IntegrityError as IntegrityError2 } from "@cloudbitmaps/core";
|
|
1290
|
+
function bodySize(c) {
|
|
1291
|
+
if (c.kind === ARRAY2) return 2 * c.len;
|
|
1292
|
+
if (c.kind === BITSET) return BITSET_BYTES;
|
|
1293
|
+
return 2 + 4 * c.len;
|
|
1294
|
+
}
|
|
1295
|
+
function putU16s(out, at, src, from, count) {
|
|
1296
|
+
if (LITTLE_ENDIAN) {
|
|
1297
|
+
out.set(new Uint8Array(src.buffer, src.byteOffset + 2 * from, 2 * count), at);
|
|
1298
|
+
return;
|
|
1299
|
+
}
|
|
1300
|
+
const view = new DataView(out.buffer, out.byteOffset, out.byteLength);
|
|
1301
|
+
for (let i = 0; i < count; i++) view.setUint16(at + 2 * i, src[from + i], true);
|
|
1302
|
+
}
|
|
1303
|
+
function writeBody(c, out, view, at) {
|
|
1304
|
+
if (c.kind === ARRAY2) {
|
|
1305
|
+
putU16s(out, at, c.data, c.off, c.len);
|
|
1306
|
+
} else if (c.kind === BITSET) {
|
|
1307
|
+
const words = c.data;
|
|
1308
|
+
if (LITTLE_ENDIAN) out.set(new Uint8Array(words.buffer, words.byteOffset, BITSET_BYTES), at);
|
|
1309
|
+
else for (let i = 0; i < WORDS; i++) view.setUint32(at + 4 * i, words[i], true);
|
|
1310
|
+
} else {
|
|
1311
|
+
view.setUint16(at, c.len, true);
|
|
1312
|
+
putU16s(out, at + 2, c.data, c.off, 2 * c.len);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
function serializeContainers(keys, conts) {
|
|
1316
|
+
const n = conts.length;
|
|
1317
|
+
let hasRun = false;
|
|
1318
|
+
for (let i = 0; i < n; i++) if (conts[i].kind === RUN2) hasRun = true;
|
|
1319
|
+
const offsets = !hasRun || n >= NO_OFFSET_THRESHOLD;
|
|
1320
|
+
let at = hasRun ? 4 + (n + 7 >>> 3) : 8;
|
|
1321
|
+
const describeAt = at;
|
|
1322
|
+
at += 4 * n;
|
|
1323
|
+
const offsetsAt = at;
|
|
1324
|
+
if (offsets) at += 4 * n;
|
|
1325
|
+
let total = at;
|
|
1326
|
+
for (let i = 0; i < n; i++) total += bodySize(conts[i]);
|
|
1327
|
+
const out = new Uint8Array(total);
|
|
1328
|
+
const view = new DataView(out.buffer);
|
|
1329
|
+
if (hasRun) {
|
|
1330
|
+
view.setUint32(0, SERIAL_COOKIE | n - 1 << 16, true);
|
|
1331
|
+
for (let i = 0; i < n; i++) {
|
|
1332
|
+
if (conts[i].kind === RUN2) {
|
|
1333
|
+
out[4 + (i >>> 3)] = out[4 + (i >>> 3)] | 1 << (i & 7);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
view.setUint32(0, SERIAL_COOKIE_NO_RUNCONTAINER, true);
|
|
1338
|
+
view.setUint32(4, n, true);
|
|
1339
|
+
}
|
|
1340
|
+
for (let i = 0; i < n; i++) {
|
|
1341
|
+
const c = conts[i];
|
|
1342
|
+
view.setUint16(describeAt + 4 * i, keys[i], true);
|
|
1343
|
+
view.setUint16(describeAt + 4 * i + 2, c.card - 1, true);
|
|
1344
|
+
if (offsets) view.setUint32(offsetsAt + 4 * i, at, true);
|
|
1345
|
+
writeBody(c, out, view, at);
|
|
1346
|
+
at += bodySize(c);
|
|
1347
|
+
}
|
|
1348
|
+
return out;
|
|
1349
|
+
}
|
|
1350
|
+
function decodeContainers(input) {
|
|
1351
|
+
const bytes = Uint8Array.prototype.slice.call(input);
|
|
1352
|
+
const layout = parsePortableLayout(bytes);
|
|
1353
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1354
|
+
const keys = new Array(layout.length);
|
|
1355
|
+
const conts = new Array(layout.length);
|
|
1356
|
+
let n = 0;
|
|
1357
|
+
let end = bytes.byteLength === 0 ? 0 : 8;
|
|
1358
|
+
let slabLength = 0;
|
|
1359
|
+
let small = 0;
|
|
1360
|
+
for (const { cardinality, kind, offset } of layout) {
|
|
1361
|
+
if (kind === ARRAY) {
|
|
1362
|
+
slabLength += cardinality;
|
|
1363
|
+
small++;
|
|
1364
|
+
} else if (kind !== BITMAP) {
|
|
1365
|
+
slabLength += 2 * view.getUint16(offset, true);
|
|
1366
|
+
small++;
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
const own = LITTLE_ENDIAN && bytes.byteOffset === 0 && bytes.buffer instanceof ArrayBuffer && bytes.buffer.byteLength === bytes.byteLength;
|
|
1370
|
+
const alias = own && small === 1;
|
|
1371
|
+
const slab = alias ? NO_U16 : new Uint16Array(slabLength);
|
|
1372
|
+
let used = 0;
|
|
1373
|
+
for (const { key, cardinality, kind, offset } of layout) {
|
|
1374
|
+
keys[n] = key;
|
|
1375
|
+
if (kind === ARRAY) {
|
|
1376
|
+
if (alias && offset % 2 === 0) {
|
|
1377
|
+
const values = new Uint16Array(bytes.buffer, offset, cardinality);
|
|
1378
|
+
conts[n++] = new Container(ARRAY2, values, 0, cardinality, cardinality, false);
|
|
1379
|
+
} else {
|
|
1380
|
+
const into = alias ? new Uint16Array(cardinality) : slab;
|
|
1381
|
+
getU16s(bytes, view, offset, into, alias ? 0 : used, cardinality);
|
|
1382
|
+
conts[n++] = new Container(
|
|
1383
|
+
ARRAY2,
|
|
1384
|
+
into,
|
|
1385
|
+
alias ? 0 : used,
|
|
1386
|
+
cardinality,
|
|
1387
|
+
cardinality,
|
|
1388
|
+
small > 1
|
|
1389
|
+
);
|
|
1390
|
+
used += cardinality;
|
|
1391
|
+
}
|
|
1392
|
+
end = offset + 2 * cardinality;
|
|
1393
|
+
} else if (kind === BITMAP) {
|
|
1394
|
+
let words;
|
|
1395
|
+
if (own && offset % 4 === 0) {
|
|
1396
|
+
words = new Uint32Array(bytes.buffer, offset, WORDS);
|
|
1397
|
+
} else {
|
|
1398
|
+
words = new Uint32Array(WORDS);
|
|
1399
|
+
if (LITTLE_ENDIAN) {
|
|
1400
|
+
new Uint8Array(words.buffer).set(bytes.subarray(offset, offset + BITSET_BYTES));
|
|
1401
|
+
} else {
|
|
1402
|
+
for (let i = 0; i < WORDS; i++) words[i] = view.getUint32(offset + 4 * i, true);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
conts[n++] = bitsetContainer(words, cardinality);
|
|
1406
|
+
end = offset + BITSET_BYTES;
|
|
1407
|
+
} else {
|
|
1408
|
+
const runs = view.getUint16(offset, true);
|
|
1409
|
+
if (alias && (offset + 2) % 2 === 0) {
|
|
1410
|
+
const pairs = new Uint16Array(bytes.buffer, offset + 2, 2 * runs);
|
|
1411
|
+
conts[n++] = new Container(RUN2, pairs, 0, runs, cardinality, false);
|
|
1412
|
+
} else {
|
|
1413
|
+
const into = alias ? new Uint16Array(2 * runs) : slab;
|
|
1414
|
+
getU16s(bytes, view, offset + 2, into, alias ? 0 : used, 2 * runs);
|
|
1415
|
+
conts[n++] = new Container(RUN2, into, alias ? 0 : used, runs, cardinality, small > 1);
|
|
1416
|
+
used += 2 * runs;
|
|
1417
|
+
}
|
|
1418
|
+
end = offset + 2 + 4 * runs;
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
return { keys, conts, end };
|
|
1422
|
+
}
|
|
1423
|
+
var NO_U16 = new Uint16Array(0);
|
|
1424
|
+
function getU16s(bytes, view, at, slab, to, count) {
|
|
1425
|
+
if (LITTLE_ENDIAN) {
|
|
1426
|
+
new Uint8Array(slab.buffer, 2 * to, 2 * count).set(bytes.subarray(at, at + 2 * count));
|
|
1427
|
+
return;
|
|
1428
|
+
}
|
|
1429
|
+
for (let i = 0; i < count; i++) slab[to + i] = view.getUint16(at + 2 * i, true);
|
|
1430
|
+
}
|
|
1431
|
+
function trailingBytes(length, end) {
|
|
1432
|
+
return new IntegrityError2(
|
|
1433
|
+
`portable roaring: ${length - end} bytes follow the bitmap's last container, which ends at byte ${end}. One buffer holds one bitmap: to load several as one segment, combine them first and serialize the result.`
|
|
1434
|
+
);
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
// src/ts-roaring/bitmap.ts
|
|
1438
|
+
var MAX_ID = 4294967295;
|
|
1439
|
+
function isId(value) {
|
|
1440
|
+
return Number.isInteger(value) && value >= 0 && value <= MAX_ID;
|
|
1441
|
+
}
|
|
1442
|
+
function badId(value) {
|
|
1443
|
+
return new ValidationError2(`${String(value)} is not an integer from 0 to ${MAX_ID}`);
|
|
1444
|
+
}
|
|
1445
|
+
function checkedId(value) {
|
|
1446
|
+
if (!isId(value)) throw badId(value);
|
|
1447
|
+
return value;
|
|
1448
|
+
}
|
|
1449
|
+
function collect(values) {
|
|
1450
|
+
if (values instanceof Uint32Array) return values;
|
|
1451
|
+
if (Array.isArray(values)) {
|
|
1452
|
+
const list = values;
|
|
1453
|
+
const out2 = new Uint32Array(list.length);
|
|
1454
|
+
for (let i = 0; i < list.length; i++) {
|
|
1455
|
+
const v = list[i];
|
|
1456
|
+
if (v >>> 0 !== v) throw badId(v);
|
|
1457
|
+
out2[i] = v;
|
|
1458
|
+
}
|
|
1459
|
+
return out2;
|
|
1460
|
+
}
|
|
1461
|
+
let out = new Uint32Array(1024);
|
|
1462
|
+
let n = 0;
|
|
1463
|
+
for (const v of values) {
|
|
1464
|
+
if (n === out.length) {
|
|
1465
|
+
const grown = new Uint32Array(2 * n);
|
|
1466
|
+
grown.set(out);
|
|
1467
|
+
out = grown;
|
|
1468
|
+
}
|
|
1469
|
+
out[n++] = checkedId(v);
|
|
1470
|
+
}
|
|
1471
|
+
return out.subarray(0, n);
|
|
1472
|
+
}
|
|
1473
|
+
function collectLenient(values) {
|
|
1474
|
+
const list = Array.isArray(values) ? values : Array.from(values);
|
|
1475
|
+
const out = new Uint32Array(list.length);
|
|
1476
|
+
let n = 0;
|
|
1477
|
+
for (let i = 0; i < list.length; i++) {
|
|
1478
|
+
const v = list[i];
|
|
1479
|
+
if (v >>> 0 === v) out[n++] = v;
|
|
1480
|
+
}
|
|
1481
|
+
return out.subarray(0, n);
|
|
1482
|
+
}
|
|
1483
|
+
function sortedIds(ids, owned) {
|
|
1484
|
+
for (let i = 1; i < ids.length; i++) {
|
|
1485
|
+
if (ids[i - 1] > ids[i]) return (owned ? ids : ids.slice()).sort();
|
|
1486
|
+
}
|
|
1487
|
+
return ids;
|
|
1488
|
+
}
|
|
1489
|
+
function lowerBound2(keys, key, from = 0) {
|
|
1490
|
+
let lo = from;
|
|
1491
|
+
let hi = keys.length;
|
|
1492
|
+
while (lo < hi) {
|
|
1493
|
+
const mid = lo + hi >>> 1;
|
|
1494
|
+
if (keys[mid] < key) lo = mid + 1;
|
|
1495
|
+
else hi = mid;
|
|
1496
|
+
}
|
|
1497
|
+
return lo;
|
|
1498
|
+
}
|
|
1499
|
+
function gallopKeys(keys, key, from) {
|
|
1500
|
+
let step = 1;
|
|
1501
|
+
let lo = from;
|
|
1502
|
+
let hi = from;
|
|
1503
|
+
while (hi < keys.length && keys[hi] < key) {
|
|
1504
|
+
lo = hi + 1;
|
|
1505
|
+
hi += step;
|
|
1506
|
+
step *= 2;
|
|
1507
|
+
}
|
|
1508
|
+
return lowerBoundIn(keys, key, lo, Math.min(hi, keys.length));
|
|
1509
|
+
}
|
|
1510
|
+
function lowerBoundIn(keys, key, from, to) {
|
|
1511
|
+
let lo = from;
|
|
1512
|
+
let hi = to;
|
|
1513
|
+
while (lo < hi) {
|
|
1514
|
+
const mid = lo + hi >>> 1;
|
|
1515
|
+
if (keys[mid] < key) lo = mid + 1;
|
|
1516
|
+
else hi = mid;
|
|
1517
|
+
}
|
|
1518
|
+
return lo;
|
|
1519
|
+
}
|
|
1520
|
+
var OR = 0;
|
|
1521
|
+
var AND = 1;
|
|
1522
|
+
var AND_NOT = 2;
|
|
1523
|
+
function combineContainers(op, a, b) {
|
|
1524
|
+
if (op === OR) return containerOr(a, b, !a.shared);
|
|
1525
|
+
if (op === AND) return containerAnd(a, b, !a.shared);
|
|
1526
|
+
return containerAndNot(a, b, !a.shared);
|
|
1527
|
+
}
|
|
1528
|
+
var TsBitmap = class _TsBitmap {
|
|
1529
|
+
#keys;
|
|
1530
|
+
#conts;
|
|
1531
|
+
#size;
|
|
1532
|
+
constructor(keys, conts, size) {
|
|
1533
|
+
this.#keys = keys;
|
|
1534
|
+
this.#conts = conts;
|
|
1535
|
+
this.#size = size;
|
|
317
1536
|
}
|
|
318
1537
|
static empty() {
|
|
319
|
-
return new
|
|
1538
|
+
return new _TsBitmap([], [], 0);
|
|
320
1539
|
}
|
|
321
1540
|
static fromValues(values) {
|
|
322
|
-
|
|
1541
|
+
const bitmap = _TsBitmap.empty();
|
|
1542
|
+
bitmap.addMany(values);
|
|
1543
|
+
return bitmap;
|
|
323
1544
|
}
|
|
324
1545
|
/**
|
|
325
|
-
* Cap the size, check the structure, then
|
|
326
|
-
*
|
|
327
|
-
* decode — the native addon is never handed unbounded, malformed or unsafe-format input. With `whole`, bytes
|
|
328
|
-
* after the bitmap's last container are refused too; without it they are ignored, as the native decoder ignores
|
|
329
|
-
* them.
|
|
1546
|
+
* Cap the size, check the structure, then build the containers. With `whole`, bytes after the last container are
|
|
1547
|
+
* refused too; without it they are ignored.
|
|
330
1548
|
*
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
* values out of order, overlapping runs, a cardinality that disagrees with the bits) decode into a bitmap
|
|
334
|
-
* whose `maximum()`, `has()` and `size` are wrong, and some of those shapes crash the process when used.
|
|
1549
|
+
* @throws {IntegrityError} if the input exceeds `maxBytes`, is not a well-formed portable bitmap, or (with `whole`)
|
|
1550
|
+
* has bytes after it.
|
|
335
1551
|
*/
|
|
336
1552
|
static safeDeserialize(bytes, maxBytes, options = {}) {
|
|
337
1553
|
if (bytes.length > maxBytes) {
|
|
338
|
-
throw new
|
|
1554
|
+
throw new IntegrityError3(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
339
1555
|
}
|
|
340
|
-
const end =
|
|
1556
|
+
const { keys, conts, end } = decodeContainers(bytes);
|
|
341
1557
|
if (options.whole === true && end !== bytes.byteLength) {
|
|
342
|
-
throw
|
|
343
|
-
`portable roaring: ${bytes.byteLength - end} bytes follow the bitmap's last container, which ends at byte ${end}. One buffer holds one bitmap: to load several as one segment, combine them first (RoaringBitmap32.orMany) and serialize the result.`
|
|
344
|
-
);
|
|
1558
|
+
throw trailingBytes(bytes.byteLength, end);
|
|
345
1559
|
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
/** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
|
|
353
|
-
unwrap() {
|
|
354
|
-
return this.bitmap;
|
|
1560
|
+
let size = 0;
|
|
1561
|
+
for (let i = 0; i < conts.length; i++) size += conts[i].card;
|
|
1562
|
+
return new _TsBitmap(keys, conts, size);
|
|
355
1563
|
}
|
|
356
1564
|
serialize() {
|
|
357
|
-
return this
|
|
1565
|
+
return serializeContainers(this.#keys, this.#conts);
|
|
358
1566
|
}
|
|
359
|
-
/**
|
|
360
|
-
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
361
|
-
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
362
|
-
*
|
|
363
|
-
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
364
|
-
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
365
|
-
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
366
|
-
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
367
|
-
* whichever encoding is smaller per container.
|
|
368
|
-
*
|
|
369
|
-
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
370
|
-
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
371
|
-
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
372
|
-
* columns; don't carry a figure between them without checking which denominator it had.
|
|
373
|
-
*
|
|
374
|
-
* Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
|
|
375
|
-
* path deliberately does not.
|
|
376
|
-
*
|
|
377
|
-
* CANONICAL, which `runOptimize()` alone is not. Where a container's run and array encodings are the same size
|
|
378
|
-
* (cardinality = 2 × runs + 1: three values in one run, five in two), CRoaring keeps whichever kind the
|
|
379
|
-
* container already has. A chunk built from ids starts as an array and stays one; the same set reached through
|
|
380
|
-
* `addRange`, an earlier `runOptimize()` or an operation over run containers is a run, and serializes to other
|
|
381
|
-
* bytes. Undoing run compression first starts every container where one built from ids starts, so the bytes
|
|
382
|
-
* depend on membership alone. On a bitmap built from ids there is nothing to undo.
|
|
383
|
-
*/
|
|
1567
|
+
/** Re-encode each container canonically; membership is untouched. */
|
|
384
1568
|
optimize() {
|
|
385
|
-
this
|
|
386
|
-
|
|
1569
|
+
const conts = this.#conts;
|
|
1570
|
+
for (let i = 0; i < conts.length; i++) conts[i] = optimize(conts[i]);
|
|
387
1571
|
}
|
|
388
1572
|
/**
|
|
389
|
-
*
|
|
390
|
-
*
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
1573
|
+
* The set as storage chunks: each payload is the portable bytes of a one-container bitmap holding that chunk's low 16
|
|
1574
|
+
* bits, optimized the way {@link optimize} does. Nothing is built until a chunk is asked for, so a writer that yields
|
|
1575
|
+
* between chunks runs this in slices. The call snapshots the containers and marks each shared, so a change to the
|
|
1576
|
+
* bitmap while the chunks are read copies a container instead of altering what the chunks will say. The bitmap's own
|
|
1577
|
+
* containers are not optimized by it.
|
|
394
1578
|
*/
|
|
395
1579
|
encodeChunks() {
|
|
396
|
-
this.
|
|
397
|
-
|
|
1580
|
+
const keys = this.#keys.slice();
|
|
1581
|
+
const conts = this.#conts.slice();
|
|
1582
|
+
for (let i = 0; i < conts.length; i++) conts[i].shared = true;
|
|
1583
|
+
return {
|
|
1584
|
+
*[Symbol.iterator]() {
|
|
1585
|
+
for (let i = 0; i < conts.length; i++) {
|
|
1586
|
+
const c = optimize(conts[i]);
|
|
1587
|
+
yield {
|
|
1588
|
+
chunkKey: keys[i],
|
|
1589
|
+
payload: serializeContainers([0], [c]),
|
|
1590
|
+
cardinality: c.card
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
};
|
|
398
1595
|
}
|
|
399
|
-
/**
|
|
400
|
-
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the last container, so the engine's
|
|
401
|
-
* per-chunk range assertion costs one call per chunk rather than a walk per id. The last container holds the
|
|
402
|
-
* largest value only because {@link SafeBitmap.safeDeserialize} refuses bytes whose containers or values are
|
|
403
|
-
* out of order.
|
|
404
|
-
*/
|
|
405
1596
|
maximum() {
|
|
406
|
-
|
|
1597
|
+
const last = this.#conts.length - 1;
|
|
1598
|
+
if (last < 0) return void 0;
|
|
1599
|
+
return this.#keys[last] * 65536 + containerMax(this.#conts[last]);
|
|
407
1600
|
}
|
|
408
1601
|
add(value) {
|
|
409
|
-
|
|
1602
|
+
checkedId(value);
|
|
1603
|
+
const key = value >>> 16;
|
|
1604
|
+
const low = value & 65535;
|
|
1605
|
+
const at = lowerBound2(this.#keys, key);
|
|
1606
|
+
const found = this.#keys[at] === key;
|
|
1607
|
+
if (!found) {
|
|
1608
|
+
this.#insert(at, key, arrayContainer(Uint16Array.of(low), 1));
|
|
1609
|
+
this.#size++;
|
|
1610
|
+
return;
|
|
1611
|
+
}
|
|
1612
|
+
const before = this.#conts[at];
|
|
1613
|
+
const after = containerAdd(before, low);
|
|
1614
|
+
this.#conts[at] = after;
|
|
1615
|
+
this.#size += after.card - before.card;
|
|
410
1616
|
}
|
|
1617
|
+
/**
|
|
1618
|
+
* Add the ids. They are grouped by key in ascending order, so each container is touched once, a key already held is
|
|
1619
|
+
* merged where it is, and the new keys are put in with one pass over the key list however many there are.
|
|
1620
|
+
*/
|
|
411
1621
|
addMany(values) {
|
|
412
|
-
|
|
1622
|
+
const ids = sortedIds(collect(values), !(values instanceof Uint32Array));
|
|
1623
|
+
const fresh = [];
|
|
1624
|
+
let at = 0;
|
|
1625
|
+
for (let i = 0; i < ids.length; ) {
|
|
1626
|
+
const key = ids[i] >>> 16;
|
|
1627
|
+
let end = i + 1;
|
|
1628
|
+
while (end < ids.length && ids[end] >>> 16 === key) end++;
|
|
1629
|
+
const group = fromSortedIds(ids, i, end);
|
|
1630
|
+
i = end;
|
|
1631
|
+
at = gallopKeys(this.#keys, key, at);
|
|
1632
|
+
if (this.#keys[at] === key) {
|
|
1633
|
+
const before = this.#conts[at];
|
|
1634
|
+
const after = containerOr(before, group, !before.shared);
|
|
1635
|
+
this.#conts[at] = after;
|
|
1636
|
+
this.#size += after.card - before.card;
|
|
1637
|
+
} else {
|
|
1638
|
+
fresh.push({ key, group });
|
|
1639
|
+
this.#size += group.card;
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
if (fresh.length > 0) this.#insertAll(fresh);
|
|
413
1643
|
}
|
|
414
1644
|
remove(value) {
|
|
415
|
-
|
|
1645
|
+
if (!isId(value)) return;
|
|
1646
|
+
const key = value >>> 16;
|
|
1647
|
+
const at = lowerBound2(this.#keys, key);
|
|
1648
|
+
if (this.#keys[at] !== key) return;
|
|
1649
|
+
const before = this.#conts[at];
|
|
1650
|
+
const after = containerRemove(before, value & 65535);
|
|
1651
|
+
if (after === null) {
|
|
1652
|
+
this.#keys.splice(at, 1);
|
|
1653
|
+
this.#conts.splice(at, 1);
|
|
1654
|
+
this.#size -= before.card;
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
this.#conts[at] = after;
|
|
1658
|
+
this.#size += after.card - before.card;
|
|
416
1659
|
}
|
|
1660
|
+
/** Remove the ids (anything that is not an id is ignored), grouped by key; the keys left empty go in one pass. */
|
|
417
1661
|
removeMany(values) {
|
|
418
|
-
|
|
1662
|
+
const ids = sortedIds(collectLenient(values), true);
|
|
1663
|
+
const keys = this.#keys;
|
|
1664
|
+
const conts = this.#conts;
|
|
1665
|
+
let at = 0;
|
|
1666
|
+
let emptied = 0;
|
|
1667
|
+
for (let i = 0; i < ids.length; ) {
|
|
1668
|
+
const key = ids[i] >>> 16;
|
|
1669
|
+
let end = i + 1;
|
|
1670
|
+
while (end < ids.length && ids[end] >>> 16 === key) end++;
|
|
1671
|
+
const from = i;
|
|
1672
|
+
i = end;
|
|
1673
|
+
at = gallopKeys(keys, key, at);
|
|
1674
|
+
if (keys[at] !== key) continue;
|
|
1675
|
+
const before = conts[at];
|
|
1676
|
+
if (before.card === 0) continue;
|
|
1677
|
+
const after = containerAndNot(before, fromSortedIds(ids, from, end), !before.shared);
|
|
1678
|
+
this.#size -= before.card - (after === null ? 0 : after.card);
|
|
1679
|
+
if (after === null) {
|
|
1680
|
+
conts[at] = EMPTIED;
|
|
1681
|
+
emptied++;
|
|
1682
|
+
} else {
|
|
1683
|
+
conts[at] = after;
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
if (emptied > 0) this.#dropEmptied();
|
|
419
1687
|
}
|
|
420
1688
|
has(value) {
|
|
421
|
-
|
|
1689
|
+
if (!isId(value)) return false;
|
|
1690
|
+
const key = value >>> 16;
|
|
1691
|
+
const at = lowerBound2(this.#keys, key);
|
|
1692
|
+
return this.#keys[at] === key && containerHas(this.#conts[at], value & 65535);
|
|
422
1693
|
}
|
|
423
1694
|
get size() {
|
|
424
|
-
return this
|
|
1695
|
+
return this.#size;
|
|
425
1696
|
}
|
|
426
1697
|
get isEmpty() {
|
|
427
|
-
return this
|
|
1698
|
+
return this.#size === 0;
|
|
428
1699
|
}
|
|
429
1700
|
clone() {
|
|
430
|
-
|
|
1701
|
+
const conts = new Array(this.#conts.length);
|
|
1702
|
+
for (let i = 0; i < conts.length; i++) {
|
|
1703
|
+
const c = this.#conts[i];
|
|
1704
|
+
c.shared = true;
|
|
1705
|
+
conts[i] = c;
|
|
1706
|
+
}
|
|
1707
|
+
return new _TsBitmap(this.#keys.slice(), conts, this.#size);
|
|
431
1708
|
}
|
|
432
|
-
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
433
|
-
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
434
|
-
// `bitmap` (accessible on same-class instances) for the native op.
|
|
435
|
-
/** In-place union: `this = this ∪ other`. */
|
|
436
1709
|
orInPlace(other) {
|
|
437
|
-
this
|
|
438
|
-
}
|
|
439
|
-
/** In-place difference: `this = this \ other`. */
|
|
440
|
-
andNotInPlace(other) {
|
|
441
|
-
this.bitmap.andNotInPlace(other.bitmap);
|
|
1710
|
+
this.#combine(_TsBitmap.#of(other), OR);
|
|
442
1711
|
}
|
|
443
|
-
/** In-place intersection: `this = this ∩ other`. */
|
|
444
1712
|
andInPlace(other) {
|
|
445
|
-
this
|
|
1713
|
+
this.#combine(_TsBitmap.#of(other), AND);
|
|
1714
|
+
}
|
|
1715
|
+
andNotInPlace(other) {
|
|
1716
|
+
this.#combine(_TsBitmap.#of(other), AND_NOT);
|
|
446
1717
|
}
|
|
447
|
-
/** Ascending iterator over the set values. */
|
|
448
1718
|
[Symbol.iterator]() {
|
|
449
|
-
return this
|
|
1719
|
+
return new BitmapIterator(this.#keys, this.#conts);
|
|
450
1720
|
}
|
|
451
1721
|
toArray() {
|
|
452
|
-
return this.
|
|
1722
|
+
return Array.from(this.toUint32Array());
|
|
453
1723
|
}
|
|
454
|
-
/** The
|
|
1724
|
+
/** The ids as a new ascending `Uint32Array`. */
|
|
455
1725
|
toUint32Array() {
|
|
456
|
-
|
|
1726
|
+
const out = new Uint32Array(this.#size);
|
|
1727
|
+
let at = 0;
|
|
1728
|
+
for (let i = 0; i < this.#conts.length; i++) {
|
|
1729
|
+
at = appendIds(this.#conts[i], out, at, this.#keys[i] * 65536);
|
|
1730
|
+
}
|
|
1731
|
+
return out;
|
|
1732
|
+
}
|
|
1733
|
+
static #of(other) {
|
|
1734
|
+
if (!(other instanceof _TsBitmap)) {
|
|
1735
|
+
throw new ValidationError2("a TypeScript roaring bitmap combines only with another one");
|
|
1736
|
+
}
|
|
1737
|
+
return other;
|
|
1738
|
+
}
|
|
1739
|
+
#insert(at, key, container) {
|
|
1740
|
+
if (at === this.#keys.length) {
|
|
1741
|
+
this.#keys.push(key);
|
|
1742
|
+
this.#conts.push(container);
|
|
1743
|
+
} else {
|
|
1744
|
+
this.#keys.splice(at, 0, key);
|
|
1745
|
+
this.#conts.splice(at, 0, container);
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1748
|
+
/** Put the new keys, ascending and none held, in: after the last key by pushing, else in one merge pass. */
|
|
1749
|
+
#insertAll(fresh) {
|
|
1750
|
+
const keys = this.#keys;
|
|
1751
|
+
const conts = this.#conts;
|
|
1752
|
+
if (keys.length === 0 || keys[keys.length - 1] < fresh[0].key) {
|
|
1753
|
+
for (const { key, group } of fresh) {
|
|
1754
|
+
keys.push(key);
|
|
1755
|
+
conts.push(group);
|
|
1756
|
+
}
|
|
1757
|
+
return;
|
|
1758
|
+
}
|
|
1759
|
+
const outKeys = [];
|
|
1760
|
+
const outConts = [];
|
|
1761
|
+
let i = 0;
|
|
1762
|
+
let j = 0;
|
|
1763
|
+
while (i < keys.length || j < fresh.length) {
|
|
1764
|
+
if (j === fresh.length || i < keys.length && keys[i] < fresh[j].key) {
|
|
1765
|
+
outKeys.push(keys[i]);
|
|
1766
|
+
outConts.push(conts[i++]);
|
|
1767
|
+
} else {
|
|
1768
|
+
const next = fresh[j++];
|
|
1769
|
+
outKeys.push(next.key);
|
|
1770
|
+
outConts.push(next.group);
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
this.#keys = outKeys;
|
|
1774
|
+
this.#conts = outConts;
|
|
1775
|
+
}
|
|
1776
|
+
/** Remove the keys whose container was emptied by {@link removeMany}, in one pass. */
|
|
1777
|
+
#dropEmptied() {
|
|
1778
|
+
let kept = 0;
|
|
1779
|
+
for (let i = 0; i < this.#conts.length; i++) {
|
|
1780
|
+
if (this.#conts[i] === EMPTIED) continue;
|
|
1781
|
+
this.#keys[kept] = this.#keys[i];
|
|
1782
|
+
this.#conts[kept++] = this.#conts[i];
|
|
1783
|
+
}
|
|
1784
|
+
this.#keys.length = kept;
|
|
1785
|
+
this.#conts.length = kept;
|
|
1786
|
+
}
|
|
1787
|
+
/**
|
|
1788
|
+
* Walk both key lists and rebuild this one: where both hold a key, the combine of the two containers; where only this
|
|
1789
|
+
* does, its container, unless the operation is an intersection; where only `that` does, a copy of its container for a
|
|
1790
|
+
* union and nothing otherwise. A combine that leaves no value drops the key. An intersection jumps over a stretch of
|
|
1791
|
+
* keys the other side lacks by doubling steps, so a few keys against many cost the few.
|
|
1792
|
+
*/
|
|
1793
|
+
#combine(that, op) {
|
|
1794
|
+
const a = this.#keys;
|
|
1795
|
+
const b = that.#keys;
|
|
1796
|
+
const ca = this.#conts;
|
|
1797
|
+
const cb = that.#conts;
|
|
1798
|
+
const keys = [];
|
|
1799
|
+
const conts = [];
|
|
1800
|
+
let size = 0;
|
|
1801
|
+
let i = 0;
|
|
1802
|
+
let j = 0;
|
|
1803
|
+
while (i < a.length && j < b.length) {
|
|
1804
|
+
const ka = a[i];
|
|
1805
|
+
const kb = b[j];
|
|
1806
|
+
if (ka === kb) {
|
|
1807
|
+
const c = combineContainers(op, ca[i++], cb[j++]);
|
|
1808
|
+
if (c !== null) {
|
|
1809
|
+
keys.push(ka);
|
|
1810
|
+
conts.push(c);
|
|
1811
|
+
size += c.card;
|
|
1812
|
+
}
|
|
1813
|
+
} else if (ka < kb) {
|
|
1814
|
+
if (op === AND) {
|
|
1815
|
+
i = gallopKeys(a, kb, i + 1);
|
|
1816
|
+
} else {
|
|
1817
|
+
keys.push(ka);
|
|
1818
|
+
conts.push(ca[i]);
|
|
1819
|
+
size += ca[i++].card;
|
|
1820
|
+
}
|
|
1821
|
+
} else if (op === AND) {
|
|
1822
|
+
j = gallopKeys(b, ka, j + 1);
|
|
1823
|
+
} else {
|
|
1824
|
+
if (op === OR) {
|
|
1825
|
+
keys.push(kb);
|
|
1826
|
+
conts.push(cloneContainer(cb[j]));
|
|
1827
|
+
size += cb[j].card;
|
|
1828
|
+
}
|
|
1829
|
+
j++;
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
if (op !== AND) {
|
|
1833
|
+
for (; i < a.length; i++) {
|
|
1834
|
+
keys.push(a[i]);
|
|
1835
|
+
conts.push(ca[i]);
|
|
1836
|
+
size += ca[i].card;
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
if (op === OR) {
|
|
1840
|
+
for (; j < b.length; j++) {
|
|
1841
|
+
keys.push(b[j]);
|
|
1842
|
+
conts.push(cloneContainer(cb[j]));
|
|
1843
|
+
size += cb[j].card;
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
this.#keys = keys;
|
|
1847
|
+
this.#conts = conts;
|
|
1848
|
+
this.#size = size;
|
|
457
1849
|
}
|
|
458
1850
|
};
|
|
1851
|
+
var EMPTIED = arrayContainer(new Uint16Array(0), 0);
|
|
1852
|
+
var tsRoaringCodec = {
|
|
1853
|
+
empty: () => TsBitmap.empty(),
|
|
1854
|
+
fromValues: (values) => TsBitmap.fromValues(values),
|
|
1855
|
+
safeDeserialize: (bytes, maxBytes, options) => TsBitmap.safeDeserialize(bytes, maxBytes, options),
|
|
1856
|
+
owns: (bitmap) => bitmap instanceof TsBitmap,
|
|
1857
|
+
intersectSmallestFirst: true
|
|
1858
|
+
};
|
|
1859
|
+
|
|
1860
|
+
// src/roaring-codec.ts
|
|
459
1861
|
function bitmapAsLoadInput(input) {
|
|
460
|
-
|
|
1862
|
+
if (typeof input !== "object" || input === null) return input;
|
|
1863
|
+
const tag = Object.prototype.toString.call(input).slice(8, -1);
|
|
1864
|
+
if (tag === "Uint8Array" || tag === "Uint8ClampedArray") return input;
|
|
1865
|
+
return Symbol.iterator in input && typeof input.serialize === "function" ? { bitmap: input } : input;
|
|
461
1866
|
}
|
|
462
|
-
var
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
1867
|
+
var activeCodec = tsRoaringCodec;
|
|
1868
|
+
function flagshipCodec() {
|
|
1869
|
+
return activeCodec;
|
|
1870
|
+
}
|
|
1871
|
+
var ReadonlyRoaringView = class {
|
|
1872
|
+
#bitmap;
|
|
1873
|
+
constructor(bitmap) {
|
|
1874
|
+
this.#bitmap = bitmap;
|
|
1875
|
+
}
|
|
1876
|
+
get size() {
|
|
1877
|
+
return this.#bitmap.size;
|
|
1878
|
+
}
|
|
1879
|
+
has(id) {
|
|
1880
|
+
return this.#bitmap.has(id);
|
|
1881
|
+
}
|
|
1882
|
+
[Symbol.iterator]() {
|
|
1883
|
+
return this.#bitmap[Symbol.iterator]();
|
|
1884
|
+
}
|
|
1885
|
+
/** The portable bytes, for exactly the argument `'portable'`; any other call is a `ValidationError`. */
|
|
1886
|
+
serialize(...args) {
|
|
1887
|
+
if (args.length !== 1 || args[0] !== "portable") {
|
|
1888
|
+
const got = args.length === 0 ? "no argument" : args.map((a) => typeof a === "string" ? `'${a}'` : typeof a).join(", ");
|
|
1889
|
+
throw new ValidationError3(
|
|
1890
|
+
`a decoded bitmap serializes only as serialize('portable'); got ${got}`
|
|
1891
|
+
);
|
|
1892
|
+
}
|
|
1893
|
+
return this.#bitmap.serialize();
|
|
1894
|
+
}
|
|
467
1895
|
};
|
|
468
1896
|
function deserializePortable(bytes) {
|
|
469
|
-
return decodeSerialized(bytes,
|
|
1897
|
+
return new ReadonlyRoaringView(decodeSerialized(bytes, flagshipCodec()));
|
|
470
1898
|
}
|
|
471
1899
|
|
|
472
1900
|
// src/system-clock.ts
|
|
@@ -607,17 +2035,18 @@ import {
|
|
|
607
2035
|
DEFAULT_RETRY_POLICY,
|
|
608
2036
|
NOOP_METRICS,
|
|
609
2037
|
RetryingStorageChunkSource,
|
|
610
|
-
ValidationError as
|
|
2038
|
+
ValidationError as ValidationError4,
|
|
611
2039
|
isStorageBackend
|
|
612
2040
|
} from "@cloudbitmaps/core";
|
|
613
2041
|
function resolveRetryPolicy(partial) {
|
|
614
|
-
const
|
|
2042
|
+
const orDefault = (field) => partial[field] === void 0 ? DEFAULT_RETRY_POLICY[field] : partial[field];
|
|
2043
|
+
const baseDelayMs = orDefault("baseDelayMs");
|
|
615
2044
|
return {
|
|
616
|
-
maxAttempts:
|
|
2045
|
+
maxAttempts: orDefault("maxAttempts"),
|
|
617
2046
|
baseDelayMs,
|
|
618
|
-
maxDelayMs: partial.maxDelayMs
|
|
619
|
-
backoffFactor:
|
|
620
|
-
jitter:
|
|
2047
|
+
maxDelayMs: partial.maxDelayMs === void 0 ? typeof baseDelayMs === "number" && baseDelayMs > DEFAULT_RETRY_POLICY.maxDelayMs ? baseDelayMs : DEFAULT_RETRY_POLICY.maxDelayMs : partial.maxDelayMs,
|
|
2048
|
+
backoffFactor: orDefault("backoffFactor"),
|
|
2049
|
+
jitter: orDefault("jitter")
|
|
621
2050
|
};
|
|
622
2051
|
}
|
|
623
2052
|
function wireStore(options, clock, rng, metrics) {
|
|
@@ -630,7 +2059,7 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
630
2059
|
if (options.retry !== false) {
|
|
631
2060
|
const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
|
|
632
2061
|
if (userOnRetry !== void 0 && typeof userOnRetry !== "function") {
|
|
633
|
-
throw new
|
|
2062
|
+
throw new ValidationError4("retry.onRetry must be a function");
|
|
634
2063
|
}
|
|
635
2064
|
retryOptions = {
|
|
636
2065
|
clock,
|
|
@@ -654,13 +2083,14 @@ function wireStore(options, clock, rng, metrics) {
|
|
|
654
2083
|
source: resolved.source,
|
|
655
2084
|
driver: resolved.driver,
|
|
656
2085
|
registry: resolved.registry,
|
|
2086
|
+
backend: isStorageBackend(options.storage) ? options.storage : void 0,
|
|
657
2087
|
retryOptions
|
|
658
2088
|
};
|
|
659
2089
|
}
|
|
660
2090
|
function resolveStorageSource(options, clock) {
|
|
661
2091
|
const storage = options.storage;
|
|
662
2092
|
if (storage === null || typeof storage !== "object") {
|
|
663
|
-
throw new
|
|
2093
|
+
throw new ValidationError4(
|
|
664
2094
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
665
2095
|
);
|
|
666
2096
|
}
|
|
@@ -670,14 +2100,14 @@ function resolveStorageSource(options, clock) {
|
|
|
670
2100
|
const hasPutImmutable = typeof storage.putImmutable === "function";
|
|
671
2101
|
const isBackend = isStorageBackend(storage);
|
|
672
2102
|
if (isBackend && (hasGetChunk || hasPutImmutable)) {
|
|
673
|
-
throw new
|
|
2103
|
+
throw new ValidationError4(
|
|
674
2104
|
`\`storage\` is a StorageBackend that also exposes ${hasGetChunk ? "`getChunk`" : "`putImmutable`"} \u2014 ambiguous. A backend must not also be a driver or a source; pass whichever one you mean.`
|
|
675
2105
|
);
|
|
676
2106
|
}
|
|
677
2107
|
const hasStorageHalf = isObject(asBackend.storage);
|
|
678
2108
|
const hasRegistryHalf = isObject(asBackend.registry);
|
|
679
2109
|
if (!isBackend && hasPutImmutable && hasRegistryHalf) {
|
|
680
|
-
throw new
|
|
2110
|
+
throw new ValidationError4(
|
|
681
2111
|
"`storage` looks like a driver that also carries a `registry`. Passed as a bare driver it would have no pointer at all \u2014 generations would resolve by list-scan, so reads could serve a generation that was written but never published. If you meant a backend, use a backend class \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage \u2014 or, to pair a driver of your own with a registry, brand the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
682
2112
|
);
|
|
683
2113
|
}
|
|
@@ -686,11 +2116,11 @@ function resolveStorageSource(options, clock) {
|
|
|
686
2116
|
const registryOk = typeof asBackend.registry.compareAndSwap === "function";
|
|
687
2117
|
if (!storageOk || !registryOk) {
|
|
688
2118
|
const bad = !storageOk ? "`storage` half is not an IStorageDriver (no `putImmutable`)" : "`registry` half is not an IRegistryDriver (no `compareAndSwap`)";
|
|
689
|
-
throw new
|
|
2119
|
+
throw new ValidationError4(
|
|
690
2120
|
"`storage` looks like a backend but its " + bad + " \u2014 use a backend class, or brand a pair of drivers with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`."
|
|
691
2121
|
);
|
|
692
2122
|
}
|
|
693
|
-
throw new
|
|
2123
|
+
throw new ValidationError4(
|
|
694
2124
|
"`storage` must be a backend \u2014 S3Storage, GcsStorage, AzureBlobStorage, LocalFsStorage or MemoryStorage. An object with `.storage` and `.registry` is not one: a backend builds both halves from a single bucket and prefix, so they cannot disagree, and hand-assembling them re-opens exactly that mismatch \u2014 a store whose pointer and generations live in different places reads as empty rather than failing. A driver author who genuinely wants halves of their own \u2014 an instrumented driver, a registry in a database you already run \u2014 brands the pair with `brandAsBackend` from `@cloudbitmaps/core/driver-kit`, which is taking on that they agree."
|
|
695
2125
|
);
|
|
696
2126
|
}
|
|
@@ -711,18 +2141,18 @@ function resolveStorageSource(options, clock) {
|
|
|
711
2141
|
};
|
|
712
2142
|
}
|
|
713
2143
|
if (hasGetChunk && hasPutImmutable) {
|
|
714
|
-
throw new
|
|
2144
|
+
throw new ValidationError4(
|
|
715
2145
|
"`storage` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass a StorageBackend, an IStorageDriver, or a StorageChunkSource, not a hybrid"
|
|
716
2146
|
);
|
|
717
2147
|
}
|
|
718
2148
|
if (!hasGetChunk && !hasPutImmutable) {
|
|
719
|
-
throw new
|
|
2149
|
+
throw new ValidationError4(
|
|
720
2150
|
"`storage` must be a StorageBackend, an IStorageDriver, or a StorageChunkSource"
|
|
721
2151
|
);
|
|
722
2152
|
}
|
|
723
2153
|
if (hasGetChunk) {
|
|
724
2154
|
if (options.encryption?.keystore !== void 0 || options.encryption?.required === true) {
|
|
725
|
-
throw new
|
|
2155
|
+
throw new ValidationError4(
|
|
726
2156
|
"keystore/requireEncryption apply only when the store builds its own read path: a StorageChunkSource of your own reads an encrypted segment by itself, if it must; to have the store decrypt, pass a backend"
|
|
727
2157
|
);
|
|
728
2158
|
}
|
|
@@ -744,32 +2174,32 @@ function resolveStorageSource(options, clock) {
|
|
|
744
2174
|
}
|
|
745
2175
|
|
|
746
2176
|
// src/options.ts
|
|
747
|
-
import { ValidationError as
|
|
2177
|
+
import { ValidationError as ValidationError5 } from "@cloudbitmaps/core";
|
|
748
2178
|
var keysOf = () => (keys) => keys;
|
|
749
2179
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
750
2180
|
function optionsError(options, op, known, booleans = [], misplaced = []) {
|
|
751
2181
|
if (options === void 0 || options === null) return void 0;
|
|
752
2182
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
753
|
-
return new
|
|
2183
|
+
return new ValidationError5(`${op}: options must be an object; got ${kindOf(options)}`);
|
|
754
2184
|
}
|
|
755
2185
|
const bag = options;
|
|
756
2186
|
const unknown = Object.keys(bag).filter((k) => !known.includes(k) && bag[k] !== void 0);
|
|
757
2187
|
for (const k of misplaced) if (!unknown.includes(k) && bag[k] !== void 0) unknown.push(k);
|
|
758
2188
|
if (unknown.length > 0) {
|
|
759
|
-
return new
|
|
2189
|
+
return new ValidationError5(
|
|
760
2190
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; it takes { ${known.join(", ")} }`
|
|
761
2191
|
);
|
|
762
2192
|
}
|
|
763
2193
|
const audit = bag.audit;
|
|
764
2194
|
if (audit !== void 0 && audit !== null && typeof audit.onEvent !== "function") {
|
|
765
|
-
return new
|
|
2195
|
+
return new ValidationError5(
|
|
766
2196
|
`${op}: audit must be a sink with an onEvent(event) method, such as a RecordingAuditSink`
|
|
767
2197
|
);
|
|
768
2198
|
}
|
|
769
2199
|
for (const name of booleans) {
|
|
770
2200
|
const v = bag[name];
|
|
771
2201
|
if (v !== void 0 && typeof v !== "boolean") {
|
|
772
|
-
return new
|
|
2202
|
+
return new ValidationError5(`${op}: \`${name}\` must be a boolean; got ${kindOf(v)}`);
|
|
773
2203
|
}
|
|
774
2204
|
}
|
|
775
2205
|
return void 0;
|
|
@@ -789,28 +2219,24 @@ function checkOptions(options, op, known, booleans, misplaced) {
|
|
|
789
2219
|
}
|
|
790
2220
|
|
|
791
2221
|
// src/index.ts
|
|
2222
|
+
import { MemoryStorage, LocalFsStorage, InProcessKeystore } from "@cloudbitmaps/core/node";
|
|
792
2223
|
import {
|
|
793
|
-
MemoryStorage,
|
|
794
|
-
LocalFsStorage,
|
|
795
|
-
destroySegment,
|
|
796
|
-
eraseNamespace,
|
|
797
2224
|
excludingReservedRows as excludingReservedRows2,
|
|
798
2225
|
readRetentionPolicy,
|
|
799
|
-
InProcessKeystore,
|
|
800
2226
|
CloudRoaringError,
|
|
801
|
-
ValidationError as
|
|
2227
|
+
ValidationError as ValidationError7,
|
|
802
2228
|
WriteConflictError as WriteConflictError2,
|
|
803
|
-
IntegrityError as
|
|
2229
|
+
IntegrityError as IntegrityError4,
|
|
804
2230
|
NotFoundError as NotFoundError2,
|
|
805
2231
|
UnsupportedError as UnsupportedError3,
|
|
806
2232
|
CapabilityError,
|
|
807
2233
|
TransientError,
|
|
808
2234
|
KeyUnavailableError,
|
|
809
|
-
BudgetExceededError,
|
|
2235
|
+
BudgetExceededError as BudgetExceededError2,
|
|
810
2236
|
LeaseExpiredError as LeaseExpiredError2,
|
|
811
2237
|
LeaseLimitError,
|
|
812
2238
|
StaleOperandError,
|
|
813
|
-
errorKind,
|
|
2239
|
+
errorKind as errorKind2,
|
|
814
2240
|
CountingMetricsSink,
|
|
815
2241
|
RecordingAuditSink
|
|
816
2242
|
} from "@cloudbitmaps/core";
|
|
@@ -820,13 +2246,25 @@ var SystemRng = class {
|
|
|
820
2246
|
return Math.random();
|
|
821
2247
|
}
|
|
822
2248
|
};
|
|
823
|
-
function
|
|
2249
|
+
function withWaits(deps) {
|
|
2250
|
+
if (deps === null || typeof deps !== "object") return deps;
|
|
2251
|
+
return { ...deps, clock: deps.clock ?? new SystemClock(), rng: deps.rng ?? new SystemRng() };
|
|
2252
|
+
}
|
|
2253
|
+
function destroySegment(ref, deps, options) {
|
|
2254
|
+
return coreDestroySegment(ref, withWaits(deps), options);
|
|
2255
|
+
}
|
|
2256
|
+
function eraseNamespace(namespace, deps, options) {
|
|
2257
|
+
return coreEraseNamespace(namespace, withWaits(deps), options);
|
|
2258
|
+
}
|
|
2259
|
+
function validateRef(ref, verb) {
|
|
824
2260
|
if (typeof ref !== "object" || ref === null) {
|
|
825
|
-
throw new
|
|
2261
|
+
throw new ValidationError6(
|
|
826
2262
|
"a segment ref is an object such as { segment: 'name' }; store.segment() takes the bare name"
|
|
827
2263
|
);
|
|
828
2264
|
}
|
|
2265
|
+
refuseUnknownRefKeys(ref, verb);
|
|
829
2266
|
validateSegmentRef(ref);
|
|
2267
|
+
return { segment: ref.segment, namespace: ref.namespace };
|
|
830
2268
|
}
|
|
831
2269
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
832
2270
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
@@ -841,15 +2279,20 @@ function validateConcurrency(concurrency) {
|
|
|
841
2279
|
function requireScope(options, op) {
|
|
842
2280
|
if (options.namespace !== void 0) {
|
|
843
2281
|
if (typeof options.namespace !== "string") {
|
|
844
|
-
throw new
|
|
2282
|
+
throw new ValidationError6(
|
|
845
2283
|
`${op}: namespace must be a string; got ${kindOf(options.namespace)}`
|
|
846
2284
|
);
|
|
847
2285
|
}
|
|
848
2286
|
validateSegmentRef({ segment: "x", namespace: options.namespace });
|
|
849
2287
|
}
|
|
850
2288
|
refuseReservedNamespace(options.namespace);
|
|
2289
|
+
if (options.namespace !== void 0 && options.allNamespaces === true) {
|
|
2290
|
+
throw new ValidationError6(
|
|
2291
|
+
`${op}: pass a namespace or { allNamespaces: true }, not both: with both, it is unclear whether to scan one namespace or every one`
|
|
2292
|
+
);
|
|
2293
|
+
}
|
|
851
2294
|
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
852
|
-
throw new
|
|
2295
|
+
throw new ValidationError6(
|
|
853
2296
|
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
854
2297
|
);
|
|
855
2298
|
}
|
|
@@ -896,7 +2339,7 @@ var CALL_KEYS = {
|
|
|
896
2339
|
"metadata",
|
|
897
2340
|
"keep"
|
|
898
2341
|
]),
|
|
899
|
-
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit"]),
|
|
2342
|
+
load: keysOf()(["allowEmpty", "guard", "keep", "metadata", "audit", "sorted"]),
|
|
900
2343
|
segments: keysOf()(["namespace"]),
|
|
901
2344
|
subjectReport: keysOf()([
|
|
902
2345
|
"namespace",
|
|
@@ -957,16 +2400,16 @@ var originOf;
|
|
|
957
2400
|
function assertOneStore(verb, role, h, origin) {
|
|
958
2401
|
if (h instanceof Segment) {
|
|
959
2402
|
if (originOf(h) === origin) return;
|
|
960
|
-
throw new
|
|
2403
|
+
throw new ValidationError6(
|
|
961
2404
|
`${verb}: ${role} belongs to another store; combine segments of one store: open each with this store's segment()`
|
|
962
2405
|
);
|
|
963
2406
|
}
|
|
964
2407
|
if (typeof h === "object" && h !== null && h[SEGMENT_BRAND] === true) {
|
|
965
|
-
throw new
|
|
2408
|
+
throw new ValidationError6(
|
|
966
2409
|
`${verb}: ${role} comes from another installed copy of @cloudbitmaps/roaring, so it belongs to another store; install one copy and combine segments of one store`
|
|
967
2410
|
);
|
|
968
2411
|
}
|
|
969
|
-
throw new
|
|
2412
|
+
throw new ValidationError6(`${verb}: ${role} must be a segment from this store's segment()`);
|
|
970
2413
|
}
|
|
971
2414
|
var MATERIALIZE_MANY_KEYS = [
|
|
972
2415
|
"operands",
|
|
@@ -1035,6 +2478,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1035
2478
|
// no underlying driver to write through).
|
|
1036
2479
|
storageDriver;
|
|
1037
2480
|
registry;
|
|
2481
|
+
// The backend itself, for its optional `checkLifecycle`; `undefined` for a bare driver or a source.
|
|
2482
|
+
backend;
|
|
1038
2483
|
keystore;
|
|
1039
2484
|
requireEncryption;
|
|
1040
2485
|
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
@@ -1054,7 +2499,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1054
2499
|
*/
|
|
1055
2500
|
static rejectUnknownOptions(options) {
|
|
1056
2501
|
if (options === null || options === void 0 || typeof options !== "object") {
|
|
1057
|
-
throw new
|
|
2502
|
+
throw new ValidationError6(
|
|
1058
2503
|
"CloudRoaring needs an options object with a `storage` key \u2014 got " + (options === null ? "null" : typeof options)
|
|
1059
2504
|
);
|
|
1060
2505
|
}
|
|
@@ -1072,7 +2517,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1072
2517
|
const tag = Object.prototype.toString.call(value).slice(8, -1);
|
|
1073
2518
|
if (typeof value !== "object" || value === null || NOT_A_GROUP[tag] !== void 0) {
|
|
1074
2519
|
const got = typeof value !== "object" ? typeof value : NOT_A_GROUP[tag];
|
|
1075
|
-
throw new
|
|
2520
|
+
throw new ValidationError6(
|
|
1076
2521
|
`CloudRoaring's \`${group}\` must be an object${offOk ? " or `false`" : ""} \u2014 got ${got}`
|
|
1077
2522
|
);
|
|
1078
2523
|
}
|
|
@@ -1082,7 +2527,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1082
2527
|
if (unknown.length === 0) return;
|
|
1083
2528
|
const takes = (keys) => keys.map((k) => `\`${k}\``).join(", ");
|
|
1084
2529
|
const named = [...new Set(unknown.map((u) => u.group))];
|
|
1085
|
-
throw new
|
|
2530
|
+
throw new ValidationError6(
|
|
1086
2531
|
`CloudRoaring does not take ${takes(unknown.map((u) => u.group === null ? u.key : `${u.group}.${u.key}`))}. ` + named.map(
|
|
1087
2532
|
(g) => g === null ? `The store takes ${takes(OPTION_KEYS.top)}.` : `\`${g}\` takes ${takes(OPTION_KEYS[g])}.`
|
|
1088
2533
|
).join(" ") + " https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/api-reference.md#build-a-store--new-cloudroaringoptions"
|
|
@@ -1100,23 +2545,51 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1100
2545
|
];
|
|
1101
2546
|
const entropy = options.seams?.entropy;
|
|
1102
2547
|
if (entropy !== void 0 && typeof entropy !== "function") {
|
|
1103
|
-
throw new
|
|
2548
|
+
throw new ValidationError6(
|
|
1104
2549
|
"CloudRoaring's `seams.entropy` must be a function `(length) => Uint8Array` returning that many random bytes"
|
|
1105
2550
|
);
|
|
1106
2551
|
}
|
|
1107
2552
|
for (const [seam, name, needs] of seams) {
|
|
1108
|
-
if (seam === void 0
|
|
1109
|
-
const has = (m) => typeof seam[m] === "function";
|
|
1110
|
-
if (typeof seam !== "object" || !needs.every(has)) {
|
|
1111
|
-
throw new
|
|
2553
|
+
if (seam === void 0) continue;
|
|
2554
|
+
const has = (m) => seam !== null && typeof seam[m] === "function";
|
|
2555
|
+
if (seam === null || typeof seam !== "object" || !needs.every(has)) {
|
|
2556
|
+
throw new ValidationError6(
|
|
1112
2557
|
`CloudRoaring's \`${name}\` must be an object with ${needs.map((m) => `\`${m}()\``).join(" and ")}`
|
|
1113
2558
|
);
|
|
1114
2559
|
}
|
|
1115
2560
|
}
|
|
1116
2561
|
}
|
|
2562
|
+
/**
|
|
2563
|
+
* Refuse a `cache` or `encryption` value that can never be honoured, by the name the caller wrote. The sizes reach
|
|
2564
|
+
* the caches under their own internal names, so left to them an error would name `maxEntries` or `maxBytes`; and
|
|
2565
|
+
* `encryption.required` must be exactly a boolean, since the read path took a truthy string for `true` while every
|
|
2566
|
+
* write path did not, which refused cleartext reads and still wrote cleartext.
|
|
2567
|
+
*/
|
|
2568
|
+
static rejectBadValues(options) {
|
|
2569
|
+
const cache = options.cache;
|
|
2570
|
+
if (cache !== void 0) {
|
|
2571
|
+
requirePositiveInt("cache.maxChunks", cache.maxChunks);
|
|
2572
|
+
requirePositiveInt("cache.readerMax", cache.readerMax);
|
|
2573
|
+
requirePositiveInt("cache.readerMaxBytes", cache.readerMaxBytes);
|
|
2574
|
+
const ttl = cache.ttlMs;
|
|
2575
|
+
if (ttl !== void 0 && (typeof ttl !== "number" || !Number.isFinite(ttl) || ttl <= 0)) {
|
|
2576
|
+
throw new ValidationError6(
|
|
2577
|
+
`cache.ttlMs must be a finite number above 0 (milliseconds); got ${typeof ttl === "string" ? JSON.stringify(ttl) : String(ttl)}`
|
|
2578
|
+
);
|
|
2579
|
+
}
|
|
2580
|
+
}
|
|
2581
|
+
requireBool("CloudRoaring's `encryption.required`", options.encryption?.required);
|
|
2582
|
+
}
|
|
2583
|
+
/**
|
|
2584
|
+
* Builds a store. Only `storage` is required: pass a backend such as `S3Storage` or `MemoryStorage`. `cache`,
|
|
2585
|
+
* `encryption`, `retry`, `metrics`, `budget` and `seams` tune it. An option it does not take, a `cache` size or `ttlMs`
|
|
2586
|
+
* that is not a usable number, an `encryption.required` that is not a boolean, or a seam missing a method the store
|
|
2587
|
+
* calls, throws `ValidationError`.
|
|
2588
|
+
*/
|
|
1117
2589
|
constructor(options) {
|
|
1118
2590
|
_CloudRoaring.rejectUnknownOptions(options);
|
|
1119
2591
|
_CloudRoaring.rejectHollowSeams(options);
|
|
2592
|
+
_CloudRoaring.rejectBadValues(options);
|
|
1120
2593
|
const clock = options.seams?.clock ?? new SystemClock();
|
|
1121
2594
|
const rng = options.seams?.rng ?? new SystemRng();
|
|
1122
2595
|
const entropy = options.seams?.entropy ?? systemEntropy;
|
|
@@ -1132,7 +2605,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1132
2605
|
const deps = {
|
|
1133
2606
|
storage: wired.storage,
|
|
1134
2607
|
cache,
|
|
1135
|
-
codec:
|
|
2608
|
+
codec: flagshipCodec(),
|
|
1136
2609
|
// the facade injects the flagship codec; core stays codec-agnostic
|
|
1137
2610
|
clock,
|
|
1138
2611
|
metrics,
|
|
@@ -1147,6 +2620,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1147
2620
|
this.metrics = metrics;
|
|
1148
2621
|
this.storageDriver = wired.driver;
|
|
1149
2622
|
this.registry = wired.registry;
|
|
2623
|
+
if (this.registry !== void 0) REGISTRY_OF.set(this.origin, this.registry);
|
|
2624
|
+
this.backend = wired.backend;
|
|
1150
2625
|
this.keystore = options.encryption?.keystore;
|
|
1151
2626
|
this.requireEncryption = options.encryption?.required ?? false;
|
|
1152
2627
|
}
|
|
@@ -1173,7 +2648,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1173
2648
|
clock: this.clock,
|
|
1174
2649
|
rng: this.rng,
|
|
1175
2650
|
entropy: this.entropy,
|
|
1176
|
-
codec:
|
|
2651
|
+
codec: flagshipCodec(),
|
|
1177
2652
|
// facade injects the flagship codec
|
|
1178
2653
|
keystore: this.keystore,
|
|
1179
2654
|
requireEncryption: this.requireEncryption,
|
|
@@ -1190,17 +2665,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1190
2665
|
*/
|
|
1191
2666
|
segment(name, options) {
|
|
1192
2667
|
if (typeof name === "object" && name !== null) {
|
|
1193
|
-
throw new
|
|
2668
|
+
throw new ValidationError6(
|
|
1194
2669
|
Array.isArray(name) ? "segment: the name is a string, store.segment('name'), not an array" : "segment: the name is a string, not a ref: for a ref, call store.segment(ref.segment, { namespace: ref.namespace })"
|
|
1195
2670
|
);
|
|
1196
2671
|
}
|
|
1197
2672
|
if (options !== void 0 && options !== null) {
|
|
1198
2673
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1199
|
-
throw new
|
|
2674
|
+
throw new ValidationError6("segment: options must be an object such as { namespace }");
|
|
1200
2675
|
}
|
|
1201
2676
|
const proto = Object.getPrototypeOf(options);
|
|
1202
2677
|
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1203
|
-
throw new
|
|
2678
|
+
throw new ValidationError6(
|
|
1204
2679
|
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1205
2680
|
);
|
|
1206
2681
|
}
|
|
@@ -1208,7 +2683,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1208
2683
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1209
2684
|
);
|
|
1210
2685
|
if (unknown.length > 0) {
|
|
1211
|
-
throw new
|
|
2686
|
+
throw new ValidationError6(
|
|
1212
2687
|
`segment: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; a handle takes { namespace } only`
|
|
1213
2688
|
);
|
|
1214
2689
|
}
|
|
@@ -1290,7 +2765,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1290
2765
|
// puts that call on the operator. Inheriting `load()`'s collection would silently delete the
|
|
1291
2766
|
// generations an operator's recovery story depends on — `rollbackSegment` refuses a collected target —
|
|
1292
2767
|
// as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
|
|
1293
|
-
keep: options?.keep
|
|
2768
|
+
keep: options?.keep === void 0 ? KEEP_EVERY_GENERATION : options.keep,
|
|
1294
2769
|
...options?.audit === void 0 ? {} : { audit: options.audit }
|
|
1295
2770
|
};
|
|
1296
2771
|
result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
|
|
@@ -1318,20 +2793,20 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1318
2793
|
/** The one body of the forms above. */
|
|
1319
2794
|
async materializeMany(options) {
|
|
1320
2795
|
if (typeof options !== "object" || options === null) {
|
|
1321
|
-
throw new
|
|
2796
|
+
throw new ValidationError6(
|
|
1322
2797
|
"materializeMany needs an options object with `operands`, `outputs` and `keep`"
|
|
1323
2798
|
);
|
|
1324
2799
|
}
|
|
1325
2800
|
const epochAt = options.feed === void 0 ? 0 : this.epochNow();
|
|
1326
2801
|
const unknown = unknownManyKeys(options);
|
|
1327
2802
|
if (unknown.length > 0) {
|
|
1328
|
-
throw new
|
|
2803
|
+
throw new ValidationError6(
|
|
1329
2804
|
`materializeMany: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}`
|
|
1330
2805
|
);
|
|
1331
2806
|
}
|
|
1332
2807
|
const dryRunIn = options.dryRun;
|
|
1333
2808
|
if (dryRunIn !== void 0 && typeof dryRunIn !== "boolean") {
|
|
1334
|
-
throw new
|
|
2809
|
+
throw new ValidationError6("materializeMany: dryRun must be a boolean");
|
|
1335
2810
|
}
|
|
1336
2811
|
const dryRun = dryRunIn === true;
|
|
1337
2812
|
const operandsIn = options.operands;
|
|
@@ -1343,7 +2818,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1343
2818
|
const feedIn = options.feed;
|
|
1344
2819
|
const mayBeEmptyIn = options.mayBeEmpty;
|
|
1345
2820
|
if (feedIn !== void 0 && options.maxBufferedBytes === void 0) {
|
|
1346
|
-
throw new
|
|
2821
|
+
throw new ValidationError6(
|
|
1347
2822
|
"materializeMany: maxBufferedBytes is required with a feed: the feed is read once, so the call runs as one group and the budget is what bounds it"
|
|
1348
2823
|
);
|
|
1349
2824
|
}
|
|
@@ -1353,15 +2828,15 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1353
2828
|
const budgetOption = options.budget;
|
|
1354
2829
|
const allowAbsentOperands = options.allowAbsentOperands;
|
|
1355
2830
|
if (typeof pinOption !== "boolean")
|
|
1356
|
-
throw new
|
|
2831
|
+
throw new ValidationError6("materializeMany: pin must be a boolean");
|
|
1357
2832
|
if (allowAbsentOperands !== void 0 && typeof allowAbsentOperands !== "boolean") {
|
|
1358
|
-
throw new
|
|
2833
|
+
throw new ValidationError6("materializeMany: allowAbsentOperands must be a boolean");
|
|
1359
2834
|
}
|
|
1360
2835
|
if (typeof operandsIn !== "object" || operandsIn === null || Array.isArray(operandsIn)) {
|
|
1361
|
-
throw new
|
|
2836
|
+
throw new ValidationError6("materializeMany: operands must be an object of name to segment");
|
|
1362
2837
|
}
|
|
1363
2838
|
if (!Array.isArray(outputsIn) || outputsIn.length === 0) {
|
|
1364
|
-
throw new
|
|
2839
|
+
throw new ValidationError6("materializeMany: outputs must be a non-empty array");
|
|
1365
2840
|
}
|
|
1366
2841
|
const names = Object.keys(operandsIn);
|
|
1367
2842
|
const handles = /* @__PURE__ */ new Map();
|
|
@@ -1372,14 +2847,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1372
2847
|
}
|
|
1373
2848
|
if (mayBeEmptyIn !== void 0 && feedIn === void 0) {
|
|
1374
2849
|
if (!Array.isArray(mayBeEmptyIn)) {
|
|
1375
|
-
throw new
|
|
2850
|
+
throw new ValidationError6(
|
|
1376
2851
|
"materializeMany: mayBeEmpty must be an array of fed operand names"
|
|
1377
2852
|
);
|
|
1378
2853
|
}
|
|
1379
2854
|
if (mayBeEmptyIn.length > 0) {
|
|
1380
2855
|
const first = mayBeEmptyIn[0];
|
|
1381
2856
|
const what = typeof first !== "string" ? "something that is not a name" : handles.has(first) ? `"${first}", a stored operand` : `"${first}", which is not an operand of this call`;
|
|
1382
|
-
throw new
|
|
2857
|
+
throw new ValidationError6(
|
|
1383
2858
|
`materializeMany: mayBeEmpty names ${what}; it names fed operands, and this call has no feed`
|
|
1384
2859
|
);
|
|
1385
2860
|
}
|
|
@@ -1389,13 +2864,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1389
2864
|
const destKeys = /* @__PURE__ */ new Set();
|
|
1390
2865
|
outputsIn.forEach((raw, i) => {
|
|
1391
2866
|
if (typeof raw !== "object" || raw === null) {
|
|
1392
|
-
throw new
|
|
2867
|
+
throw new ValidationError6(`outputs[${i}] must be an object`);
|
|
1393
2868
|
}
|
|
1394
2869
|
const bad = Object.keys(raw).filter(
|
|
1395
2870
|
(k) => !MATERIALIZE_MANY_OUTPUT_KEYS.includes(k)
|
|
1396
2871
|
);
|
|
1397
2872
|
if (bad.length > 0) {
|
|
1398
|
-
throw new
|
|
2873
|
+
throw new ValidationError6(
|
|
1399
2874
|
`outputs[${i}]: unknown option${bad.length > 1 ? "s" : ""} ${bad.map((k) => `"${k}"`).join(", ")}`
|
|
1400
2875
|
);
|
|
1401
2876
|
}
|
|
@@ -1403,12 +2878,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1403
2878
|
const dest = spec.dest;
|
|
1404
2879
|
assertOneStore("materializeMany", `outputs[${i}].dest`, dest, this.origin);
|
|
1405
2880
|
if (operandKeys.has(dest.key())) {
|
|
1406
|
-
throw new
|
|
2881
|
+
throw new ValidationError6(
|
|
1407
2882
|
`outputs[${i}].dest is also an operand of the call: one call is one pass over operands that do not change under it, so chain two calls`
|
|
1408
2883
|
);
|
|
1409
2884
|
}
|
|
1410
2885
|
if (destKeys.has(dest.key())) {
|
|
1411
|
-
throw new
|
|
2886
|
+
throw new ValidationError6(
|
|
1412
2887
|
`outputs[${i}].dest names a segment another output of the call writes`
|
|
1413
2888
|
);
|
|
1414
2889
|
}
|
|
@@ -1481,13 +2956,13 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1481
2956
|
held.set(key, { at, handle });
|
|
1482
2957
|
} else if (prior.at !== void 0 && at !== void 0) {
|
|
1483
2958
|
if (!samePin(prior.at, at)) {
|
|
1484
|
-
throw new
|
|
2959
|
+
throw new ValidationError6(
|
|
1485
2960
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1486
2961
|
);
|
|
1487
2962
|
}
|
|
1488
2963
|
} else if (prior.at !== void 0 || at !== void 0) {
|
|
1489
2964
|
if (!pinOption) {
|
|
1490
|
-
throw new
|
|
2965
|
+
throw new ValidationError6(
|
|
1491
2966
|
`materializeMany: operand "${name}" holds a segment another operand holds, ${twoPins(prior.at, at)}: one call reads a segment at one generation`
|
|
1492
2967
|
);
|
|
1493
2968
|
}
|
|
@@ -1515,7 +2990,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1515
2990
|
const final = rebindCombineMany(compiled, operandSpecs(pins));
|
|
1516
2991
|
return runCombineMany(final, {
|
|
1517
2992
|
source,
|
|
1518
|
-
codec:
|
|
2993
|
+
codec: flagshipCodec(),
|
|
1519
2994
|
clock: this.clock,
|
|
1520
2995
|
...metricsOn ? { metrics: this.metrics } : {}
|
|
1521
2996
|
});
|
|
@@ -1575,7 +3050,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1575
3050
|
/** A call's feed with the store's erasure counter: read once here, checked by the combine before any record is read. */
|
|
1576
3051
|
feedOf(feed, mayBeEmpty, epochAt) {
|
|
1577
3052
|
if (typeof feed !== "object" || feed === null || Array.isArray(feed)) {
|
|
1578
|
-
throw new
|
|
3053
|
+
throw new ValidationError6(
|
|
1579
3054
|
"materializeMany: feed must be an object with names, records and counts"
|
|
1580
3055
|
);
|
|
1581
3056
|
}
|
|
@@ -1625,13 +3100,32 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1625
3100
|
* tax every load for a rare request; this admin scan is `O(registered segments)` and touches no hot path. Needs
|
|
1626
3101
|
* a storage backend (throws {@link UnsupportedError} otherwise).
|
|
1627
3102
|
*
|
|
3103
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
3104
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
3105
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
3106
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
3107
|
+
*
|
|
1628
3108
|
* **Current as of the registry row, not as of the reader's cache.** Each segment's resolved generation is
|
|
1629
3109
|
* compared with the row the scan listed, and a segment whose cached generation differs is re-resolved before the
|
|
1630
3110
|
* read, so a load or an erasure from another process shows at once — whatever `cache.genTtlMs` is, `0` included.
|
|
1631
3111
|
* The comparison is on the generation **and** the row's token, so a segment retired, purged and loaded again from
|
|
1632
3112
|
* generation 0 while this store held its old generation 0 is told apart too.
|
|
3113
|
+
*
|
|
3114
|
+
* **A destroyed segment is searched as {@link CloudRoaring.eraseSubject} searches it.** No read resolves a
|
|
3115
|
+
* tombstone, so the id is a member of none, but a cleartext `destroySegment` (`allowCleartext`), or a `dropSegment`
|
|
3116
|
+
* whose sweep missed an object a load was still writing, leaves objects anyone with bucket access can read. Each
|
|
3117
|
+
* object left under the tombstone is opened newest first until one holds the id, and the segment is then listed in
|
|
3118
|
+
* `tombstones`, never in `segments`. A crypto-shredded segment's own objects are sealed under a key no row holds and are
|
|
3119
|
+
* never a holder: it is listed only when an object left under it in the clear holds the id. The cost is one listing per destroyed segment in scope (1 request when nothing is left), and for each
|
|
3120
|
+
* object opened a tail read and, where its index lists the id's chunk, a chunk read (two tail reads for an encrypted
|
|
3121
|
+
* object, which never holds the id): 7 requests for a cleartext destroy with 3 objects, 21 for 10. The budget is
|
|
3122
|
+
* charged one unit for each object opened past the one the row names, as `eraseSubject` charges. A report that runs
|
|
3123
|
+
* out, or whose listing or read fails after the store's retry, rejects rather than under-reporting, so one leftover
|
|
3124
|
+
* object that cannot be opened makes every report over its scope reject until it is removed (`generations(ref)` lists
|
|
3125
|
+
* what is left, `dropSegment` deletes it). A report refused for budget while it searches a destroyed segment reports
|
|
3126
|
+
* `spent` as `limit + 1`, a lower bound.
|
|
1633
3127
|
*/
|
|
1634
|
-
async subjectReport(id, options
|
|
3128
|
+
async subjectReport(id, options) {
|
|
1635
3129
|
checkOptions(options, "subjectReport", CALL_KEYS.subjectReport, ["allNamespaces"]);
|
|
1636
3130
|
options ??= {};
|
|
1637
3131
|
const registry = this.requireRegistry("subjectReport");
|
|
@@ -1646,26 +3140,53 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1646
3140
|
budget,
|
|
1647
3141
|
"subjectReport"
|
|
1648
3142
|
);
|
|
3143
|
+
let opens = 0;
|
|
3144
|
+
const charge = () => {
|
|
3145
|
+
opens += 1;
|
|
3146
|
+
if (budget !== null && recs.length + opens > budget.maxRequests) {
|
|
3147
|
+
throw new BudgetExceededError(
|
|
3148
|
+
`subjectReport would fan out to more than ${budget.maxRequests} units, over the per-op budget: the search under a destroyed segment was abandoned at that point, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``,
|
|
3149
|
+
{
|
|
3150
|
+
option: "budget.maxRequests",
|
|
3151
|
+
limit: budget.maxRequests,
|
|
3152
|
+
spent: budget.maxRequests + 1
|
|
3153
|
+
}
|
|
3154
|
+
);
|
|
3155
|
+
}
|
|
3156
|
+
};
|
|
1649
3157
|
const membership = await mapWithConcurrency(
|
|
1650
3158
|
recs,
|
|
1651
3159
|
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
1652
3160
|
async (rec) => {
|
|
1653
|
-
if (rec.status === "destroyed") return null;
|
|
1654
3161
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
3162
|
+
if (rec.status === "destroyed") {
|
|
3163
|
+
const deps = this.lifecycleDeps("subjectReport");
|
|
3164
|
+
const storage = budget === null ? deps.storage : new OpenChargingStorage(deps.storage, rec.currentGen, charge);
|
|
3165
|
+
const held = await findIdUnderTombstone(ref, id, { ...deps, storage });
|
|
3166
|
+
return held === null ? null : { tombstone: true, ref };
|
|
3167
|
+
}
|
|
1655
3168
|
if (this.crbmSource !== void 0) {
|
|
1656
3169
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1657
|
-
const
|
|
1658
|
-
if (rowVersionOfLive(held) !==
|
|
3170
|
+
const listed2 = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
3171
|
+
if (rowVersionOfLive(held) !== listed2) this.engine.invalidate(ref);
|
|
1659
3172
|
}
|
|
1660
|
-
return await this.engine.has(ref, id) ? {
|
|
3173
|
+
return await this.engine.has(ref, id) ? { tombstone: false, ref } : null;
|
|
1661
3174
|
}
|
|
1662
3175
|
);
|
|
1663
|
-
const
|
|
1664
|
-
|
|
3176
|
+
const listed = membership.filter((m) => m !== null);
|
|
3177
|
+
const segments = listed.filter((m) => !m.tombstone).map((m) => m.ref);
|
|
3178
|
+
const tombstones = listed.filter((m) => m.tombstone).map((m) => m.ref);
|
|
3179
|
+
return { id, segments, tombstones, scannedSegments: recs.length };
|
|
1665
3180
|
}
|
|
1666
3181
|
/**
|
|
1667
3182
|
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with the bit physically gone
|
|
1668
|
-
* from the bucket on return
|
|
3183
|
+
* from the bucket on return for each segment the ledger reports `erased: true`.** A noncurrent version, a replica
|
|
3184
|
+
* or a backup keeps the old object until its own lifecycle removes it.
|
|
3185
|
+
*
|
|
3186
|
+
* **Scope: one `namespace`, or `{ allNamespaces: true }`, never both** (both throws `ValidationError`, as does
|
|
3187
|
+
* neither). A segment loaded with no `namespace` is in no namespace you can name: only `{ allNamespaces: true }`
|
|
3188
|
+
* reaches it. `'_default'`, the label the export tool and the disaster-recovery samples print, and the bucket's
|
|
3189
|
+
* object keys use, for such a segment, is an ordinary namespace name here, and passing it scans a namespace of that name, not those segments.
|
|
1669
3190
|
*
|
|
1670
3191
|
* For each **registered** segment the id is a member of, `eraseIdFromSegment` rewrites the current generation
|
|
1671
3192
|
* without the id — every chunk streamed through, one bit cleared — publishes the rewrite **fenced on the
|
|
@@ -1730,15 +3251,17 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1730
3251
|
* `O(registered segments + superseded generations)`, no hot-path cost: a segment whose current generation lacks
|
|
1731
3252
|
* the id is searched generation by generation, one open for each other generation in its bucket, and with the
|
|
1732
3253
|
* keep-everything default of the `*Into` verbs that is one per generation the segment ever had. The per-op
|
|
1733
|
-
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names,
|
|
1734
|
-
*
|
|
3254
|
+
* `budget` is charged one unit for each segment and one for each generation opened beyond the one its row names, so
|
|
3255
|
+
* a fleet with long histories can exhaust it where the segment count alone would not. A generation is charged at its
|
|
3256
|
+
* first open only: a later open of it in the call, a retry of a failed read included, costs no more units. A segment
|
|
3257
|
+
* that exhausts the budget is
|
|
1735
3258
|
* reported `erased: false` with an `error:` note, before it deletes anything (the one refusal that can come after a
|
|
1736
3259
|
* rewrite is the last check for a generation a concurrent writer left behind), and the
|
|
1737
3260
|
* rest of the scan continues: the call itself does not throw for it. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
1738
3261
|
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
1739
3262
|
* is `dropSegment` / the `destroySegment`/`eraseNamespace` free functions.
|
|
1740
3263
|
*/
|
|
1741
|
-
async eraseSubject(id, options
|
|
3264
|
+
async eraseSubject(id, options) {
|
|
1742
3265
|
this.erasureEpoch++;
|
|
1743
3266
|
checkOptions(options, "eraseSubject", CALL_KEYS.eraseSubject, ["allNamespaces"]);
|
|
1744
3267
|
options ??= {};
|
|
@@ -1794,7 +3317,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1794
3317
|
segment: rec.segment,
|
|
1795
3318
|
namespace: rec.namespace,
|
|
1796
3319
|
erased: false,
|
|
1797
|
-
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
3320
|
+
note: `error: ${errorKind(err) ?? (err instanceof Error ? err.name : "Error")}: ${err instanceof Error ? err.message : String(err)}`
|
|
3321
|
+
// the class lets a note be matched to the guide's table
|
|
1798
3322
|
};
|
|
1799
3323
|
} finally {
|
|
1800
3324
|
try {
|
|
@@ -1815,11 +3339,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1815
3339
|
* choose from, which is the reason to look at it. One registry read and one listing; it does not open the objects.
|
|
1816
3340
|
* Each entry says whether the row knows the generation (`known`): a known generation needs no `allowUnknown`; an unknown
|
|
1817
3341
|
* one does, except below a row's pointer when the row has no usable record or the target is older than a full one reaches.
|
|
3342
|
+
* No entry is `current` on a destroyed segment: its row keeps the pointer it had, but no read resolves through it.
|
|
1818
3343
|
*
|
|
1819
3344
|
* Needs a backend.
|
|
1820
3345
|
*/
|
|
1821
3346
|
async generations(ref) {
|
|
1822
|
-
validateRef(ref);
|
|
3347
|
+
ref = validateRef(ref, "generations");
|
|
1823
3348
|
return listGenerations(ref, this.lifecycleDeps("generations"));
|
|
1824
3349
|
}
|
|
1825
3350
|
/**
|
|
@@ -1849,7 +3374,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1849
3374
|
* ```
|
|
1850
3375
|
*/
|
|
1851
3376
|
async exists(ref) {
|
|
1852
|
-
validateRef(ref);
|
|
3377
|
+
ref = validateRef(ref, "exists");
|
|
1853
3378
|
return segmentExists(ref, this.requireRegistry("exists"));
|
|
1854
3379
|
}
|
|
1855
3380
|
/**
|
|
@@ -1906,8 +3431,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1906
3431
|
* {@link ValidationError} before any request to storage, and a known one whose object carries another checksum than
|
|
1907
3432
|
* the recorded one throws {@link IntegrityError}, with or without a keystore, since the footer is in the clear. A
|
|
1908
3433
|
* generation not in the bucket (collected, or never written) throws `NotFoundError` naming what *is* available, a
|
|
1909
|
-
* crypto-shredded
|
|
1910
|
-
*
|
|
3434
|
+
* destroyed segment (crypto-shredded, destroyed in the clear or dropped) throws {@link ValidationError}, because no
|
|
3435
|
+
* generation resolves through a destroyed row, and a target that is cleartext under an encrypted segment, or
|
|
3436
|
+
* encrypted under a cleartext one, throws {@link IntegrityError} from
|
|
1911
3437
|
* one tail read of it, because every read would refuse it. Rolling to the generation already current is a no-op that
|
|
1912
3438
|
* reports itself, once one footer read finds its object in the bucket (and the one the row recorded); when the object is
|
|
1913
3439
|
* gone it throws `NotFoundError` naming what is.
|
|
@@ -1918,8 +3444,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1918
3444
|
* key (it holds none of the key's KEKs, one it holds does not unwrap it, or it throws `TransientError` or times out
|
|
1919
3445
|
* as `AbortSignal.timeout()` and the AWS SDK report a timeout), a rollback onto a known generation still happens,
|
|
1920
3446
|
* opening nothing, and the row carries no summary of the target. Any other failure of the keystore is
|
|
1921
|
-
* thrown. A swap of the pointer that gets no answer is settled by reading the row: one
|
|
1922
|
-
*
|
|
3447
|
+
* thrown. A swap of the pointer that gets no answer, or that meets a conflict, is settled by reading the row: one
|
|
3448
|
+
* that landed, the row's pointer being at the target over a write this call sent (the write token, on a registry that
|
|
3449
|
+
* reports them), is checked as any is, and otherwise its `TransientError` is thrown, or a `WriteConflictError` after
|
|
3450
|
+
* a conflict, with no move recorded. After the swap the target's
|
|
1923
3451
|
* footer is read once more, and the pointer put back if the object is gone or is another.
|
|
1924
3452
|
*
|
|
1925
3453
|
* It deletes nothing. The generations above the new pointer stay put — which is what makes this reversible —
|
|
@@ -1949,7 +3477,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1949
3477
|
* Needs a backend.
|
|
1950
3478
|
*/
|
|
1951
3479
|
async rollback(ref, toGeneration, options = {}) {
|
|
1952
|
-
validateRef(ref);
|
|
3480
|
+
ref = validateRef(ref, "rollback");
|
|
1953
3481
|
checkOptions(options, "rollback", CALL_KEYS.rollback, ["allowUnknown"]);
|
|
1954
3482
|
options ??= {};
|
|
1955
3483
|
const deps = this.lifecycleDeps("rollback");
|
|
@@ -1965,9 +3493,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1965
3493
|
* `input` is ids (any sync or async iterable of integers in `[0, 2^32)`), or a whole bitmap: `{ bitmap }`, anything
|
|
1966
3494
|
* with `serialize('portable')` such as `roaring`'s `RoaringBitmap32`, or `{ serialized }`, portable Roaring bytes.
|
|
1967
3495
|
* A bitmap is checked (size cap, structure, safe deserializer) before the first request, written from its own
|
|
1968
|
-
* containers with no per-id work, and gives the generation chunk for chunk what its ids would.
|
|
1969
|
-
* `
|
|
1970
|
-
* `
|
|
3496
|
+
* containers with no per-id work, and gives the generation chunk for chunk what its ids would. An
|
|
3497
|
+
* iterable with a `serialize` method (a `RoaringBitmap32`, another copy's bitmap, or a bitmap `deserializePortable`
|
|
3498
|
+
* returned) passed as ids loads as `{ bitmap }` and is not iterated. A `Uint8Array` or `Buffer` passed as ids is
|
|
3499
|
+
* refused with `ValidationError`, since each byte would load as an id, even when it has a `serialize` of its own:
|
|
3500
|
+
* pass bytes as `{ serialized }`.
|
|
3501
|
+
*
|
|
3502
|
+
* Ids are held whole in memory before the object is written: the generation's bitmaps and a buffer of up to about 28
|
|
3503
|
+
* MB. Ids that already ascend can be loaded with `sorted: true` instead, which writes them one chunk at a time while
|
|
3504
|
+
* the object uploads, so the load holds one chunk and the writer's index (up to about 7 MiB, about 8 MiB for an
|
|
3505
|
+
* encrypted segment, derived), whatever the segment's size, and writes the same object ({@link LoadOptions.sorted}).
|
|
1971
3506
|
*
|
|
1972
3507
|
* The whole write path in one call: take the next generation number, write the object, check the result is
|
|
1973
3508
|
* plausible, move the pointer, collect what the move superseded. Composed by hand those are four functions and
|
|
@@ -2026,7 +3561,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2026
3561
|
* does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
|
|
2027
3562
|
* what the pass deleted by name, and one name may have been gone already.
|
|
2028
3563
|
*
|
|
2029
|
-
*
|
|
3564
|
+
* `input` cannot be a stream of this library (`iterate`, `everyNth`, `intersect`, `union`, `andNot`, as returned) that
|
|
3565
|
+
* read the segment being loaded, from this store or one built over the same registry driver: a load publishes over
|
|
3566
|
+
* whatever the segment is by then, so it would replace another writer's newer load with a result built from an older
|
|
3567
|
+
* one, and it throws {@link ValidationError}. Ids read from another store over another backend load as any ids do.
|
|
3568
|
+
* Two separate store instances over one bucket cannot be told apart, so this does not cover them; this release has no
|
|
3569
|
+
* fenced in-place update, so write the result into a new segment and point readers at it.
|
|
3570
|
+
*
|
|
3571
|
+
* What it **throws** for is a fault rather than an outcome: invalid options or ids, and a destroyed
|
|
2030
3572
|
* segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
|
|
2031
3573
|
* that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
|
|
2032
3574
|
* (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
|
|
@@ -2049,8 +3591,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2049
3591
|
* Needs a backend (throws {@link UnsupportedError} otherwise).
|
|
2050
3592
|
*/
|
|
2051
3593
|
async load(ref, input, options = {}) {
|
|
2052
|
-
validateRef(ref);
|
|
2053
|
-
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty"]);
|
|
3594
|
+
ref = validateRef(ref, "load");
|
|
3595
|
+
checkOptions(options, "load", CALL_KEYS.load, ["allowEmpty", "sorted"]);
|
|
3596
|
+
if (typeof input === "object" && input !== null) {
|
|
3597
|
+
const reads = STREAM_SOURCES.get(input);
|
|
3598
|
+
const sameBackend = reads !== void 0 && (reads.origin === this.origin || this.registry !== void 0 && REGISTRY_OF.get(reads.origin) === this.registry);
|
|
3599
|
+
if (sameBackend && reads.keys.includes(segmentKey(ref))) {
|
|
3600
|
+
const named = ref.namespace === void 0 ? ref.segment : `${ref.namespace}/${ref.segment}`;
|
|
3601
|
+
throw new ValidationError6(
|
|
3602
|
+
`load: the ids come from a read of the segment "${named}" that is being loaded, and a load publishes over whatever the segment is by then, so another process's newer load of it can be replaced by a result built from the one before, with no conflict reported. This release has no fenced in-place update: write the result into a new segment and point readers at it`
|
|
3603
|
+
);
|
|
3604
|
+
}
|
|
3605
|
+
}
|
|
2054
3606
|
options ??= {};
|
|
2055
3607
|
const deps = this.lifecycleDeps("load");
|
|
2056
3608
|
try {
|
|
@@ -2060,11 +3612,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2060
3612
|
}
|
|
2061
3613
|
}
|
|
2062
3614
|
/**
|
|
2063
|
-
* **Dispose of a segment — tombstone it, then delete its storage objects.**
|
|
3615
|
+
* **Dispose of a segment — tombstone it, then delete its storage objects.** It cannot be undone through the
|
|
3616
|
+
* library (a restore from the registry's and the bucket's versions or backups can).
|
|
2064
3617
|
*
|
|
2065
|
-
* The operation a rolling window needs: `destroySegment` crypto-shreds (
|
|
2066
|
-
*
|
|
2067
|
-
*
|
|
3618
|
+
* The operation a rolling window needs: `destroySegment` crypto-shreds (it removes the key from the segment's
|
|
3619
|
+
* current registry row, so the bytes are unreadable everywhere, backups included, once no retained copy of the row
|
|
3620
|
+
* holds the key or every KEK that wrapped it is destroyed; the objects still sit in your bucket and are still
|
|
3621
|
+
* billed, and it requires encryption), while this one removes the storage and works on a cleartext segment. On an
|
|
3622
|
+
* encrypted segment it does both.
|
|
2068
3623
|
*
|
|
2069
3624
|
* Pass `{ dryRun: true }` first — it reports the generations it *would* delete and changes nothing. That
|
|
2070
3625
|
* matters more than the `confirmSegment` guard for anything automated, because in a loop the guard is the same
|
|
@@ -2105,9 +3660,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2105
3660
|
* because it has to enumerate and delete generations — a pre-built `StorageChunkSource` only reads.
|
|
2106
3661
|
*/
|
|
2107
3662
|
async dropSegment(ref, options) {
|
|
2108
|
-
validateRef(ref);
|
|
3663
|
+
ref = validateRef(ref, "dropSegment");
|
|
2109
3664
|
if (options === void 0 || options === null) {
|
|
2110
|
-
throw new
|
|
3665
|
+
throw new ValidationError6("dropSegment: options { confirmSegment } are required");
|
|
2111
3666
|
}
|
|
2112
3667
|
checkOptions(options, "dropSegment", CALL_KEYS.dropSegment, ["dryRun"]);
|
|
2113
3668
|
const deps = this.lifecycleDeps("dropSegment");
|
|
@@ -2122,8 +3677,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2122
3677
|
}
|
|
2123
3678
|
}
|
|
2124
3679
|
/**
|
|
2125
|
-
* **Record when this segment becomes eligible for retirement.** One
|
|
2126
|
-
*
|
|
3680
|
+
* **Record when this segment becomes eligible for retirement.** One policy write, plus a best-effort write of its
|
|
3681
|
+
* due-index pointer (and removal of the old pointer when the expiry day moves); no data is deleted here, and
|
|
3682
|
+
* nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
2127
3683
|
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
2128
3684
|
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
2129
3685
|
*
|
|
@@ -2145,10 +3701,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2145
3701
|
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
2146
3702
|
* buckets is normal. A value below `1_000_000_000_000` (2001-09-09) is rejected: it is almost certainly epoch
|
|
2147
3703
|
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a storage
|
|
2148
|
-
* backend (throws {@link UnsupportedError} otherwise), and refuses a
|
|
3704
|
+
* backend (throws {@link UnsupportedError} otherwise), and refuses a destroyed segment.
|
|
2149
3705
|
*/
|
|
2150
3706
|
async setRetention(ref, policy) {
|
|
2151
|
-
validateRef(ref);
|
|
3707
|
+
ref = validateRef(ref, "setRetention");
|
|
2152
3708
|
return setSegmentRetention(
|
|
2153
3709
|
ref,
|
|
2154
3710
|
{ registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
|
|
@@ -2162,16 +3718,18 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2162
3718
|
* "never expires" on a segment someone believes is expiring.
|
|
2163
3719
|
*/
|
|
2164
3720
|
async getRetention(ref) {
|
|
2165
|
-
validateRef(ref);
|
|
3721
|
+
ref = validateRef(ref, "getRetention");
|
|
2166
3722
|
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
2167
3723
|
}
|
|
2168
3724
|
/**
|
|
2169
|
-
* **Cancel a segment's expiry** so no sweep retires it. Returns whether
|
|
2170
|
-
* when there was none
|
|
3725
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether this call removed a policy (`false`
|
|
3726
|
+
* when there was none, and also when a write that got no answer cannot be shown to be this call's and the
|
|
3727
|
+
* row holds no expiry: the expiry is gone either way, and its due-index pointer is removed. A later write that set an
|
|
3728
|
+
* expiry is cleared too, and the answer is `true`). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
2171
3729
|
* to the setter is how a typo becomes a deletion.
|
|
2172
3730
|
*/
|
|
2173
3731
|
async clearRetention(ref) {
|
|
2174
|
-
validateRef(ref);
|
|
3732
|
+
ref = validateRef(ref, "clearRetention");
|
|
2175
3733
|
return clearSegmentRetention(ref, {
|
|
2176
3734
|
registry: this.requireRegistry("clearRetention"),
|
|
2177
3735
|
clock: this.clock,
|
|
@@ -2196,20 +3754,22 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2196
3754
|
*
|
|
2197
3755
|
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
2198
3756
|
* for (const e of swept.entries) {
|
|
2199
|
-
*
|
|
3757
|
+
* // invalid-policy / policy-changed / tombstone-not-empty / failed: …
|
|
3758
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`);
|
|
2200
3759
|
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
2201
3760
|
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
2202
3761
|
* }
|
|
2203
3762
|
* }
|
|
2204
|
-
* if (swept.limited) scheduleAnotherPassSoon(); // more
|
|
3763
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more work (retirements or purges) remains
|
|
2205
3764
|
* ```
|
|
2206
3765
|
*
|
|
2207
3766
|
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
2208
3767
|
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
2209
3768
|
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
2210
|
-
* `limit` (default 100) cut the pass short and more
|
|
2211
|
-
*
|
|
2212
|
-
*
|
|
3769
|
+
* `limit` (default 100) cut the pass short, and more work (retirements or purges) remains: `limit` caps the
|
|
3770
|
+
* retirements and tombstone purges one call attempts, together, and a segment it deferred gets no entry. That cap
|
|
3771
|
+
* is charged on **attempts**, not successes, which is what makes it a real bound: `dropSegment` writes the
|
|
3772
|
+
* tombstone before sweeping Storage, so a fault in the Storage phase is a segment that is already retired. Retirements are sequential, so `limit` is a
|
|
2213
3773
|
* wall-clock knob too, and `retired` counts deletions only — a dry run reports `wouldRetire` instead, so a
|
|
2214
3774
|
* dashboard summing `retired` can never show a phantom deletion.
|
|
2215
3775
|
*
|
|
@@ -2219,9 +3779,11 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2219
3779
|
* charged to `limit`, so it never holds the retirements behind it, and purging stops for the rest of the call after
|
|
2220
3780
|
* three refused purges in a row (a purge that succeeds starts the count again).
|
|
2221
3781
|
*
|
|
2222
|
-
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
3782
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h; at least one hour, else `ValidationError`)
|
|
2223
3783
|
* and only once that segment's Storage generations are provably gone — collecting a straggler generation itself
|
|
2224
|
-
* first, since nothing else
|
|
3784
|
+
* first, since nothing else runs that collection on its own for a tombstoned segment (an erasure collects there
|
|
3785
|
+
* only
|
|
3786
|
+
* when an object holds its id, and a re-run drop sweeps it). Attribution is a **positive marker the sweep
|
|
2225
3787
|
* stamps on its own retirements**, not an inference from "destroyed + an expired policy": a crypto-shred leaves
|
|
2226
3788
|
* `retention` untouched, so setting a policy and then honouring a right-to-erasure request mid-window produces
|
|
2227
3789
|
* exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the name. By default a
|
|
@@ -2242,7 +3804,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2242
3804
|
const deps = this.lifecycleDeps("retireExpired");
|
|
2243
3805
|
const result = await retireExpired(
|
|
2244
3806
|
{ registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
|
|
2245
|
-
{ ...opts, now: opts.now
|
|
3807
|
+
{ ...opts, now: opts.now === void 0 ? this.clock.now() : opts.now }
|
|
2246
3808
|
);
|
|
2247
3809
|
if (opts.dryRun !== true) {
|
|
2248
3810
|
for (const entry of result.entries) {
|
|
@@ -2289,7 +3851,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2289
3851
|
* ```
|
|
2290
3852
|
*/
|
|
2291
3853
|
invalidate(ref) {
|
|
2292
|
-
validateRef(ref);
|
|
3854
|
+
ref = validateRef(ref, "invalidate");
|
|
2293
3855
|
refuseReservedNamespace(ref.namespace);
|
|
2294
3856
|
this.engine.invalidate(ref);
|
|
2295
3857
|
}
|
|
@@ -2349,11 +3911,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2349
3911
|
if (hold !== void 0) await hold.release().catch(() => void 0);
|
|
2350
3912
|
throw err;
|
|
2351
3913
|
}
|
|
2352
|
-
const pinnedAt = {
|
|
2353
|
-
generation: at?.generation ?? null,
|
|
2354
|
-
version: at?.version ?? null,
|
|
2355
|
-
fingerprint: at?.fingerprint ?? null
|
|
2356
|
-
};
|
|
3914
|
+
const pinnedAt = at === null ? { generation: null, version: null, fingerprint: null } : { generation: at.generation, version: at.version, fingerprint: at.fingerprint };
|
|
2357
3915
|
const pins = /* @__PURE__ */ new Map([[segmentKey(ref), pinnedAt]]);
|
|
2358
3916
|
return makeSegment({
|
|
2359
3917
|
engine: this.engineWithPins(pins),
|
|
@@ -2393,7 +3951,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2393
3951
|
return new SegmentEngine({
|
|
2394
3952
|
storage: this.retryOptions === void 0 ? pinned : new RetryingStorageChunkSource2(pinned, this.retryOptions),
|
|
2395
3953
|
cache: this.cache,
|
|
2396
|
-
codec:
|
|
3954
|
+
codec: flagshipCodec(),
|
|
2397
3955
|
clock: this.clock,
|
|
2398
3956
|
metrics: this.metrics,
|
|
2399
3957
|
budget: this.budget,
|
|
@@ -2423,8 +3981,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2423
3981
|
if (held.has(key)) {
|
|
2424
3982
|
const before = held.get(key);
|
|
2425
3983
|
if (!samePin(before, at)) {
|
|
2426
|
-
throw new
|
|
2427
|
-
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`
|
|
3984
|
+
throw new ValidationError6(
|
|
3985
|
+
`the same segment is in this combine twice, ${twoPins(before, at)}: one call reads a segment at one generation, so materialise one of them into a segment of its own first, with \`materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })\``
|
|
2428
3986
|
);
|
|
2429
3987
|
}
|
|
2430
3988
|
} else {
|
|
@@ -2455,8 +4013,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2455
4013
|
* throw). Read-only, bounded fan-out; run it at startup after a restore. Returns `{ checked, inconsistent,
|
|
2456
4014
|
* errored }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to recover (restore the object
|
|
2457
4015
|
* store, or roll the registry back to a coherent point). Needs the store built with a **backend**
|
|
2458
|
-
* (throws {@link UnsupportedError} otherwise). `destroyed` (crypto-shredded
|
|
2459
|
-
*
|
|
4016
|
+
* (throws {@link UnsupportedError} otherwise). `destroyed` segments (crypto-shredded, destroyed in the clear or
|
|
4017
|
+
* dropped) are skipped. Pair
|
|
4018
|
+
* it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
2460
4019
|
*
|
|
2461
4020
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2462
4021
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
@@ -2489,6 +4048,56 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2489
4048
|
options
|
|
2490
4049
|
);
|
|
2491
4050
|
}
|
|
4051
|
+
/**
|
|
4052
|
+
* **First-run check of the store's storage and registry.** Call it once at deploy time, or on a first run, before
|
|
4053
|
+
* the first load: it finds a missing permission, a storage that does not refuse a second write to a key, a listing
|
|
4054
|
+
* that lags, or a lifecycle rule that would delete the store's objects, while nothing depends on them yet.
|
|
4055
|
+
*
|
|
4056
|
+
* It runs seven checks and always lists all seven, in this order: `registry` (one read of a row that must be
|
|
4057
|
+
* absent, and the registry able to write), `write` (a new object), `write-once` (a second write to the same key must
|
|
4058
|
+
* be refused), `read` (the object reads back as written), `list` (the listing names it), `delete` (this run's own
|
|
4059
|
+
* object is removed, with the version the read gave where the driver applies one) and `lifecycle` (the backend's
|
|
4060
|
+
* lifecycle check). The probe is one small object at `cbm.due.preflight/segments/probe.<number>.crbm` under the
|
|
4061
|
+
* store's prefix; no registry row is written, and no segment of yours can share the name. A probe left by a run that
|
|
4062
|
+
* was cut off is named in the `list` detail and never deleted, since it could be a run in progress.
|
|
4063
|
+
*
|
|
4064
|
+
* It **never throws over a finding**. A failed step, a transient fault included, is a `problem` whose `detail`
|
|
4065
|
+
* says to run again when the fault was transient, and the checks that cannot run without it are `unchecked`. The
|
|
4066
|
+
* exception is the `lifecycle` step, whose failures, its timeout included, are `unchecked`. `ok` is true when no
|
|
4067
|
+
* check is a `problem`: read an `unchecked` check yourself, as `ok` does not vouch for it. The `lifecycle` check is
|
|
4068
|
+
* `unchecked` for a backend whose check says so (Azure Blob always), for a backend that has no such method (a
|
|
4069
|
+
* third-party backend, a `brandAsBackend` wrapper that does not forward it), when the identity lacks the optional
|
|
4070
|
+
* permission to read the lifecycle rules (`s3:GetLifecycleConfiguration`, `storage.buckets.get`), and when the check
|
|
4071
|
+
* takes longer than 30 seconds of the store's clock. The other steps are the store's ordinary calls and keep their
|
|
4072
|
+
* timing.
|
|
4073
|
+
*
|
|
4074
|
+
* **Not checked:** read-after-delete, the registry's `If-Match` on compare-and-swap, multipart abort permission,
|
|
4075
|
+
* versioning, backups, the keystore's backup and an S3 SDK from 3.641.0 to 3.699.0 (an older SDK shows as a
|
|
4076
|
+
* `write-once` problem; these drop `If-Match`, which only a registry write refuses). It makes no kind of request that a
|
|
4077
|
+
* load does not, though the lifecycle read needs its own optional permission. A run costs about seven requests (a
|
|
4078
|
+
* first run on GCS makes eight, the eighth a second listing that checks the bucket exists, and Azure Blob always
|
|
4079
|
+
* eight): two writes and a delete, the rest reads and one listing.
|
|
4080
|
+
*
|
|
4081
|
+
* Throws {@link UnsupportedError}, and only that, for a store with no backend, since there is nothing to check.
|
|
4082
|
+
*/
|
|
4083
|
+
async preflight() {
|
|
4084
|
+
const deps = this.lifecycleDeps("preflight");
|
|
4085
|
+
const backend = this.backend;
|
|
4086
|
+
let checkLifecycle;
|
|
4087
|
+
try {
|
|
4088
|
+
const method = backend?.checkLifecycle;
|
|
4089
|
+
if (typeof method === "function")
|
|
4090
|
+
checkLifecycle = method.bind(backend);
|
|
4091
|
+
} catch {
|
|
4092
|
+
}
|
|
4093
|
+
return runPreflight({
|
|
4094
|
+
storage: deps.storage,
|
|
4095
|
+
registry: deps.registry,
|
|
4096
|
+
clock: this.clock,
|
|
4097
|
+
entropy: this.entropy,
|
|
4098
|
+
checkLifecycle
|
|
4099
|
+
});
|
|
4100
|
+
}
|
|
2492
4101
|
/**
|
|
2493
4102
|
* **Export ("eject") every registered segment's current generation** through the injected `sink`, using only
|
|
2494
4103
|
* public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's "Your
|
|
@@ -2497,15 +4106,16 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2497
4106
|
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
2498
4107
|
* enumeration and the read path provably share one registry. Encrypted segments are decrypted transparently
|
|
2499
4108
|
* **iff** this store was wired with their keystore — the export is therefore **cleartext**; protect it.
|
|
2500
|
-
*
|
|
2501
|
-
* (the run continues), so "a manifest exists" means the run finished —
|
|
4109
|
+
* Destroyed segments (crypto-shredded, destroyed in the clear or dropped) are skipped. A segment that can't be read
|
|
4110
|
+
* is isolated into the manifest's `failed[]` (the run continues), so "a manifest exists" means the run finished —
|
|
4111
|
+
* check `failed`. The `export-segments`
|
|
2502
4112
|
* CLI wraps this with a filesystem sink. The `'roaring'` format's bitmaps are built by the store's own codec.
|
|
2503
4113
|
*/
|
|
2504
4114
|
async exportSegments(sink, options = {}) {
|
|
2505
4115
|
checkOptions(options, "exportSegments", CALL_KEYS.exportSegments, [], MISPLACED.exportSegments);
|
|
2506
4116
|
const opts = picked(options, CALL_KEYS.exportSegments);
|
|
2507
4117
|
const registry = this.requireRegistry("exportSegments");
|
|
2508
|
-
return runExport(this, registry, sink, { ...opts, codec:
|
|
4118
|
+
return runExport(this, registry, sink, { ...opts, codec: flagshipCodec() });
|
|
2509
4119
|
}
|
|
2510
4120
|
};
|
|
2511
4121
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
@@ -2533,12 +4143,23 @@ var readOptions = (options) => ({
|
|
|
2533
4143
|
});
|
|
2534
4144
|
var NO_SEGMENTS = [];
|
|
2535
4145
|
function segmentList(list, what) {
|
|
2536
|
-
if (!Array.isArray(list)) throw new
|
|
4146
|
+
if (!Array.isArray(list)) throw new ValidationError6(`${what} must be an array of segments`);
|
|
2537
4147
|
return list;
|
|
2538
4148
|
}
|
|
4149
|
+
var STREAM_SOURCES = /* @__PURE__ */ new WeakMap();
|
|
4150
|
+
var REGISTRY_OF = /* @__PURE__ */ new WeakMap();
|
|
4151
|
+
function readsFrom(stream, origin, keys) {
|
|
4152
|
+
STREAM_SOURCES.set(stream, { origin, keys });
|
|
4153
|
+
return stream;
|
|
4154
|
+
}
|
|
2539
4155
|
function operandsLabel(verb) {
|
|
2540
4156
|
return verb === "andNot" ? "andNot: `excludes`" : `${verb}: \`others\``;
|
|
2541
4157
|
}
|
|
4158
|
+
function refuseNoOperands(verb) {
|
|
4159
|
+
throw new ValidationError6(
|
|
4160
|
+
verb === "intersect" ? "intersect requires at least one segment to intersect with; to read every id of one segment, use iterate()" : "intersectInto requires at least one segment to intersect with; to copy this segment into dest, call materializeMany({ operands: { src: thisSegment }, outputs: [{ dest, expr: { or: ['src'] } }], keep })"
|
|
4161
|
+
);
|
|
4162
|
+
}
|
|
2542
4163
|
var AS_IDS = {
|
|
2543
4164
|
failing,
|
|
2544
4165
|
guard: guardIds,
|
|
@@ -2555,9 +4176,9 @@ var CombineChunks = class {
|
|
|
2555
4176
|
var AS_CHUNKS = {
|
|
2556
4177
|
guard: guardItems,
|
|
2557
4178
|
failing: (err) => refusing(err)(),
|
|
2558
|
-
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
|
|
2559
|
-
union: (engine, refs, opts) => engine.unionChunks(refs, opts),
|
|
2560
|
-
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
|
|
4179
|
+
intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts, "intersectInto"),
|
|
4180
|
+
union: (engine, refs, opts) => engine.unionChunks(refs, opts, "unionInto"),
|
|
4181
|
+
andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts, "andNotInto")
|
|
2561
4182
|
};
|
|
2562
4183
|
var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
|
|
2563
4184
|
async function timeOp(metrics, clock, name, fn) {
|
|
@@ -2616,7 +4237,7 @@ var Segment = class _Segment {
|
|
|
2616
4237
|
*/
|
|
2617
4238
|
constructor(parts) {
|
|
2618
4239
|
if (!minting) {
|
|
2619
|
-
throw new
|
|
4240
|
+
throw new ValidationError6(
|
|
2620
4241
|
"a Segment is not constructed directly; call `store.segment(name)`"
|
|
2621
4242
|
);
|
|
2622
4243
|
}
|
|
@@ -2655,7 +4276,7 @@ var Segment = class _Segment {
|
|
|
2655
4276
|
*
|
|
2656
4277
|
* **One call reads a segment at one generation**, so a combine that holds this segment at two —
|
|
2657
4278
|
* `snap0.andNot([snap1])`, or `live.intersect([snap])` — throws {@link ValidationError} when it is read, and an
|
|
2658
|
-
* `*Into` of one throws before it reads anything. Materialise one side first, with `
|
|
4279
|
+
* `*Into` of one throws before it reads anything. Materialise one side first, with `materializeMany({ operands: { src: seg }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`.
|
|
2659
4280
|
*
|
|
2660
4281
|
* **A pin knows its object, not only its number.** A name purged and loaded again starts again at generation
|
|
2661
4282
|
* 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
|
|
@@ -2730,11 +4351,11 @@ var Segment = class _Segment {
|
|
|
2730
4351
|
leaseUntilOf(op, options) {
|
|
2731
4352
|
if (options === void 0 || options === null) return void 0;
|
|
2732
4353
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
2733
|
-
throw new
|
|
4354
|
+
throw new ValidationError6(`${op}: options must be an object such as { leaseUntil }`);
|
|
2734
4355
|
}
|
|
2735
4356
|
const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
|
|
2736
4357
|
if (unknown.length > 0) {
|
|
2737
|
-
throw new
|
|
4358
|
+
throw new ValidationError6(
|
|
2738
4359
|
`${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
|
|
2739
4360
|
);
|
|
2740
4361
|
}
|
|
@@ -2742,17 +4363,17 @@ var Segment = class _Segment {
|
|
|
2742
4363
|
if (until === void 0) return void 0;
|
|
2743
4364
|
const now = this.clock.now();
|
|
2744
4365
|
if (typeof until !== "number" || !Number.isSafeInteger(until)) {
|
|
2745
|
-
throw new
|
|
4366
|
+
throw new ValidationError6(
|
|
2746
4367
|
`${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
|
|
2747
4368
|
);
|
|
2748
4369
|
}
|
|
2749
4370
|
if (until <= now) {
|
|
2750
|
-
throw new
|
|
4371
|
+
throw new ValidationError6(
|
|
2751
4372
|
`${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
|
|
2752
4373
|
);
|
|
2753
4374
|
}
|
|
2754
4375
|
if (until > now + MAX_LEASE_MS) {
|
|
2755
|
-
throw new
|
|
4376
|
+
throw new ValidationError6(
|
|
2756
4377
|
`${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
|
|
2757
4378
|
);
|
|
2758
4379
|
}
|
|
@@ -2830,10 +4451,10 @@ var Segment = class _Segment {
|
|
|
2830
4451
|
* `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
|
|
2831
4452
|
* from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
|
|
2832
4453
|
* name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
|
|
2833
|
-
* than `generation` and `
|
|
4454
|
+
* than `generation`, `fingerprint` and `version` is a {@link ValidationError}. Identify a pin by its `generation` and
|
|
2834
4455
|
* `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
|
|
2835
4456
|
*
|
|
2836
|
-
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a
|
|
4457
|
+
* Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a destroyed segment),
|
|
2837
4458
|
* is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
|
|
2838
4459
|
* deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
|
|
2839
4460
|
* reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
|
|
@@ -2846,18 +4467,32 @@ var Segment = class _Segment {
|
|
|
2846
4467
|
* then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
|
|
2847
4468
|
* swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
|
|
2848
4469
|
* is in `options`, the second.
|
|
4470
|
+
*
|
|
4471
|
+
* It takes what {@link Segment.pinnedAt} holds: once `generation` is not `null`, the record passes as it is (its
|
|
4472
|
+
* `version` is not read), and `{ generation, fingerprint }` does as well. A pin of a segment that had no generation
|
|
4473
|
+
* has nothing to reopen and throws {@link ValidationError}.
|
|
2849
4474
|
*/
|
|
2850
4475
|
async pinAt(at, options) {
|
|
2851
4476
|
this.assertLease();
|
|
2852
|
-
const { generation, fingerprint } = at ?? {};
|
|
2853
|
-
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) =>
|
|
4477
|
+
const { generation, fingerprint, version } = at ?? {};
|
|
4478
|
+
const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => !["generation", "fingerprint", "version"].includes(k)) : [];
|
|
2854
4479
|
if (unknown.length > 0) {
|
|
2855
|
-
throw new
|
|
2856
|
-
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")};
|
|
4480
|
+
throw new ValidationError6(
|
|
4481
|
+
`pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; pinAt takes { generation, fingerprint } and a pin's \`version\`, which it does not read`
|
|
4482
|
+
);
|
|
4483
|
+
}
|
|
4484
|
+
if (generation === null) {
|
|
4485
|
+
throw new ValidationError6(
|
|
4486
|
+
"pinAt: the pin named no generation (the segment had none when it was pinned), so there is nothing to reopen"
|
|
4487
|
+
);
|
|
4488
|
+
}
|
|
4489
|
+
if (version !== void 0 && typeof version !== "string") {
|
|
4490
|
+
throw new ValidationError6(
|
|
4491
|
+
"pinAt: `version`, when given, is the string a pin recorded in its pinnedAt; pinAt does not read it, so leave it out if you have none"
|
|
2857
4492
|
);
|
|
2858
4493
|
}
|
|
2859
4494
|
if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
|
|
2860
|
-
throw new
|
|
4495
|
+
throw new ValidationError6(
|
|
2861
4496
|
"pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
|
|
2862
4497
|
);
|
|
2863
4498
|
}
|
|
@@ -2874,9 +4509,11 @@ var Segment = class _Segment {
|
|
|
2874
4509
|
return segmentKey(this.ref);
|
|
2875
4510
|
}
|
|
2876
4511
|
/**
|
|
2877
|
-
* Refuse an `*Into`
|
|
2878
|
-
* segment of another store
|
|
2879
|
-
*
|
|
4512
|
+
* Refuse an `*Into` that cannot run, before any request, in this order: anything that is not a segment or is a
|
|
4513
|
+
* segment of another store, an empty operand list for `intersectInto`, and a `dest` that is also a source (all
|
|
4514
|
+
* {@link ValidationError}); then a handle (this one, `dest`, an operand or an exclude) whose lease has ended or was
|
|
4515
|
+
* released ({@link LeaseExpiredError}). A call with both a lapsed lease and one of the first three refusals throws the
|
|
4516
|
+
* first. Called by the three `*Into` verbs only.
|
|
2880
4517
|
*
|
|
2881
4518
|
* The verbs are `async` so this surfaces as a **rejected promise**, like every other validation in the facade —
|
|
2882
4519
|
* a synchronous throw out of a promise-returning method escapes a caller who attached `.catch()` instead of
|
|
@@ -2886,8 +4523,28 @@ var Segment = class _Segment {
|
|
|
2886
4523
|
this.assertSegments(verb, "the destination", [dest]);
|
|
2887
4524
|
this.assertSegments(verb, verb === "andNotInto" ? "an exclude" : "an operand", operands);
|
|
2888
4525
|
this.assertSegments(verb, "an exclude", excludes);
|
|
4526
|
+
const base = verb.slice(0, -"Into".length);
|
|
4527
|
+
if (base === "intersect" && operands.length === 0) refuseNoOperands("intersectInto");
|
|
4528
|
+
this.refuseSourceDestination(verb, dest, operands, excludes);
|
|
2889
4529
|
this.assertLeases([this, dest, ...operands, ...excludes]);
|
|
2890
4530
|
}
|
|
4531
|
+
/**
|
|
4532
|
+
* Refuse an `*Into` whose destination is also one of its sources (this segment, an operand or an exclude, by the key
|
|
4533
|
+
* a handle gives, so a pin of the source and a ref spelled with the default namespace count), before any request, as
|
|
4534
|
+
* `materializeMany` refuses it. The combine would read the destination at the generation this store last resolved,
|
|
4535
|
+
* up to `cache.genTtlMs` old, while the publish is fenced on the row as it stands: another process's newer load of
|
|
4536
|
+
* the destination would be overwritten by a result built from the one before it, with no conflict to report.
|
|
4537
|
+
*/
|
|
4538
|
+
refuseSourceDestination(verb, dest, operands, excludes) {
|
|
4539
|
+
const key = dest.key();
|
|
4540
|
+
const role = this.key() === key ? "this segment" : operands.some((o) => o.key() === key) ? verb === "andNotInto" ? "an exclude" : "an operand" : excludes.some((o) => o.key() === key) ? "an exclude" : void 0;
|
|
4541
|
+
if (role === void 0) return;
|
|
4542
|
+
const { namespace, segment } = dest.ref;
|
|
4543
|
+
const named = namespace === void 0 ? segment : `${namespace}/${segment}`;
|
|
4544
|
+
throw new ValidationError6(
|
|
4545
|
+
`${verb}: the destination "${named}" is also ${role} of the call, and a call that reads a segment while it publishes to it can replace a newer load of it with a result built from an older one, with no conflict reported. This release has no fenced in-place update: combine into a new segment and point readers at it`
|
|
4546
|
+
);
|
|
4547
|
+
}
|
|
2891
4548
|
/** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
|
|
2892
4549
|
guarded(out, handles, result) {
|
|
2893
4550
|
return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
|
|
@@ -2999,7 +4656,11 @@ var Segment = class _Segment {
|
|
|
2999
4656
|
engine.iterate(ref, bounds),
|
|
3000
4657
|
() => engine.iterateBatches(ref, bounds)
|
|
3001
4658
|
);
|
|
3002
|
-
return
|
|
4659
|
+
return readsFrom(
|
|
4660
|
+
this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLease()),
|
|
4661
|
+
this.origin,
|
|
4662
|
+
[this.key()]
|
|
4663
|
+
);
|
|
3003
4664
|
}
|
|
3004
4665
|
/**
|
|
3005
4666
|
* **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
|
|
@@ -3048,7 +4709,11 @@ var Segment = class _Segment {
|
|
|
3048
4709
|
n,
|
|
3049
4710
|
options == null ? void 0 : rangeOf(options)
|
|
3050
4711
|
);
|
|
3051
|
-
return
|
|
4712
|
+
return readsFrom(
|
|
4713
|
+
this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLease()),
|
|
4714
|
+
this.origin,
|
|
4715
|
+
[this.key()]
|
|
4716
|
+
);
|
|
3052
4717
|
}
|
|
3053
4718
|
/**
|
|
3054
4719
|
* The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
|
|
@@ -3087,7 +4752,8 @@ var Segment = class _Segment {
|
|
|
3087
4752
|
*
|
|
3088
4753
|
* Of {@link CombineOptions} it takes the range, `concurrency`, `budget`, `allowAbsentOperands` and `exclude`; a write
|
|
3089
4754
|
* option (`audit`, `allowEmpty`, `guard`, `metadata`, `keep`) fails the stream with {@link ValidationError} when it is
|
|
3090
|
-
* first read, before any request.
|
|
4755
|
+
* first read, before any request. So does an empty `others`: an intersection over nothing has no meaning, and
|
|
4756
|
+
* to read every id of one segment, use {@link iterate}. (`union` and `andNot` read an empty list as `this`.)
|
|
3091
4757
|
*/
|
|
3092
4758
|
intersect(others, options) {
|
|
3093
4759
|
const bad = optionsError(
|
|
@@ -3119,6 +4785,15 @@ var Segment = class _Segment {
|
|
|
3119
4785
|
* materialisation that silently did not take effect is the one thing a caller cannot detect on its own.
|
|
3120
4786
|
*
|
|
3121
4787
|
* A call involving a handle whose lease has ended, or anything that is not a segment, is refused before any of this.
|
|
4788
|
+
* So is a `dest` that is also `this`, an operand or an exclude (the same segment by name and namespace, however the
|
|
4789
|
+
* handle was made), and, for `intersectInto` alone, an empty `others` (`unionInto` and `andNotInto` publish `this` for an
|
|
4790
|
+
* empty list): each throws {@link ValidationError} before any request. A destination that
|
|
4791
|
+
* is also a source would publish a result built from the generation this store last resolved over a newer load by
|
|
4792
|
+
* another process, with no conflict to report. This release has no fenced in-place update: write the result into a
|
|
4793
|
+
* new segment and point readers at it; a single writer that knows nothing else writes the segment may load the result
|
|
4794
|
+
* back, unfenced. To copy `src` into `dest`, call
|
|
4795
|
+
* `materializeMany({ operands: { src }, outputs: [{ dest, expr: { or: ['src'] } }], keep })`, which reads each chunk
|
|
4796
|
+
* once, at one instant.
|
|
3122
4797
|
* It takes every field of {@link CombineOptions}, as `unionInto` does.
|
|
3123
4798
|
*/
|
|
3124
4799
|
async intersectInto(dest, others, options) {
|
|
@@ -3133,7 +4808,8 @@ var Segment = class _Segment {
|
|
|
3133
4808
|
* `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
|
|
3134
4809
|
* same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
|
|
3135
4810
|
* deliberately. If you find yourself unioning the same segments on every read, materializing the combined
|
|
3136
|
-
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes
|
|
4811
|
+
* segment once (`unionInto`, or a load) is the cheaper shape. It takes the options {@link intersect} takes, and an
|
|
4812
|
+
* empty `others` reads `this` itself.
|
|
3137
4813
|
*/
|
|
3138
4814
|
union(others, options) {
|
|
3139
4815
|
const bad = optionsError(
|
|
@@ -3201,6 +4877,7 @@ var Segment = class _Segment {
|
|
|
3201
4877
|
try {
|
|
3202
4878
|
exclude = verb === "andNot" ? NO_SEGMENTS : this.excludesOf(options);
|
|
3203
4879
|
const others = segmentList(operands, operandsLabel(verb));
|
|
4880
|
+
if (verb === "intersect" && others.length === 0) refuseNoOperands("intersect");
|
|
3204
4881
|
handles = [this, ...others, ...exclude];
|
|
3205
4882
|
this.assertSegments(verb, verb === "andNot" ? "an exclude" : "an operand", others);
|
|
3206
4883
|
this.assertSegments(verb, "an exclude", exclude);
|
|
@@ -3214,17 +4891,21 @@ var Segment = class _Segment {
|
|
|
3214
4891
|
} catch (err) {
|
|
3215
4892
|
return out.failing(err);
|
|
3216
4893
|
}
|
|
4894
|
+
const keys = handles.map((h) => h.key());
|
|
4895
|
+
const known = (result) => out === AS_IDS ? readsFrom(result, this.origin, keys) : result;
|
|
3217
4896
|
if (verb === "andNot") {
|
|
3218
4897
|
const refs2 = operands.map((o) => o.ref);
|
|
3219
4898
|
const opts2 = options == null ? void 0 : readOptions(options);
|
|
3220
|
-
return this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2));
|
|
4899
|
+
return known(this.guarded(out, handles, out.andNot(engine, this.ref, refs2, opts2)));
|
|
3221
4900
|
}
|
|
3222
4901
|
const refs = [this.ref, ...operands.map((o) => o.ref)];
|
|
3223
4902
|
const opts = this.refsIn(options, exclude);
|
|
3224
|
-
return
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
4903
|
+
return known(
|
|
4904
|
+
this.guarded(
|
|
4905
|
+
out,
|
|
4906
|
+
handles,
|
|
4907
|
+
verb === "union" ? out.union(engine, refs, opts) : out.intersect(engine, refs, opts)
|
|
4908
|
+
)
|
|
3228
4909
|
);
|
|
3229
4910
|
}
|
|
3230
4911
|
/** The one body of the three `*Into` verbs, after each has checked its own options. */
|
|
@@ -3250,13 +4931,13 @@ var Segment = class _Segment {
|
|
|
3250
4931
|
}
|
|
3251
4932
|
};
|
|
3252
4933
|
export {
|
|
3253
|
-
BudgetExceededError,
|
|
4934
|
+
BudgetExceededError2 as BudgetExceededError,
|
|
3254
4935
|
CapabilityError,
|
|
3255
4936
|
CloudRoaring,
|
|
3256
4937
|
CloudRoaringError,
|
|
3257
4938
|
CountingMetricsSink,
|
|
3258
4939
|
InProcessKeystore,
|
|
3259
|
-
|
|
4940
|
+
IntegrityError4 as IntegrityError,
|
|
3260
4941
|
KeyUnavailableError,
|
|
3261
4942
|
LeaseExpiredError2 as LeaseExpiredError,
|
|
3262
4943
|
LeaseLimitError,
|
|
@@ -3268,12 +4949,12 @@ export {
|
|
|
3268
4949
|
StaleOperandError,
|
|
3269
4950
|
TransientError,
|
|
3270
4951
|
UnsupportedError3 as UnsupportedError,
|
|
3271
|
-
|
|
4952
|
+
ValidationError7 as ValidationError,
|
|
3272
4953
|
WriteConflictError2 as WriteConflictError,
|
|
3273
4954
|
deserializePortable,
|
|
3274
4955
|
destroySegment,
|
|
3275
4956
|
eraseNamespace,
|
|
3276
|
-
errorKind,
|
|
4957
|
+
errorKind2 as errorKind,
|
|
3277
4958
|
excludingReservedRows2 as excludingReservedRows,
|
|
3278
4959
|
readRetentionPolicy
|
|
3279
4960
|
};
|