@cloudbitmaps/core 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/chunk-window.d.ts +24 -0
- package/dist/core/codec.d.ts +11 -0
- package/dist/core/crbm/format.d.ts +11 -2
- package/dist/core/crbm/reader.d.ts +1 -1
- package/dist/core/crbm-storage-source.d.ts +1 -1
- package/dist/core/engine.d.ts +44 -3
- package/dist/core/erase-id.d.ts +3 -3
- package/dist/core/metrics.d.ts +7 -1
- package/dist/index.js +268 -121
- package/dist/index.js.map +4 -4
- package/dist/testing/fuzz-core.d.ts +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -102,6 +102,110 @@ async function collectWithinBudget(source, budget, op) {
|
|
|
102
102
|
return out;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
// src/core/chunk-window.ts
|
|
106
|
+
var ChunkWindow = class {
|
|
107
|
+
constructor(keys, fetch, max, ramp) {
|
|
108
|
+
this.keys = keys;
|
|
109
|
+
this.fetch = fetch;
|
|
110
|
+
this.max = max;
|
|
111
|
+
this.ramp = ramp;
|
|
112
|
+
}
|
|
113
|
+
keys;
|
|
114
|
+
fetch;
|
|
115
|
+
max;
|
|
116
|
+
ramp;
|
|
117
|
+
open = [];
|
|
118
|
+
launched = 0;
|
|
119
|
+
taken = 0;
|
|
120
|
+
/** The next read's result in key order, or the error that read raised. */
|
|
121
|
+
async take() {
|
|
122
|
+
const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
|
|
123
|
+
while (this.launched < this.keys.length && this.launched - this.taken < width) {
|
|
124
|
+
this.open.push(
|
|
125
|
+
this.fetch(this.keys[this.launched++]).then(
|
|
126
|
+
(value) => ({ value }),
|
|
127
|
+
(cause) => ({ value: void 0, error: { cause } })
|
|
128
|
+
)
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
const slot = await this.open.shift();
|
|
132
|
+
this.taken += 1;
|
|
133
|
+
if (slot.error) throw slot.error.cause;
|
|
134
|
+
return slot.value;
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
// src/core/crbm/format.ts
|
|
139
|
+
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
140
|
+
var VERSION_MAJOR = 1;
|
|
141
|
+
var VERSION_MINOR = 0;
|
|
142
|
+
var PREAMBLE_BYTES = 8;
|
|
143
|
+
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
144
|
+
var FOOTER_BYTES = 104;
|
|
145
|
+
var FOOTER = {
|
|
146
|
+
indexOffset: 0,
|
|
147
|
+
// u64
|
|
148
|
+
indexLength: 8,
|
|
149
|
+
// u64
|
|
150
|
+
indexCrc32c: 16,
|
|
151
|
+
// u32
|
|
152
|
+
flags: 20,
|
|
153
|
+
// u32
|
|
154
|
+
payloadCodecId: 24,
|
|
155
|
+
// u16 — which codec wrote the chunk payloads
|
|
156
|
+
elementWidth: 26,
|
|
157
|
+
// u8
|
|
158
|
+
containerCodec: 27,
|
|
159
|
+
// u8
|
|
160
|
+
versionMajor: 28,
|
|
161
|
+
// u8
|
|
162
|
+
versionMinor: 29,
|
|
163
|
+
// u8
|
|
164
|
+
reserved2: 30,
|
|
165
|
+
// u16
|
|
166
|
+
generation: 32,
|
|
167
|
+
// u64
|
|
168
|
+
indexNonce: 40,
|
|
169
|
+
// 12 B
|
|
170
|
+
indexTag: 52,
|
|
171
|
+
// 16 B
|
|
172
|
+
keyId: 68,
|
|
173
|
+
// 16 B
|
|
174
|
+
chunkCount: 84,
|
|
175
|
+
// u32
|
|
176
|
+
totalCardinality: 88,
|
|
177
|
+
// u64
|
|
178
|
+
footerCrc32c: 96,
|
|
179
|
+
// u32 — covers footer bytes [0, 96)
|
|
180
|
+
endMagic: 100
|
|
181
|
+
// char[4]
|
|
182
|
+
};
|
|
183
|
+
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
184
|
+
var FLAG_ENCRYPTED = 1 << 0;
|
|
185
|
+
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
186
|
+
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
187
|
+
var FLAG_EXTENSION = 1 << 3;
|
|
188
|
+
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
|
|
189
|
+
var ELEMENT_WIDTH_32 = 32;
|
|
190
|
+
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
191
|
+
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
192
|
+
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
193
|
+
]);
|
|
194
|
+
var CONTAINER_CODEC_NONE = 0;
|
|
195
|
+
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
196
|
+
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
197
|
+
var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
|
|
198
|
+
var MAX_CHUNK_CARDINALITY = 65536;
|
|
199
|
+
var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
|
|
200
|
+
var EXT_TRAILER_BYTES = 12;
|
|
201
|
+
var EXT_SECTION_HEADER_BYTES = 5;
|
|
202
|
+
var MAX_EXT_BYTES = 4 * 1024;
|
|
203
|
+
var EXT_SECTION_METADATA = 1;
|
|
204
|
+
var CRC32C_BYTES = 4;
|
|
205
|
+
var AEAD_NONCE_BYTES = 12;
|
|
206
|
+
var AEAD_TAG_BYTES = 16;
|
|
207
|
+
var DEFAULT_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BITMAP_BYTES + AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
|
|
208
|
+
|
|
105
209
|
// src/core/keys.ts
|
|
106
210
|
var FIELD = " ";
|
|
107
211
|
var NO_NAMESPACE = "/";
|
|
@@ -214,8 +318,8 @@ var CountingMetricsSink = class _CountingMetricsSink {
|
|
|
214
318
|
};
|
|
215
319
|
|
|
216
320
|
// src/core/engine.ts
|
|
217
|
-
var
|
|
218
|
-
var
|
|
321
|
+
var DEFAULT_INTERSECT_CONCURRENCY = 32;
|
|
322
|
+
var COMBINE_WINDOW_START = 8;
|
|
219
323
|
var ZERO_CLOCK = { now: () => 0 };
|
|
220
324
|
function windowOf(range) {
|
|
221
325
|
const after = range?.after;
|
|
@@ -268,37 +372,65 @@ function keysWithin(keys, w) {
|
|
|
268
372
|
};
|
|
269
373
|
return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
|
|
270
374
|
}
|
|
271
|
-
var
|
|
272
|
-
constructor(keys,
|
|
375
|
+
var CombineWindow = class {
|
|
376
|
+
constructor(keys, limit, read) {
|
|
273
377
|
this.keys = keys;
|
|
274
|
-
this.
|
|
275
|
-
this.
|
|
276
|
-
this.
|
|
378
|
+
this.limit = limit;
|
|
379
|
+
this.read = read;
|
|
380
|
+
this.fill();
|
|
277
381
|
}
|
|
278
382
|
keys;
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
launched = 0;
|
|
383
|
+
limit;
|
|
384
|
+
read;
|
|
385
|
+
inFlight = [];
|
|
386
|
+
next_ = 0;
|
|
284
387
|
taken = 0;
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
388
|
+
fill() {
|
|
389
|
+
const width = Math.min(
|
|
390
|
+
this.limit,
|
|
391
|
+
Math.max(COMBINE_WINDOW_START, 2 ** Math.min(this.taken, 30))
|
|
392
|
+
);
|
|
393
|
+
while (this.next_ < this.keys.length && this.inFlight.length < width) {
|
|
394
|
+
const key = this.keys[this.next_++];
|
|
395
|
+
this.inFlight.push(
|
|
396
|
+
this.read(key).then(
|
|
397
|
+
(result) => ({ key, result }),
|
|
398
|
+
(error) => ({ key, result: null, error })
|
|
293
399
|
)
|
|
294
400
|
);
|
|
295
401
|
}
|
|
296
|
-
|
|
402
|
+
}
|
|
403
|
+
/** The next key's chunk in ascending key order, `undefined` when the keys are used up; throws that key's error. */
|
|
404
|
+
async next() {
|
|
405
|
+
if (this.inFlight.length === 0) return void 0;
|
|
406
|
+
const slot = await this.inFlight.shift();
|
|
297
407
|
this.taken += 1;
|
|
298
|
-
|
|
299
|
-
|
|
408
|
+
this.fill();
|
|
409
|
+
if (slot.error !== void 0) throw slot.error;
|
|
410
|
+
return slot;
|
|
300
411
|
}
|
|
301
412
|
};
|
|
413
|
+
function chunkIds(chunk, chunkKey, w) {
|
|
414
|
+
let rem = chunk.toUint32Array ? chunk.toUint32Array() : Uint32Array.from(chunk);
|
|
415
|
+
if (w !== null && isEdge(chunkKey, w)) {
|
|
416
|
+
const from = chunkKey === w.loKey ? w.loRem : 0;
|
|
417
|
+
const to = chunkKey === w.hiKey ? w.hiRem : MAX_REMAINDER;
|
|
418
|
+
rem = rem.slice(firstAbove(rem, from - 1), firstAbove(rem, to));
|
|
419
|
+
}
|
|
420
|
+
const base = chunkKey * CHUNK_COUNT;
|
|
421
|
+
for (let i = 0; i < rem.length; i++) rem[i] = base + (rem[i] & MAX_REMAINDER);
|
|
422
|
+
return rem;
|
|
423
|
+
}
|
|
424
|
+
function firstAbove(a, value) {
|
|
425
|
+
let lo = 0;
|
|
426
|
+
let hi = a.length;
|
|
427
|
+
while (lo < hi) {
|
|
428
|
+
const mid = lo + hi >>> 1;
|
|
429
|
+
if (a[mid] <= value) lo = mid + 1;
|
|
430
|
+
else hi = mid;
|
|
431
|
+
}
|
|
432
|
+
return lo;
|
|
433
|
+
}
|
|
302
434
|
var SegmentEngine = class {
|
|
303
435
|
storage;
|
|
304
436
|
cache;
|
|
@@ -309,6 +441,12 @@ var SegmentEngine = class {
|
|
|
309
441
|
metricsOn;
|
|
310
442
|
/** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
|
|
311
443
|
budget;
|
|
444
|
+
/**
|
|
445
|
+
* The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
|
|
446
|
+
* read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
|
|
447
|
+
* promise per distinct key in flight.
|
|
448
|
+
*/
|
|
449
|
+
openReads = /* @__PURE__ */ new Map();
|
|
312
450
|
constructor(deps) {
|
|
313
451
|
this.storage = deps.storage;
|
|
314
452
|
this.cache = deps.cache;
|
|
@@ -389,7 +527,7 @@ var SegmentEngine = class {
|
|
|
389
527
|
);
|
|
390
528
|
}
|
|
391
529
|
/**
|
|
392
|
-
* Every id, ascending, reading ahead through a window of up to
|
|
530
|
+
* Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
|
|
393
531
|
* read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
|
|
394
532
|
* chunks the range overlaps (see {@link IdRange}).
|
|
395
533
|
*
|
|
@@ -428,6 +566,26 @@ var SegmentEngine = class {
|
|
|
428
566
|
}
|
|
429
567
|
}
|
|
430
568
|
}
|
|
569
|
+
/**
|
|
570
|
+
* {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
|
|
571
|
+
* the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
|
|
572
|
+
* reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
|
|
573
|
+
* the caller's to keep.
|
|
574
|
+
*/
|
|
575
|
+
async *iterateBatches(seg, range) {
|
|
576
|
+
const w = windowOf(range);
|
|
577
|
+
if (w === "empty") return;
|
|
578
|
+
const chunkKeys = w === null ? await this.chunkKeys(seg) : keysWithin(await this.chunkKeys(seg), w);
|
|
579
|
+
checkBudget(this.budget, chunkKeys.length, "iterate");
|
|
580
|
+
const gen = await this.cacheVersion(seg);
|
|
581
|
+
const window = this.chunkWindow(seg, chunkKeys, gen, true);
|
|
582
|
+
for (const chunkKey of chunkKeys) {
|
|
583
|
+
const chunk = await window.take();
|
|
584
|
+
if (chunk === null) continue;
|
|
585
|
+
const ids = chunkIds(chunk, chunkKey, w);
|
|
586
|
+
if (ids.length > 0) yield ids;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
431
589
|
/**
|
|
432
590
|
* `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
|
|
433
591
|
*
|
|
@@ -482,7 +640,24 @@ var SegmentEngine = class {
|
|
|
482
640
|
andNot(seg, excludes, options) {
|
|
483
641
|
return this.combine([seg], excludes, "all", "andNot", options);
|
|
484
642
|
}
|
|
485
|
-
|
|
643
|
+
/** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
|
|
644
|
+
intersectBatches(segs, options) {
|
|
645
|
+
return this.combineBatches(segs, options?.exclude ?? [], "all", "intersect", options);
|
|
646
|
+
}
|
|
647
|
+
/** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
|
|
648
|
+
unionBatches(segs, options) {
|
|
649
|
+
return this.combineBatches(segs, options?.exclude ?? [], "any", "union", options);
|
|
650
|
+
}
|
|
651
|
+
/** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
|
|
652
|
+
andNotBatches(seg, excludes, options) {
|
|
653
|
+
return this.combineBatches([seg], excludes, "all", "andNot", options);
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
|
|
657
|
+
* absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
|
|
658
|
+
* read and the batch read share it, so the two cannot disagree about what is read or charged.
|
|
659
|
+
*/
|
|
660
|
+
async combinePlan(segs, excludeSegs, mode, op, options) {
|
|
486
661
|
if (segs.length === 0) {
|
|
487
662
|
throw new ValidationError(`${op} requires at least one segment`);
|
|
488
663
|
}
|
|
@@ -497,7 +672,7 @@ var SegmentEngine = class {
|
|
|
497
672
|
}
|
|
498
673
|
const budget = resolvePerOpBudget(options?.budget, this.budget);
|
|
499
674
|
const w = windowOf(options);
|
|
500
|
-
if (w === "empty") return;
|
|
675
|
+
if (w === "empty") return null;
|
|
501
676
|
const extract = async (seg) => {
|
|
502
677
|
const all = await this.chunkKeys(seg);
|
|
503
678
|
const gen = await this.cacheVersion(seg);
|
|
@@ -542,17 +717,20 @@ var SegmentEngine = class {
|
|
|
542
717
|
skippedChunks: distinctKeys.size - common.length
|
|
543
718
|
});
|
|
544
719
|
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
720
|
+
return {
|
|
721
|
+
w,
|
|
722
|
+
window: new CombineWindow(
|
|
723
|
+
common,
|
|
724
|
+
limit,
|
|
725
|
+
(key) => this.combineChunk(operands, excludes, mode, key)
|
|
726
|
+
)
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
async *combine(segs, excludeSegs, mode, op, options) {
|
|
730
|
+
const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
|
|
731
|
+
if (plan === null) return;
|
|
732
|
+
const { w, window } = plan;
|
|
733
|
+
for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
|
|
556
734
|
if (slot.result && !slot.result.isEmpty) {
|
|
557
735
|
if (w !== null && isEdge(slot.key, w)) {
|
|
558
736
|
for (const id of edgeIds(slot.result, slot.key, w)) yield id;
|
|
@@ -562,6 +740,17 @@ var SegmentEngine = class {
|
|
|
562
740
|
}
|
|
563
741
|
}
|
|
564
742
|
}
|
|
743
|
+
async *combineBatches(segs, excludeSegs, mode, op, options) {
|
|
744
|
+
const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
|
|
745
|
+
if (plan === null) return;
|
|
746
|
+
const { w, window } = plan;
|
|
747
|
+
for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
|
|
748
|
+
if (slot.result) {
|
|
749
|
+
const ids = chunkIds(slot.result, slot.key, w);
|
|
750
|
+
if (ids.length > 0) yield ids;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
}
|
|
565
754
|
/**
|
|
566
755
|
* The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
|
|
567
756
|
* Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
|
|
@@ -574,9 +763,13 @@ var SegmentEngine = class {
|
|
|
574
763
|
async combineChunk(operands, excludes, mode, chunkKey) {
|
|
575
764
|
const present = mode === "all" ? operands : operands.filter((o) => o.keys.has(chunkKey));
|
|
576
765
|
if (present.length === 0) return null;
|
|
577
|
-
const
|
|
578
|
-
|
|
766
|
+
const relevant = excludes.filter((e) => e.keys.has(chunkKey));
|
|
767
|
+
const together = relevant.length > 0 && (mode === "any" || present.length === 1);
|
|
768
|
+
const reads = together ? [...present, ...relevant] : present;
|
|
769
|
+
const fetched = await Promise.all(
|
|
770
|
+
reads.map((op) => this.storageChunk({ ...op.seg, chunkKey }, op.gen))
|
|
579
771
|
);
|
|
772
|
+
const chunks = together ? fetched.slice(0, present.length) : fetched;
|
|
580
773
|
if (mode === "all" && chunks.some((c) => c === null)) return null;
|
|
581
774
|
const first = chunks[0];
|
|
582
775
|
const acc = first ? first.clone() : this.codec.empty();
|
|
@@ -591,11 +784,8 @@ var SegmentEngine = class {
|
|
|
591
784
|
}
|
|
592
785
|
}
|
|
593
786
|
if (acc.isEmpty) return null;
|
|
594
|
-
const relevant = excludes.filter((e) => e.keys.has(chunkKey));
|
|
595
787
|
if (relevant.length > 0) {
|
|
596
|
-
const cuts = await Promise.all(
|
|
597
|
-
relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen))
|
|
598
|
-
);
|
|
788
|
+
const cuts = together ? fetched.slice(present.length) : await Promise.all(relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen)));
|
|
599
789
|
for (const cut of cuts) {
|
|
600
790
|
if (cut === null) continue;
|
|
601
791
|
acc.andNotInPlace(cut);
|
|
@@ -699,6 +889,7 @@ var SegmentEngine = class {
|
|
|
699
889
|
invalidate(ref) {
|
|
700
890
|
const prefix = segmentPrefix(ref);
|
|
701
891
|
this.cache?.deleteWhere((key) => key.startsWith(prefix));
|
|
892
|
+
for (const key of this.openReads.keys()) if (key.startsWith(prefix)) this.openReads.delete(key);
|
|
702
893
|
this.storage.invalidate?.(ref);
|
|
703
894
|
}
|
|
704
895
|
/** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
|
|
@@ -728,7 +919,13 @@ var SegmentEngine = class {
|
|
|
728
919
|
* `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
|
|
729
920
|
* generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
|
|
730
921
|
*
|
|
731
|
-
*
|
|
922
|
+
* Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
|
|
923
|
+
* later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
|
|
924
|
+
* next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
|
|
925
|
+
* are two reads, never one.
|
|
926
|
+
*
|
|
927
|
+
* The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
|
|
928
|
+
* one): callers read it or clone it, never mutate it.
|
|
732
929
|
*/
|
|
733
930
|
async storageChunk(ref, gen) {
|
|
734
931
|
if (gen === null) return null;
|
|
@@ -741,6 +938,19 @@ var SegmentEngine = class {
|
|
|
741
938
|
}
|
|
742
939
|
if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
|
|
743
940
|
}
|
|
941
|
+
const open4 = this.openReads.get(cacheKey);
|
|
942
|
+
if (open4) return open4.read;
|
|
943
|
+
const token = {};
|
|
944
|
+
const entry = { token, read: this.fetchChunk(ref, cacheKey, token) };
|
|
945
|
+
this.openReads.set(cacheKey, entry);
|
|
946
|
+
const settled = () => {
|
|
947
|
+
if (this.openReads.get(cacheKey) === entry) this.openReads.delete(cacheKey);
|
|
948
|
+
};
|
|
949
|
+
entry.read.then(settled, settled);
|
|
950
|
+
return entry.read;
|
|
951
|
+
}
|
|
952
|
+
/** One request for one chunk, decoded, range-checked and cached. */
|
|
953
|
+
async fetchChunk(ref, cacheKey, token) {
|
|
744
954
|
const startedAt = this.metricsOn ? this.clock.now() : 0;
|
|
745
955
|
const bytes = await this.storage.getChunk(ref);
|
|
746
956
|
if (this.metricsOn) {
|
|
@@ -755,7 +965,7 @@ var SegmentEngine = class {
|
|
|
755
965
|
if (!bytes) return null;
|
|
756
966
|
const bitmap = this.codec.safeDeserialize(bytes, this.maxBitmapBytes);
|
|
757
967
|
this.assertChunkPayloadInRange(bitmap, ref.chunkKey);
|
|
758
|
-
this.cache?.set(cacheKey, bitmap);
|
|
968
|
+
if (this.openReads.get(cacheKey)?.token === token) this.cache?.set(cacheKey, bitmap);
|
|
759
969
|
return bitmap;
|
|
760
970
|
}
|
|
761
971
|
};
|
|
@@ -2269,76 +2479,6 @@ function aadFor(ref, generation, scope) {
|
|
|
2269
2479
|
return out;
|
|
2270
2480
|
}
|
|
2271
2481
|
|
|
2272
|
-
// src/core/crbm/format.ts
|
|
2273
|
-
var MAGIC = Uint8Array.of(67, 82, 66, 77);
|
|
2274
|
-
var VERSION_MAJOR = 1;
|
|
2275
|
-
var VERSION_MINOR = 0;
|
|
2276
|
-
var PREAMBLE_BYTES = 8;
|
|
2277
|
-
var PAYLOAD_START = PREAMBLE_BYTES;
|
|
2278
|
-
var FOOTER_BYTES = 104;
|
|
2279
|
-
var FOOTER = {
|
|
2280
|
-
indexOffset: 0,
|
|
2281
|
-
// u64
|
|
2282
|
-
indexLength: 8,
|
|
2283
|
-
// u64
|
|
2284
|
-
indexCrc32c: 16,
|
|
2285
|
-
// u32
|
|
2286
|
-
flags: 20,
|
|
2287
|
-
// u32
|
|
2288
|
-
payloadCodecId: 24,
|
|
2289
|
-
// u16 — which codec wrote the chunk payloads
|
|
2290
|
-
elementWidth: 26,
|
|
2291
|
-
// u8
|
|
2292
|
-
containerCodec: 27,
|
|
2293
|
-
// u8
|
|
2294
|
-
versionMajor: 28,
|
|
2295
|
-
// u8
|
|
2296
|
-
versionMinor: 29,
|
|
2297
|
-
// u8
|
|
2298
|
-
reserved2: 30,
|
|
2299
|
-
// u16
|
|
2300
|
-
generation: 32,
|
|
2301
|
-
// u64
|
|
2302
|
-
indexNonce: 40,
|
|
2303
|
-
// 12 B
|
|
2304
|
-
indexTag: 52,
|
|
2305
|
-
// 16 B
|
|
2306
|
-
keyId: 68,
|
|
2307
|
-
// 16 B
|
|
2308
|
-
chunkCount: 84,
|
|
2309
|
-
// u32
|
|
2310
|
-
totalCardinality: 88,
|
|
2311
|
-
// u64
|
|
2312
|
-
footerCrc32c: 96,
|
|
2313
|
-
// u32 — covers footer bytes [0, 96)
|
|
2314
|
-
endMagic: 100
|
|
2315
|
-
// char[4]
|
|
2316
|
-
};
|
|
2317
|
-
var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
|
|
2318
|
-
var FLAG_ENCRYPTED = 1 << 0;
|
|
2319
|
-
var FLAG_INDEX_COMPRESSED = 1 << 1;
|
|
2320
|
-
var FLAG_LITTLE_ENDIAN = 1 << 2;
|
|
2321
|
-
var FLAG_EXTENSION = 1 << 3;
|
|
2322
|
-
var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
|
|
2323
|
-
var ELEMENT_WIDTH_32 = 32;
|
|
2324
|
-
var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
|
|
2325
|
-
var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
|
|
2326
|
-
PAYLOAD_CODEC_ROARING_PORTABLE
|
|
2327
|
-
]);
|
|
2328
|
-
var CONTAINER_CODEC_NONE = 0;
|
|
2329
|
-
var DEFAULT_TAIL_BYTES = 256 * 1024;
|
|
2330
|
-
var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
|
|
2331
|
-
var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
|
|
2332
|
-
var MAX_CHUNK_CARDINALITY = 65536;
|
|
2333
|
-
var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
|
|
2334
|
-
var EXT_TRAILER_BYTES = 12;
|
|
2335
|
-
var EXT_SECTION_HEADER_BYTES = 5;
|
|
2336
|
-
var MAX_EXT_BYTES = 4 * 1024;
|
|
2337
|
-
var EXT_SECTION_METADATA = 1;
|
|
2338
|
-
var CRC32C_BYTES = 4;
|
|
2339
|
-
var AEAD_NONCE_BYTES = 12;
|
|
2340
|
-
var AEAD_TAG_BYTES = 16;
|
|
2341
|
-
|
|
2342
2482
|
// src/core/crbm/crc32c.ts
|
|
2343
2483
|
var TABLES = (() => {
|
|
2344
2484
|
const POLY = 2197175160;
|
|
@@ -2540,7 +2680,6 @@ var CrbmReader = class _CrbmReader {
|
|
|
2540
2680
|
}
|
|
2541
2681
|
static async open(blob, options = {}) {
|
|
2542
2682
|
const tailBytes = Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
|
|
2543
|
-
const maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
|
|
2544
2683
|
const maxIndexBytes = options.maxIndexBytes ?? DEFAULT_MAX_INDEX_BYTES;
|
|
2545
2684
|
const { bytes: tail, size } = await blob.getTail(tailBytes);
|
|
2546
2685
|
const { footer, fview, storedFooterCrc } = checkedFooter(tail, size);
|
|
@@ -2682,7 +2821,7 @@ var CrbmReader = class _CrbmReader {
|
|
|
2682
2821
|
const index = parseIndex(
|
|
2683
2822
|
indexForParse,
|
|
2684
2823
|
payloadEnd,
|
|
2685
|
-
maxPayloadBytes,
|
|
2824
|
+
options.maxPayloadBytes ?? (encrypted ? DEFAULT_MAX_PAYLOAD_BYTES : DEFAULT_MAX_BITMAP_BYTES),
|
|
2686
2825
|
encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
|
|
2687
2826
|
);
|
|
2688
2827
|
if (encrypted) {
|
|
@@ -2869,7 +3008,9 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
|
|
|
2869
3008
|
}
|
|
2870
3009
|
if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
|
|
2871
3010
|
if (len < minPayloadBytes || len > maxPayloadBytes) {
|
|
2872
|
-
throw new IntegrityError(
|
|
3011
|
+
throw new IntegrityError(
|
|
3012
|
+
`.crbm chunk ${chunkKey} length ${len} invalid (most is ${maxPayloadBytes})`
|
|
3013
|
+
);
|
|
2873
3014
|
}
|
|
2874
3015
|
if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
|
|
2875
3016
|
throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
|
|
@@ -5094,12 +5235,11 @@ function describingSummary(record) {
|
|
|
5094
5235
|
}
|
|
5095
5236
|
|
|
5096
5237
|
// src/core/erase-id.ts
|
|
5097
|
-
var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
|
|
5098
5238
|
async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
5099
5239
|
validateUserRef(ref);
|
|
5100
5240
|
const { chunkKey, remainder } = splitId(id);
|
|
5101
5241
|
const codec = requireCodec(deps.codec, "eraseIdFromSegment");
|
|
5102
|
-
const maxBytes = deps.maxBitmapBytes ??
|
|
5242
|
+
const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
|
|
5103
5243
|
const base = { segment: ref.segment, namespace: ref.namespace };
|
|
5104
5244
|
const read = (op) => retryRead(op, deps.readRetry);
|
|
5105
5245
|
const record = await deps.registry.get(ref);
|
|
@@ -5311,14 +5451,21 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
|
|
|
5311
5451
|
if (left !== void 0) throw cannotRemove(left);
|
|
5312
5452
|
return { ...base, erased: true, fromGeneration: from, generation, collected };
|
|
5313
5453
|
}
|
|
5454
|
+
var REWRITE_READ_AHEAD = 32;
|
|
5314
5455
|
async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
|
|
5315
5456
|
const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
|
|
5457
|
+
const window = new ChunkWindow(
|
|
5458
|
+
keys.filter((k) => k !== chunkKey),
|
|
5459
|
+
(k) => read(() => reader.getChunk(k)),
|
|
5460
|
+
REWRITE_READ_AHEAD,
|
|
5461
|
+
false
|
|
5462
|
+
);
|
|
5316
5463
|
for (const k of keys) {
|
|
5317
5464
|
if (k === chunkKey) {
|
|
5318
5465
|
yield { chunkKey: k, bitmap: replacement };
|
|
5319
5466
|
continue;
|
|
5320
5467
|
}
|
|
5321
|
-
const bytes = await
|
|
5468
|
+
const bytes = await window.take();
|
|
5322
5469
|
if (bytes === null) continue;
|
|
5323
5470
|
const bitmap = codec.safeDeserialize(bytes, maxBytes);
|
|
5324
5471
|
assertRemaindersInRange(bitmap, k);
|