@cloudbitmaps/core 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -102,6 +102,110 @@ async function collectWithinBudget(source, budget, op) {
102
102
  return out;
103
103
  }
104
104
 
105
+ // src/core/chunk-window.ts
106
+ var ChunkWindow = class {
107
+ constructor(keys, fetch, max, ramp) {
108
+ this.keys = keys;
109
+ this.fetch = fetch;
110
+ this.max = max;
111
+ this.ramp = ramp;
112
+ }
113
+ keys;
114
+ fetch;
115
+ max;
116
+ ramp;
117
+ open = [];
118
+ launched = 0;
119
+ taken = 0;
120
+ /** The next read's result in key order, or the error that read raised. */
121
+ async take() {
122
+ const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
123
+ while (this.launched < this.keys.length && this.launched - this.taken < width) {
124
+ this.open.push(
125
+ this.fetch(this.keys[this.launched++]).then(
126
+ (value) => ({ value }),
127
+ (cause) => ({ value: void 0, error: { cause } })
128
+ )
129
+ );
130
+ }
131
+ const slot = await this.open.shift();
132
+ this.taken += 1;
133
+ if (slot.error) throw slot.error.cause;
134
+ return slot.value;
135
+ }
136
+ };
137
+
138
+ // src/core/crbm/format.ts
139
+ var MAGIC = Uint8Array.of(67, 82, 66, 77);
140
+ var VERSION_MAJOR = 1;
141
+ var VERSION_MINOR = 0;
142
+ var PREAMBLE_BYTES = 8;
143
+ var PAYLOAD_START = PREAMBLE_BYTES;
144
+ var FOOTER_BYTES = 104;
145
+ var FOOTER = {
146
+ indexOffset: 0,
147
+ // u64
148
+ indexLength: 8,
149
+ // u64
150
+ indexCrc32c: 16,
151
+ // u32
152
+ flags: 20,
153
+ // u32
154
+ payloadCodecId: 24,
155
+ // u16 — which codec wrote the chunk payloads
156
+ elementWidth: 26,
157
+ // u8
158
+ containerCodec: 27,
159
+ // u8
160
+ versionMajor: 28,
161
+ // u8
162
+ versionMinor: 29,
163
+ // u8
164
+ reserved2: 30,
165
+ // u16
166
+ generation: 32,
167
+ // u64
168
+ indexNonce: 40,
169
+ // 12 B
170
+ indexTag: 52,
171
+ // 16 B
172
+ keyId: 68,
173
+ // 16 B
174
+ chunkCount: 84,
175
+ // u32
176
+ totalCardinality: 88,
177
+ // u64
178
+ footerCrc32c: 96,
179
+ // u32 — covers footer bytes [0, 96)
180
+ endMagic: 100
181
+ // char[4]
182
+ };
183
+ var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
184
+ var FLAG_ENCRYPTED = 1 << 0;
185
+ var FLAG_INDEX_COMPRESSED = 1 << 1;
186
+ var FLAG_LITTLE_ENDIAN = 1 << 2;
187
+ var FLAG_EXTENSION = 1 << 3;
188
+ var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
189
+ var ELEMENT_WIDTH_32 = 32;
190
+ var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
191
+ var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
192
+ PAYLOAD_CODEC_ROARING_PORTABLE
193
+ ]);
194
+ var CONTAINER_CODEC_NONE = 0;
195
+ var DEFAULT_TAIL_BYTES = 256 * 1024;
196
+ var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
197
+ var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
198
+ var MAX_CHUNK_CARDINALITY = 65536;
199
+ var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
200
+ var EXT_TRAILER_BYTES = 12;
201
+ var EXT_SECTION_HEADER_BYTES = 5;
202
+ var MAX_EXT_BYTES = 4 * 1024;
203
+ var EXT_SECTION_METADATA = 1;
204
+ var CRC32C_BYTES = 4;
205
+ var AEAD_NONCE_BYTES = 12;
206
+ var AEAD_TAG_BYTES = 16;
207
+ var DEFAULT_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BITMAP_BYTES + AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
208
+
105
209
  // src/core/keys.ts
106
210
  var FIELD = " ";
107
211
  var NO_NAMESPACE = "/";
@@ -214,8 +318,8 @@ var CountingMetricsSink = class _CountingMetricsSink {
214
318
  };
215
319
 
216
320
  // src/core/engine.ts
217
- var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
218
- var DEFAULT_INTERSECT_CONCURRENCY = 8;
321
+ var DEFAULT_INTERSECT_CONCURRENCY = 32;
322
+ var COMBINE_WINDOW_START = 8;
219
323
  var ZERO_CLOCK = { now: () => 0 };
220
324
  function windowOf(range) {
221
325
  const after = range?.after;
@@ -268,37 +372,65 @@ function keysWithin(keys, w) {
268
372
  };
269
373
  return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
270
374
  }
271
- var ChunkWindow = class {
272
- constructor(keys, fetch, max, ramp) {
375
+ var CombineWindow = class {
376
+ constructor(keys, limit, read) {
273
377
  this.keys = keys;
274
- this.fetch = fetch;
275
- this.max = max;
276
- this.ramp = ramp;
378
+ this.limit = limit;
379
+ this.read = read;
380
+ this.fill();
277
381
  }
278
382
  keys;
279
- fetch;
280
- max;
281
- ramp;
282
- open = [];
283
- launched = 0;
383
+ limit;
384
+ read;
385
+ inFlight = [];
386
+ next_ = 0;
284
387
  taken = 0;
285
- /** The next chunk in key order (null if the source holds none), or the error its read raised. */
286
- async take() {
287
- const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
288
- while (this.launched < this.keys.length && this.launched - this.taken < width) {
289
- this.open.push(
290
- this.fetch(this.keys[this.launched++]).then(
291
- (chunk) => ({ chunk }),
292
- (cause) => ({ chunk: null, error: { cause } })
388
+ fill() {
389
+ const width = Math.min(
390
+ this.limit,
391
+ Math.max(COMBINE_WINDOW_START, 2 ** Math.min(this.taken, 30))
392
+ );
393
+ while (this.next_ < this.keys.length && this.inFlight.length < width) {
394
+ const key = this.keys[this.next_++];
395
+ this.inFlight.push(
396
+ this.read(key).then(
397
+ (result) => ({ key, result }),
398
+ (error) => ({ key, result: null, error })
293
399
  )
294
400
  );
295
401
  }
296
- const slot = await this.open.shift();
402
+ }
403
+ /** The next key's chunk in ascending key order, `undefined` when the keys are used up; throws that key's error. */
404
+ async next() {
405
+ if (this.inFlight.length === 0) return void 0;
406
+ const slot = await this.inFlight.shift();
297
407
  this.taken += 1;
298
- if (slot.error) throw slot.error.cause;
299
- return slot.chunk;
408
+ this.fill();
409
+ if (slot.error !== void 0) throw slot.error;
410
+ return slot;
300
411
  }
301
412
  };
413
+ function chunkIds(chunk, chunkKey, w) {
414
+ let rem = chunk.toUint32Array ? chunk.toUint32Array() : Uint32Array.from(chunk);
415
+ if (w !== null && isEdge(chunkKey, w)) {
416
+ const from = chunkKey === w.loKey ? w.loRem : 0;
417
+ const to = chunkKey === w.hiKey ? w.hiRem : MAX_REMAINDER;
418
+ rem = rem.slice(firstAbove(rem, from - 1), firstAbove(rem, to));
419
+ }
420
+ const base = chunkKey * CHUNK_COUNT;
421
+ for (let i = 0; i < rem.length; i++) rem[i] = base + (rem[i] & MAX_REMAINDER);
422
+ return rem;
423
+ }
424
+ function firstAbove(a, value) {
425
+ let lo = 0;
426
+ let hi = a.length;
427
+ while (lo < hi) {
428
+ const mid = lo + hi >>> 1;
429
+ if (a[mid] <= value) lo = mid + 1;
430
+ else hi = mid;
431
+ }
432
+ return lo;
433
+ }
302
434
  var SegmentEngine = class {
303
435
  storage;
304
436
  cache;
@@ -309,6 +441,12 @@ var SegmentEngine = class {
309
441
  metricsOn;
310
442
  /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
311
443
  budget;
444
+ /**
445
+ * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
446
+ * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
447
+ * promise per distinct key in flight.
448
+ */
449
+ openReads = /* @__PURE__ */ new Map();
312
450
  constructor(deps) {
313
451
  this.storage = deps.storage;
314
452
  this.cache = deps.cache;
@@ -389,7 +527,7 @@ var SegmentEngine = class {
389
527
  );
390
528
  }
391
529
  /**
392
- * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
530
+ * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
393
531
  * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
394
532
  * chunks the range overlaps (see {@link IdRange}).
395
533
  *
@@ -428,6 +566,26 @@ var SegmentEngine = class {
428
566
  }
429
567
  }
430
568
  }
569
+ /**
570
+ * {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
571
+ * the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
572
+ * reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
573
+ * the caller's to keep.
574
+ */
575
+ async *iterateBatches(seg, range) {
576
+ const w = windowOf(range);
577
+ if (w === "empty") return;
578
+ const chunkKeys = w === null ? await this.chunkKeys(seg) : keysWithin(await this.chunkKeys(seg), w);
579
+ checkBudget(this.budget, chunkKeys.length, "iterate");
580
+ const gen = await this.cacheVersion(seg);
581
+ const window = this.chunkWindow(seg, chunkKeys, gen, true);
582
+ for (const chunkKey of chunkKeys) {
583
+ const chunk = await window.take();
584
+ if (chunk === null) continue;
585
+ const ids = chunkIds(chunk, chunkKey, w);
586
+ if (ids.length > 0) yield ids;
587
+ }
588
+ }
431
589
  /**
432
590
  * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
433
591
  *
@@ -482,7 +640,24 @@ var SegmentEngine = class {
482
640
  andNot(seg, excludes, options) {
483
641
  return this.combine([seg], excludes, "all", "andNot", options);
484
642
  }
485
- async *combine(segs, excludeSegs, mode, op, options) {
643
+ /** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
644
+ intersectBatches(segs, options) {
645
+ return this.combineBatches(segs, options?.exclude ?? [], "all", "intersect", options);
646
+ }
647
+ /** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
648
+ unionBatches(segs, options) {
649
+ return this.combineBatches(segs, options?.exclude ?? [], "any", "union", options);
650
+ }
651
+ /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
652
+ andNotBatches(seg, excludes, options) {
653
+ return this.combineBatches([seg], excludes, "all", "andNot", options);
654
+ }
655
+ /**
656
+ * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
657
+ * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
658
+ * read and the batch read share it, so the two cannot disagree about what is read or charged.
659
+ */
660
+ async combinePlan(segs, excludeSegs, mode, op, options) {
486
661
  if (segs.length === 0) {
487
662
  throw new ValidationError(`${op} requires at least one segment`);
488
663
  }
@@ -497,7 +672,7 @@ var SegmentEngine = class {
497
672
  }
498
673
  const budget = resolvePerOpBudget(options?.budget, this.budget);
499
674
  const w = windowOf(options);
500
- if (w === "empty") return;
675
+ if (w === "empty") return null;
501
676
  const extract = async (seg) => {
502
677
  const all = await this.chunkKeys(seg);
503
678
  const gen = await this.cacheVersion(seg);
@@ -542,17 +717,20 @@ var SegmentEngine = class {
542
717
  skippedChunks: distinctKeys.size - common.length
543
718
  });
544
719
  }
545
- const startAt = (key) => this.combineChunk(operands, excludes, mode, key).then(
546
- (result) => ({ key, result }),
547
- (error) => ({ key, result: null, error })
548
- );
549
- const inFlight = [];
550
- let next = 0;
551
- while (next < common.length && inFlight.length < limit) inFlight.push(startAt(common[next++]));
552
- while (inFlight.length > 0) {
553
- const slot = await inFlight.shift();
554
- if (next < common.length) inFlight.push(startAt(common[next++]));
555
- if (slot.error !== void 0) throw slot.error;
720
+ return {
721
+ w,
722
+ window: new CombineWindow(
723
+ common,
724
+ limit,
725
+ (key) => this.combineChunk(operands, excludes, mode, key)
726
+ )
727
+ };
728
+ }
729
+ async *combine(segs, excludeSegs, mode, op, options) {
730
+ const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
731
+ if (plan === null) return;
732
+ const { w, window } = plan;
733
+ for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
556
734
  if (slot.result && !slot.result.isEmpty) {
557
735
  if (w !== null && isEdge(slot.key, w)) {
558
736
  for (const id of edgeIds(slot.result, slot.key, w)) yield id;
@@ -562,6 +740,17 @@ var SegmentEngine = class {
562
740
  }
563
741
  }
564
742
  }
743
+ async *combineBatches(segs, excludeSegs, mode, op, options) {
744
+ const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
745
+ if (plan === null) return;
746
+ const { w, window } = plan;
747
+ for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
748
+ if (slot.result) {
749
+ const ids = chunkIds(slot.result, slot.key, w);
750
+ if (ids.length > 0) yield ids;
751
+ }
752
+ }
753
+ }
565
754
  /**
566
755
  * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
567
756
  * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
@@ -574,9 +763,13 @@ var SegmentEngine = class {
574
763
  async combineChunk(operands, excludes, mode, chunkKey) {
575
764
  const present = mode === "all" ? operands : operands.filter((o) => o.keys.has(chunkKey));
576
765
  if (present.length === 0) return null;
577
- const chunks = await Promise.all(
578
- present.map((op) => this.storageChunk({ ...op.seg, chunkKey }, op.gen))
766
+ const relevant = excludes.filter((e) => e.keys.has(chunkKey));
767
+ const together = relevant.length > 0 && (mode === "any" || present.length === 1);
768
+ const reads = together ? [...present, ...relevant] : present;
769
+ const fetched = await Promise.all(
770
+ reads.map((op) => this.storageChunk({ ...op.seg, chunkKey }, op.gen))
579
771
  );
772
+ const chunks = together ? fetched.slice(0, present.length) : fetched;
580
773
  if (mode === "all" && chunks.some((c) => c === null)) return null;
581
774
  const first = chunks[0];
582
775
  const acc = first ? first.clone() : this.codec.empty();
@@ -591,11 +784,8 @@ var SegmentEngine = class {
591
784
  }
592
785
  }
593
786
  if (acc.isEmpty) return null;
594
- const relevant = excludes.filter((e) => e.keys.has(chunkKey));
595
787
  if (relevant.length > 0) {
596
- const cuts = await Promise.all(
597
- relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen))
598
- );
788
+ const cuts = together ? fetched.slice(present.length) : await Promise.all(relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen)));
599
789
  for (const cut of cuts) {
600
790
  if (cut === null) continue;
601
791
  acc.andNotInPlace(cut);
@@ -699,6 +889,7 @@ var SegmentEngine = class {
699
889
  invalidate(ref) {
700
890
  const prefix = segmentPrefix(ref);
701
891
  this.cache?.deleteWhere((key) => key.startsWith(prefix));
892
+ for (const key of this.openReads.keys()) if (key.startsWith(prefix)) this.openReads.delete(key);
702
893
  this.storage.invalidate?.(ref);
703
894
  }
704
895
  /** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
@@ -728,7 +919,13 @@ var SegmentEngine = class {
728
919
  * `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
729
920
  * generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
730
921
  *
731
- * The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
922
+ * Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
923
+ * later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
924
+ * next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
925
+ * are two reads, never one.
926
+ *
927
+ * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
928
+ * one): callers read it or clone it, never mutate it.
732
929
  */
733
930
  async storageChunk(ref, gen) {
734
931
  if (gen === null) return null;
@@ -741,6 +938,19 @@ var SegmentEngine = class {
741
938
  }
742
939
  if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
743
940
  }
941
+ const open4 = this.openReads.get(cacheKey);
942
+ if (open4) return open4.read;
943
+ const token = {};
944
+ const entry = { token, read: this.fetchChunk(ref, cacheKey, token) };
945
+ this.openReads.set(cacheKey, entry);
946
+ const settled = () => {
947
+ if (this.openReads.get(cacheKey) === entry) this.openReads.delete(cacheKey);
948
+ };
949
+ entry.read.then(settled, settled);
950
+ return entry.read;
951
+ }
952
+ /** One request for one chunk, decoded, range-checked and cached. */
953
+ async fetchChunk(ref, cacheKey, token) {
744
954
  const startedAt = this.metricsOn ? this.clock.now() : 0;
745
955
  const bytes = await this.storage.getChunk(ref);
746
956
  if (this.metricsOn) {
@@ -755,7 +965,7 @@ var SegmentEngine = class {
755
965
  if (!bytes) return null;
756
966
  const bitmap = this.codec.safeDeserialize(bytes, this.maxBitmapBytes);
757
967
  this.assertChunkPayloadInRange(bitmap, ref.chunkKey);
758
- this.cache?.set(cacheKey, bitmap);
968
+ if (this.openReads.get(cacheKey)?.token === token) this.cache?.set(cacheKey, bitmap);
759
969
  return bitmap;
760
970
  }
761
971
  };
@@ -2269,76 +2479,6 @@ function aadFor(ref, generation, scope) {
2269
2479
  return out;
2270
2480
  }
2271
2481
 
2272
- // src/core/crbm/format.ts
2273
- var MAGIC = Uint8Array.of(67, 82, 66, 77);
2274
- var VERSION_MAJOR = 1;
2275
- var VERSION_MINOR = 0;
2276
- var PREAMBLE_BYTES = 8;
2277
- var PAYLOAD_START = PREAMBLE_BYTES;
2278
- var FOOTER_BYTES = 104;
2279
- var FOOTER = {
2280
- indexOffset: 0,
2281
- // u64
2282
- indexLength: 8,
2283
- // u64
2284
- indexCrc32c: 16,
2285
- // u32
2286
- flags: 20,
2287
- // u32
2288
- payloadCodecId: 24,
2289
- // u16 — which codec wrote the chunk payloads
2290
- elementWidth: 26,
2291
- // u8
2292
- containerCodec: 27,
2293
- // u8
2294
- versionMajor: 28,
2295
- // u8
2296
- versionMinor: 29,
2297
- // u8
2298
- reserved2: 30,
2299
- // u16
2300
- generation: 32,
2301
- // u64
2302
- indexNonce: 40,
2303
- // 12 B
2304
- indexTag: 52,
2305
- // 16 B
2306
- keyId: 68,
2307
- // 16 B
2308
- chunkCount: 84,
2309
- // u32
2310
- totalCardinality: 88,
2311
- // u64
2312
- footerCrc32c: 96,
2313
- // u32 — covers footer bytes [0, 96)
2314
- endMagic: 100
2315
- // char[4]
2316
- };
2317
- var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
2318
- var FLAG_ENCRYPTED = 1 << 0;
2319
- var FLAG_INDEX_COMPRESSED = 1 << 1;
2320
- var FLAG_LITTLE_ENDIAN = 1 << 2;
2321
- var FLAG_EXTENSION = 1 << 3;
2322
- var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
2323
- var ELEMENT_WIDTH_32 = 32;
2324
- var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
2325
- var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
2326
- PAYLOAD_CODEC_ROARING_PORTABLE
2327
- ]);
2328
- var CONTAINER_CODEC_NONE = 0;
2329
- var DEFAULT_TAIL_BYTES = 256 * 1024;
2330
- var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
2331
- var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
2332
- var MAX_CHUNK_CARDINALITY = 65536;
2333
- var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
2334
- var EXT_TRAILER_BYTES = 12;
2335
- var EXT_SECTION_HEADER_BYTES = 5;
2336
- var MAX_EXT_BYTES = 4 * 1024;
2337
- var EXT_SECTION_METADATA = 1;
2338
- var CRC32C_BYTES = 4;
2339
- var AEAD_NONCE_BYTES = 12;
2340
- var AEAD_TAG_BYTES = 16;
2341
-
2342
2482
  // src/core/crbm/crc32c.ts
2343
2483
  var TABLES = (() => {
2344
2484
  const POLY = 2197175160;
@@ -2540,7 +2680,6 @@ var CrbmReader = class _CrbmReader {
2540
2680
  }
2541
2681
  static async open(blob, options = {}) {
2542
2682
  const tailBytes = Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
2543
- const maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
2544
2683
  const maxIndexBytes = options.maxIndexBytes ?? DEFAULT_MAX_INDEX_BYTES;
2545
2684
  const { bytes: tail, size } = await blob.getTail(tailBytes);
2546
2685
  const { footer, fview, storedFooterCrc } = checkedFooter(tail, size);
@@ -2682,7 +2821,7 @@ var CrbmReader = class _CrbmReader {
2682
2821
  const index = parseIndex(
2683
2822
  indexForParse,
2684
2823
  payloadEnd,
2685
- maxPayloadBytes,
2824
+ options.maxPayloadBytes ?? (encrypted ? DEFAULT_MAX_PAYLOAD_BYTES : DEFAULT_MAX_BITMAP_BYTES),
2686
2825
  encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
2687
2826
  );
2688
2827
  if (encrypted) {
@@ -2869,7 +3008,9 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
2869
3008
  }
2870
3009
  if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
2871
3010
  if (len < minPayloadBytes || len > maxPayloadBytes) {
2872
- throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len} invalid`);
3011
+ throw new IntegrityError(
3012
+ `.crbm chunk ${chunkKey} length ${len} invalid (most is ${maxPayloadBytes})`
3013
+ );
2873
3014
  }
2874
3015
  if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
2875
3016
  throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
@@ -5094,12 +5235,11 @@ function describingSummary(record) {
5094
5235
  }
5095
5236
 
5096
5237
  // src/core/erase-id.ts
5097
- var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
5098
5238
  async function eraseIdFromSegment(ref, id, deps, options = {}) {
5099
5239
  validateUserRef(ref);
5100
5240
  const { chunkKey, remainder } = splitId(id);
5101
5241
  const codec = requireCodec(deps.codec, "eraseIdFromSegment");
5102
- const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
5242
+ const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
5103
5243
  const base = { segment: ref.segment, namespace: ref.namespace };
5104
5244
  const read = (op) => retryRead(op, deps.readRetry);
5105
5245
  const record = await deps.registry.get(ref);
@@ -5311,14 +5451,21 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
5311
5451
  if (left !== void 0) throw cannotRemove(left);
5312
5452
  return { ...base, erased: true, fromGeneration: from, generation, collected };
5313
5453
  }
5454
+ var REWRITE_READ_AHEAD = 32;
5314
5455
  async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
5315
5456
  const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
5457
+ const window = new ChunkWindow(
5458
+ keys.filter((k) => k !== chunkKey),
5459
+ (k) => read(() => reader.getChunk(k)),
5460
+ REWRITE_READ_AHEAD,
5461
+ false
5462
+ );
5316
5463
  for (const k of keys) {
5317
5464
  if (k === chunkKey) {
5318
5465
  yield { chunkKey: k, bitmap: replacement };
5319
5466
  continue;
5320
5467
  }
5321
- const bytes = await read(() => reader.getChunk(k));
5468
+ const bytes = await window.take();
5322
5469
  if (bytes === null) continue;
5323
5470
  const bitmap = codec.safeDeserialize(bytes, maxBytes);
5324
5471
  assertRemaindersInRange(bitmap, k);