@cloudbitmaps/core 0.12.0 → 0.13.0

package/dist/core/chunk-window.d.ts

@@ -0,0 +1,24 @@
+/**
+ * An ordered window of reads: up to `max` reads open ahead of the one being consumed, handed back in key order.
+ * With `ramp`, the window opens 1, 2, 4 … wide up to `max` instead of `max` at once, so a consumer that stops
+ * after a few chunks has fetched a handful, not a full window.
+ *
+ * `T` is whatever one read yields (a decoded chunk, or the raw bytes of one). Each read is wrapped to resolve and
+ * never reject, so a read nobody consumes (the consumer stopped, or an earlier chunk failed) cannot raise an
+ * unhandled rejection; its error surfaces from {@link take} only if the consumer reaches that chunk. Memory is
+ * bounded by the window: at most `max` results are held ahead.
+ *
+ * Internal to core: not exported from any entry point.
+ */
+export declare class ChunkWindow<T> {
+    private readonly keys;
+    private readonly fetch;
+    private readonly max;
+    private readonly ramp;
+    private readonly open;
+    private launched;
+    private taken;
+    constructor(keys: readonly number[], fetch: (chunkKey: number) => Promise<T>, max: number, ramp: boolean);
+    /** The next read's result in key order, or the error that read raised. */
+    take(): Promise<T>;
+}

package/dist/core/crbm/format.d.ts

@@ -104,8 +104,11 @@ export declare const DEFAULT_TAIL_BYTES: number;
  * above the ~650 KB full-span worst case, well below a denial-of-wallet allocation.
  */
 export declare const DEFAULT_MAX_INDEX_BYTES: number;
-/** Hard cap on a single chunk payload, defending the native deserializer against oversized input. */
-export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;
+/**
+ * Largest serialized bitmap the engine decodes (1 MiB, per chunk). A well-formed 16-bit chunk serializes to at most
+ * about 8.2 KiB, so no legitimate chunk is near it; both the read engine and the erase path decode under this one cap.
+ */
+export declare const DEFAULT_MAX_BITMAP_BYTES: number;
 /** A chunk's cardinality is in `[1, 65536]` (empty chunks are never written). */
 export declare const MAX_CHUNK_CARDINALITY = 65536;
 /**
@@ -142,3 +145,9 @@ export declare const CRC32C_BYTES = 4;
  */
 export declare const AEAD_NONCE_BYTES = 12;
 export declare const AEAD_TAG_BYTES = 16;
+/**
+ * Hard cap on one encrypted chunk's stored payload (a cleartext one is capped at the decode cap): the decode cap plus the AEAD framing, so an
+ * entry the engine would refuse to decode is refused when the object is opened, before any payload is read. No writer
+ * exceeds it: a stored payload is a serialized 16-bit chunk (at most about 8.2 KiB) plus, when encrypted, that framing.
+ */
+export declare const DEFAULT_MAX_PAYLOAD_BYTES: number;

package/dist/core/crbm/reader.d.ts

@@ -31,7 +31,7 @@ export declare function indexCapacity(indexLength: number): number;
 export interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
-    /** Hard cap on a single chunk payload length (default 16 MB). */
+    /** Hard cap on a single chunk payload length (default: the 1 MiB decode cap, plus the 28-byte nonce and tag on an encrypted object). The engine decodes under `maxBitmapBytes`; raise this with it. */
     readonly maxPayloadBytes?: number;
     /** Hard cap on the whole index region fetched/parsed from one object (default 8 MB). */
     readonly maxIndexBytes?: number;

package/dist/core/crbm-storage-source.d.ts

@@ -426,7 +426,7 @@ export interface StreamWriteResult {
  * Streaming variant of {@link writeCrbmGeneration} for a **constant-memory** rewrite (the erasure rewrite, a
  * sorted load): consumes an **already-ascending** async stream of `{ chunkKey, bitmap }`, feeding each to the
  * codec and freeing it, instead of materializing the whole generation. Paired with a streaming storage sink (S3
- * multipart / LocalFs temp file), peak memory is ~one chunk + one part. Returns a tally so the caller can verify
+ * multipart / LocalFs temp file), the writer itself holds ~one chunk + one part. Returns a tally so the caller can verify
  * the re-opened object without re-iterating the (now-consumed) stream. Input **must** be ascending by `chunkKey`
  * — the codec rejects an out-of-order chunk; empty bitmaps are skipped.
  */

package/dist/core/engine.d.ts

@@ -8,6 +8,7 @@ export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
     readonly cache?: BoundedLru<string, CodecBitmap>;
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
     /**
      * The bitmap codec — **required**. `core/` is codec-agnostic: it can have no default, because the concrete codec
@@ -47,7 +48,13 @@ export interface IdRange {
 }
 /** Options common to the chunk-aligned combines. */
 export interface CombineOptions extends IdRange {
-    /** Max chunk keys resolved concurrently — bounds the Storage footprint. A positive integer. */
+    /**
+     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
+     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
+     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
+     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
+     * `concurrency` keys per operand past the last one it used.
+     */
     readonly concurrency?: number;
     /** Override the store's per-op budget for this call (`false` lifts it). */
     readonly budget?: BudgetOption;
@@ -70,6 +77,12 @@ export declare class SegmentEngine {
     private readonly metricsOn;
     /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
     private readonly budget;
+    /**
+     * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
+     * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
+     * promise per distinct key in flight.
+     */
+    private readonly openReads;
     constructor(deps: EngineDeps);
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
@@ -104,7 +117,7 @@ export declare class SegmentEngine {
     /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
     private chunkWindow;
     /**
-     * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
+     * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
      * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
      * chunks the range overlaps (see {@link IdRange}).
      *
@@ -255,7 +268,15 @@ export declare class SegmentEngine {
      * `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
      * generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
      *
-     * The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
+     * Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
+     * later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
+     * next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
+     * are two reads, never one.
+     *
+     * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
+     * one): callers read it or clone it, never mutate it.
      */
     private storageChunk;
+    /** One request for one chunk, decoded, range-checked and cached. */
+    private fetchChunk;
 }

package/dist/core/erase-id.d.ts

@@ -6,8 +6,8 @@
  * is what every other write in this library is: a new generation. The current generation is streamed chunk by
  * chunk through the ascending writer — every chunk decoded, range-checked and re-encoded, the one holding the id
  * with that bit cleared — then published fenced on the generation it streamed, and that generation is collected immediately
- * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Constant memory: one chunk in
- * flight, never the whole segment.
+ * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Bounded memory: a window of 32
+ * chunk reads ahead of the writer, never the whole segment.
  *
  * The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
  * metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
@@ -103,7 +103,7 @@ export interface EraseIdDeps {
      * source is used if it has one, and otherwise each wait is its bound.
      */
     readonly rng?: Rng;
-    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. */
+    /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. The `.crbm` reader refuses an entry above its own `maxPayloadBytes` (1 MiB, plus 28 bytes when encrypted) at open, so raise that on the chunk source too. */
     readonly maxBitmapBytes?: number;
     /**
      * The store's read retry, for the reads the rewrite makes along the way: the generation it rewrites and each of its

package/dist/core/metrics.d.ts

@@ -26,7 +26,7 @@ export type MetricEvent = {
     readonly kind: 'storage.get';
     readonly namespace?: string;
     readonly segment: string;
-    /** Bytes returned (0 if the chunk was absent — a GET still happened). */
+    /** One event per request, however many callers were waiting on it. Bytes returned (0 if the chunk was absent — a GET still happened). */
     readonly bytes: number;
     /**
      * Elapsed wall time of the read — includes any transient-retry backoff on the storage call. From the
@@ -34,6 +34,12 @@ export type MetricEvent = {
      */
     readonly ms: number;
 } | {
+    /**
+     * One decoded-chunk cache lookup (emitted only when a cache is configured). `hit: false` means the lookup found
+     * no cached chunk. A caller that missed while another caller's read of the same chunk was open waits on that
+     * read and adds no `storage.get`, so the misses can outnumber the `storage.get` events; the `storage.get`
+     * count is the number of requests.
+     */
     readonly kind: 'cache';
     readonly hit: boolean;
 } | {

package/dist/index.js

@@ -102,6 +102,110 @@ async function collectWithinBudget(source, budget, op) {
   return out;
 }
 
+// src/core/chunk-window.ts
+var ChunkWindow = class {
+  constructor(keys, fetch, max, ramp) {
+    this.keys = keys;
+    this.fetch = fetch;
+    this.max = max;
+    this.ramp = ramp;
+  }
+  keys;
+  fetch;
+  max;
+  ramp;
+  open = [];
+  launched = 0;
+  taken = 0;
+  /** The next read's result in key order, or the error that read raised. */
+  async take() {
+    const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
+    while (this.launched < this.keys.length && this.launched - this.taken < width) {
+      this.open.push(
+        this.fetch(this.keys[this.launched++]).then(
+          (value) => ({ value }),
+          (cause) => ({ value: void 0, error: { cause } })
+        )
+      );
+    }
+    const slot = await this.open.shift();
+    this.taken += 1;
+    if (slot.error) throw slot.error.cause;
+    return slot.value;
+  }
+};
+
+// src/core/crbm/format.ts
+var MAGIC = Uint8Array.of(67, 82, 66, 77);
+var VERSION_MAJOR = 1;
+var VERSION_MINOR = 0;
+var PREAMBLE_BYTES = 8;
+var PAYLOAD_START = PREAMBLE_BYTES;
+var FOOTER_BYTES = 104;
+var FOOTER = {
+  indexOffset: 0,
+  // u64
+  indexLength: 8,
+  // u64
+  indexCrc32c: 16,
+  // u32
+  flags: 20,
+  // u32
+  payloadCodecId: 24,
+  // u16 — which codec wrote the chunk payloads
+  elementWidth: 26,
+  // u8
+  containerCodec: 27,
+  // u8
+  versionMajor: 28,
+  // u8
+  versionMinor: 29,
+  // u8
+  reserved2: 30,
+  // u16
+  generation: 32,
+  // u64
+  indexNonce: 40,
+  // 12 B
+  indexTag: 52,
+  // 16 B
+  keyId: 68,
+  // 16 B
+  chunkCount: 84,
+  // u32
+  totalCardinality: 88,
+  // u64
+  footerCrc32c: 96,
+  // u32 — covers footer bytes [0, 96)
+  endMagic: 100
+  // char[4]
+};
+var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
+var FLAG_ENCRYPTED = 1 << 0;
+var FLAG_INDEX_COMPRESSED = 1 << 1;
+var FLAG_LITTLE_ENDIAN = 1 << 2;
+var FLAG_EXTENSION = 1 << 3;
+var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
+var ELEMENT_WIDTH_32 = 32;
+var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
+var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
+  PAYLOAD_CODEC_ROARING_PORTABLE
+]);
+var CONTAINER_CODEC_NONE = 0;
+var DEFAULT_TAIL_BYTES = 256 * 1024;
+var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
+var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
+var MAX_CHUNK_CARDINALITY = 65536;
+var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
+var EXT_TRAILER_BYTES = 12;
+var EXT_SECTION_HEADER_BYTES = 5;
+var MAX_EXT_BYTES = 4 * 1024;
+var EXT_SECTION_METADATA = 1;
+var CRC32C_BYTES = 4;
+var AEAD_NONCE_BYTES = 12;
+var AEAD_TAG_BYTES = 16;
+var DEFAULT_MAX_PAYLOAD_BYTES = DEFAULT_MAX_BITMAP_BYTES + AEAD_NONCE_BYTES + AEAD_TAG_BYTES;
+
 // src/core/keys.ts
 var FIELD = " ";
 var NO_NAMESPACE = "/";
@@ -214,8 +318,8 @@ var CountingMetricsSink = class _CountingMetricsSink {
 };
 
 // src/core/engine.ts
-var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
-var DEFAULT_INTERSECT_CONCURRENCY = 8;
+var DEFAULT_INTERSECT_CONCURRENCY = 32;
+var COMBINE_WINDOW_START = 8;
 var ZERO_CLOCK = { now: () => 0 };
 function windowOf(range) {
   const after = range?.after;
@@ -268,37 +372,6 @@ function keysWithin(keys, w) {
   };
   return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
 }
-var ChunkWindow = class {
-  constructor(keys, fetch, max, ramp) {
-    this.keys = keys;
-    this.fetch = fetch;
-    this.max = max;
-    this.ramp = ramp;
-  }
-  keys;
-  fetch;
-  max;
-  ramp;
-  open = [];
-  launched = 0;
-  taken = 0;
-  /** The next chunk in key order (null if the source holds none), or the error its read raised. */
-  async take() {
-    const width = this.ramp ? Math.min(this.max, 2 ** Math.min(this.taken, 30)) : this.max;
-    while (this.launched < this.keys.length && this.launched - this.taken < width) {
-      this.open.push(
-        this.fetch(this.keys[this.launched++]).then(
-          (chunk) => ({ chunk }),
-          (cause) => ({ chunk: null, error: { cause } })
-        )
-      );
-    }
-    const slot = await this.open.shift();
-    this.taken += 1;
-    if (slot.error) throw slot.error.cause;
-    return slot.chunk;
-  }
-};
 var SegmentEngine = class {
   storage;
   cache;
@@ -309,6 +382,12 @@ var SegmentEngine = class {
   metricsOn;
   /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. */
   budget;
+  /**
+   * The chunk reads open right now, by cache key: a caller that misses the cache and finds its key here awaits that
+   * read instead of making its own. An entry lives from the request until it settles, so the map holds at most one
+   * promise per distinct key in flight.
+   */
+  openReads = /* @__PURE__ */ new Map();
   constructor(deps) {
     this.storage = deps.storage;
     this.cache = deps.cache;
@@ -389,7 +468,7 @@ var SegmentEngine = class {
     );
   }
   /**
-   * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
+   * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
    * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
    * chunks the range overlaps (see {@link IdRange}).
    *
@@ -548,10 +627,17 @@ var SegmentEngine = class {
     );
     const inFlight = [];
     let next = 0;
-    while (next < common.length && inFlight.length < limit) inFlight.push(startAt(common[next++]));
+    let taken = 0;
+    const fill = () => {
+      const width = Math.min(limit, Math.max(COMBINE_WINDOW_START, 2 ** Math.min(taken, 30)));
+      while (next < common.length && inFlight.length < width)
+        inFlight.push(startAt(common[next++]));
+    };
+    fill();
     while (inFlight.length > 0) {
       const slot = await inFlight.shift();
-      if (next < common.length) inFlight.push(startAt(common[next++]));
+      taken += 1;
+      fill();
       if (slot.error !== void 0) throw slot.error;
       if (slot.result && !slot.result.isEmpty) {
         if (w !== null && isEdge(slot.key, w)) {
@@ -574,9 +660,13 @@ var SegmentEngine = class {
   async combineChunk(operands, excludes, mode, chunkKey) {
     const present = mode === "all" ? operands : operands.filter((o) => o.keys.has(chunkKey));
     if (present.length === 0) return null;
-    const chunks = await Promise.all(
-      present.map((op) => this.storageChunk({ ...op.seg, chunkKey }, op.gen))
+    const relevant = excludes.filter((e) => e.keys.has(chunkKey));
+    const together = relevant.length > 0 && (mode === "any" || present.length === 1);
+    const reads = together ? [...present, ...relevant] : present;
+    const fetched = await Promise.all(
+      reads.map((op) => this.storageChunk({ ...op.seg, chunkKey }, op.gen))
     );
+    const chunks = together ? fetched.slice(0, present.length) : fetched;
     if (mode === "all" && chunks.some((c) => c === null)) return null;
     const first = chunks[0];
     const acc = first ? first.clone() : this.codec.empty();
@@ -591,11 +681,8 @@ var SegmentEngine = class {
       }
     }
     if (acc.isEmpty) return null;
-    const relevant = excludes.filter((e) => e.keys.has(chunkKey));
     if (relevant.length > 0) {
-      const cuts = await Promise.all(
-        relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen))
-      );
+      const cuts = together ? fetched.slice(present.length) : await Promise.all(relevant.map((e) => this.storageChunk({ ...e.seg, chunkKey }, e.gen)));
       for (const cut of cuts) {
         if (cut === null) continue;
         acc.andNotInPlace(cut);
@@ -699,6 +786,7 @@ var SegmentEngine = class {
   invalidate(ref) {
     const prefix = segmentPrefix(ref);
     this.cache?.deleteWhere((key) => key.startsWith(prefix));
+    for (const key of this.openReads.keys()) if (key.startsWith(prefix)) this.openReads.delete(key);
     this.storage.invalidate?.(ref);
   }
   /** The segment's current generation, resolved once per op (`undefined` ⇒ source can't report it). */
@@ -728,7 +816,13 @@ var SegmentEngine = class {
    * `gen === undefined` ⇒ the source can't report a generation (it only ever serves one) ⇒ the key stays
    * generation-free. Superseded-generation entries age out under the LRU ceiling — no active purge.
    *
-   * The returned instance is **shared** (it may be the cached one): callers read it or clone it, never mutate it.
+   * Concurrent misses of one key share **one** request, with or without a cache: the first caller starts the read,
+   * later ones await it, and the entry goes when the read settles, so a failure is every waiting caller's and the
+   * next call reads again. The key carries the version exactly as the cache's does, so two versions of one chunk
+   * are two reads, never one.
+   *
+   * The returned instance is **shared** (it may be the cached one, and every caller that joined a read gets the same
+   * one): callers read it or clone it, never mutate it.
    */
   async storageChunk(ref, gen) {
     if (gen === null) return null;
@@ -741,6 +835,19 @@ var SegmentEngine = class {
       }
       if (this.metricsOn) this.metrics.onEvent({ kind: "cache", hit: false });
     }
+    const open4 = this.openReads.get(cacheKey);
+    if (open4) return open4.read;
+    const token = {};
+    const entry = { token, read: this.fetchChunk(ref, cacheKey, token) };
+    this.openReads.set(cacheKey, entry);
+    const settled = () => {
+      if (this.openReads.get(cacheKey) === entry) this.openReads.delete(cacheKey);
+    };
+    entry.read.then(settled, settled);
+    return entry.read;
+  }
+  /** One request for one chunk, decoded, range-checked and cached. */
+  async fetchChunk(ref, cacheKey, token) {
     const startedAt = this.metricsOn ? this.clock.now() : 0;
     const bytes = await this.storage.getChunk(ref);
     if (this.metricsOn) {
@@ -755,7 +862,7 @@ var SegmentEngine = class {
     if (!bytes) return null;
     const bitmap = this.codec.safeDeserialize(bytes, this.maxBitmapBytes);
     this.assertChunkPayloadInRange(bitmap, ref.chunkKey);
-    this.cache?.set(cacheKey, bitmap);
+    if (this.openReads.get(cacheKey)?.token === token) this.cache?.set(cacheKey, bitmap);
     return bitmap;
   }
 };
@@ -2269,76 +2376,6 @@ function aadFor(ref, generation, scope) {
   return out;
 }
 
-// src/core/crbm/format.ts
-var MAGIC = Uint8Array.of(67, 82, 66, 77);
-var VERSION_MAJOR = 1;
-var VERSION_MINOR = 0;
-var PREAMBLE_BYTES = 8;
-var PAYLOAD_START = PREAMBLE_BYTES;
-var FOOTER_BYTES = 104;
-var FOOTER = {
-  indexOffset: 0,
-  // u64
-  indexLength: 8,
-  // u64
-  indexCrc32c: 16,
-  // u32
-  flags: 20,
-  // u32
-  payloadCodecId: 24,
-  // u16 — which codec wrote the chunk payloads
-  elementWidth: 26,
-  // u8
-  containerCodec: 27,
-  // u8
-  versionMajor: 28,
-  // u8
-  versionMinor: 29,
-  // u8
-  reserved2: 30,
-  // u16
-  generation: 32,
-  // u64
-  indexNonce: 40,
-  // 12 B
-  indexTag: 52,
-  // 16 B
-  keyId: 68,
-  // 16 B
-  chunkCount: 84,
-  // u32
-  totalCardinality: 88,
-  // u64
-  footerCrc32c: 96,
-  // u32 — covers footer bytes [0, 96)
-  endMagic: 100
-  // char[4]
-};
-var FOOTER_CRC_COVERAGE = FOOTER.footerCrc32c;
-var FLAG_ENCRYPTED = 1 << 0;
-var FLAG_INDEX_COMPRESSED = 1 << 1;
-var FLAG_LITTLE_ENDIAN = 1 << 2;
-var FLAG_EXTENSION = 1 << 3;
-var KNOWN_FLAGS = FLAG_ENCRYPTED | FLAG_LITTLE_ENDIAN | FLAG_EXTENSION;
-var ELEMENT_WIDTH_32 = 32;
-var PAYLOAD_CODEC_ROARING_PORTABLE = 1;
-var KNOWN_PAYLOAD_CODEC_IDS = /* @__PURE__ */ new Set([
-  PAYLOAD_CODEC_ROARING_PORTABLE
-]);
-var CONTAINER_CODEC_NONE = 0;
-var DEFAULT_TAIL_BYTES = 256 * 1024;
-var DEFAULT_MAX_INDEX_BYTES = 8 * 1024 * 1024;
-var DEFAULT_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;
-var MAX_CHUNK_CARDINALITY = 65536;
-var EXT_MAGIC = Uint8Array.of(67, 82, 66, 88);
-var EXT_TRAILER_BYTES = 12;
-var EXT_SECTION_HEADER_BYTES = 5;
-var MAX_EXT_BYTES = 4 * 1024;
-var EXT_SECTION_METADATA = 1;
-var CRC32C_BYTES = 4;
-var AEAD_NONCE_BYTES = 12;
-var AEAD_TAG_BYTES = 16;
-
 // src/core/crbm/crc32c.ts
 var TABLES = (() => {
   const POLY = 2197175160;
@@ -2540,7 +2577,6 @@ var CrbmReader = class _CrbmReader {
   }
   static async open(blob, options = {}) {
     const tailBytes = Math.max(options.tailBytes ?? DEFAULT_TAIL_BYTES, FOOTER_BYTES);
-    const maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
     const maxIndexBytes = options.maxIndexBytes ?? DEFAULT_MAX_INDEX_BYTES;
     const { bytes: tail, size } = await blob.getTail(tailBytes);
     const { footer, fview, storedFooterCrc } = checkedFooter(tail, size);
@@ -2682,7 +2718,7 @@ var CrbmReader = class _CrbmReader {
     const index = parseIndex(
       indexForParse,
       payloadEnd,
-      maxPayloadBytes,
+      options.maxPayloadBytes ?? (encrypted ? DEFAULT_MAX_PAYLOAD_BYTES : DEFAULT_MAX_BITMAP_BYTES),
       encrypted ? AEAD_NONCE_BYTES + AEAD_TAG_BYTES : 1
     );
     if (encrypted) {
@@ -2869,7 +2905,9 @@ function parseIndex(indexBytes, payloadEnd, maxPayloadBytes, minPayloadBytes = 1
     }
     if (chunkKey > 65535) throw new IntegrityError(`.crbm chunkKey ${chunkKey} out of range`);
     if (len < minPayloadBytes || len > maxPayloadBytes) {
-      throw new IntegrityError(`.crbm chunk ${chunkKey} length ${len} invalid`);
+      throw new IntegrityError(
+        `.crbm chunk ${chunkKey} length ${len} invalid (most is ${maxPayloadBytes})`
+      );
     }
     if (card < 1 || card > MAX_CHUNK_CARDINALITY) {
       throw new IntegrityError(`.crbm chunk ${chunkKey} cardinality ${card} invalid`);
@@ -5094,12 +5132,11 @@ function describingSummary(record) {
 }
 
 // src/core/erase-id.ts
-var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
 async function eraseIdFromSegment(ref, id, deps, options = {}) {
   validateUserRef(ref);
   const { chunkKey, remainder } = splitId(id);
   const codec = requireCodec(deps.codec, "eraseIdFromSegment");
-  const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES2;
+  const maxBytes = deps.maxBitmapBytes ?? DEFAULT_MAX_BITMAP_BYTES;
   const base = { segment: ref.segment, namespace: ref.namespace };
   const read = (op) => retryRead(op, deps.readRetry);
   const record = await deps.registry.get(ref);
@@ -5311,14 +5348,21 @@ async function eraseIdFromSegment(ref, id, deps, options = {}) {
   if (left !== void 0) throw cannotRemove(left);
   return { ...base, erased: true, fromGeneration: from, generation, collected };
 }
+var REWRITE_READ_AHEAD = 32;
 async function* rewrite(reader, chunkKey, replacement, codec, maxBytes, read) {
   const keys = [...reader.chunkKeys()].sort((a, b) => a - b);
+  const window = new ChunkWindow(
+    keys.filter((k) => k !== chunkKey),
+    (k) => read(() => reader.getChunk(k)),
+    REWRITE_READ_AHEAD,
+    false
+  );
   for (const k of keys) {
     if (k === chunkKey) {
       yield { chunkKey: k, bitmap: replacement };
       continue;
     }
-    const bytes = await read(() => reader.getChunk(k));
+    const bytes = await window.take();
     if (bytes === null) continue;
     const bitmap = codec.safeDeserialize(bytes, maxBytes);
     assertRemaindersInRange(bitmap, k);