@cloudbitmaps/core 0.4.0 → 0.4.1

package/dist/index.d.cts

@@ -314,14 +314,22 @@ type MetricEvent = {
      * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
      * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
      *
-     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
-     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
-     * `skippedChunks: 0` is the expected shape, not a regression.
+     * For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
+     * every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
+     * carries `exclude` operands, because a suppression list may hold keys no include does and those are
+     * legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
      */
     readonly op?: 'intersect' | 'union' | 'andNot';
     /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
+    /**
+     * Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
+     *
+     * **Keys, not requests.** The actual cold GETs are roughly `fetchedChunks × operands`, plus one per
+     * `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
+     * charges for the same call. The two are different units by design; if you are reconciling a bill, the
+     * budget's accounting is the one that models requests.
+     */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -498,8 +506,8 @@ declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
  * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
  * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
  * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
- * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
- * counting a lost id or a surfaced error as a failure:
+ * jitter). Swept against a **modelled** backend — an in-process fake that throttles on concurrent requests in
+ * flight, not a real DynamoDB — 64 chunks, 5 trials each, counting a lost id or a surfaced error as a failure:
  *
  * ```text
  *   backend capacity:      1      2      4      8
@@ -609,6 +617,17 @@ declare class SegmentEngine {
     segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
     iterate(seg: SegmentRef): AsyncGenerator<number>;
     /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+     * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+     * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+     *
+     * ---
+     *
      * Chunk-skipping intersection. Computes
      * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
      * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -633,15 +652,6 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
-    /**
-     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-     *
-     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-     * intersected against a narrow audience costs a handful of reads, not 61,000.
-     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
@@ -730,8 +740,10 @@ declare class SegmentEngine {
      * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
      * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
      * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+     * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+     * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+     * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
      *
      * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
      * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -1055,8 +1067,16 @@ interface CompactionDeps {
     readonly cold: IColdDriver;
     readonly warm: IWarmDriver;
     readonly registry: IRegistryDriver;
-    /** Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time). */
-    readonly clock: Pick<Clock, 'now'>;
+    /**
+     * Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time).
+     *
+     * `now` is required; `sleep`/`yieldNow` are **optional and used only for cooperative yielding** while writing
+     * a generation. Supplying a full {@link Clock} (the facade does) makes compaction hand the event loop back
+     * periodically instead of blocking it for the whole write; supplying a bare `{ now }` keeps the previous
+     * uninterrupted behaviour. Optional rather than required because every existing caller passes `{ now }`, and
+     * requiring more would be a breaking change — the reason the yield below shipped unreachable the first time.
+     */
+    readonly clock: Pick<Clock, 'now'> & Partial<Pick<Clock, 'sleep' | 'yieldNow'>>;
     readonly maxBitmapBytes?: number;
     /**
      * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
@@ -1515,6 +1535,56 @@ declare class LocalFsRegistryDriver implements IRegistryDriver {
     private withRowLock;
 }
 
+/**
+ * Cooperative yielding for the long CPU-bound loops.
+ *
+ * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
+ * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
+ * to occupy. Measured on a 1M-id load, 62,455 chunks: **442 ms wall, and 450 ms during which the event loop did
+ * not turn at all**. Wire that to a request handler and every other in-flight request on the instance — health
+ * checks included — waits out the whole load.
+ *
+ * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
+ * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
+ * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
+ * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
+ * synchronous and periodically hand the loop back.
+ *
+ * TWO TRAPS, both of which make a plausible implementation measure as no change at all:
+ *
+ * 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
+ *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
+ *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
+ *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
+ *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    differ. The yield must be a **macrotask** — hence
+ *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
+ * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
+ *    large enough that its cost disappears into the batch it interrupts.
+ *
+ * At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
+ * baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
+ *
+ * **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
+ * 442 ms wall / 450 ms blocked before, 256 ms / 19 ms after. (b) This synthetic yield-primitive comparison:
+ * 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
+ * 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another is
+ * how this file previously published a 92 ms operation with 819 ms of starvation inside it.
+ */
+/**
+ * The only part of a `Clock` that yielding actually needs.
+ *
+ * Deliberately looser than `Clock` so the compaction path can opt in: `CompactionDeps.clock` is declared as
+ * `Pick<Clock, 'now'>` plus optional yield members, because its callers (including the `compact-segments` CLI)
+ * have always supplied a bare `{ now }`. Requiring a full `Clock` there would have been a breaking change for
+ * every one of them — which is exactly why the compaction yield shipped as dead code: the type could not carry
+ * a clock capable of yielding.
+ */
+interface Yielder {
+    yieldNow?(): Promise<void>;
+    sleep?(ms: number): Promise<void>;
+}
+
 interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
@@ -1704,7 +1774,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
-    clock?: Clock;
+    clock?: Yielder;
 }): Promise<{
     size: number;
     sha256: string;
@@ -2097,6 +2167,26 @@ declare class RetryingWarmDriver implements IWarmDriver {
         token: Token;
     }>;
     deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
+    /**
+     * Stream the inner driver's rows, retrying only until the first one arrives.
+     *
+     * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+     * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+     * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+     * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+     * after the entire segment was already in memory. Measured on a 500-row segment under
+     * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+     * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+     * distinguishing observable, the row count was.
+     *
+     * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+     * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+     * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+     * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+     * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+     * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+     * where re-reading from the start is safe.
+     */
     listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
         chunkKey: number;
     } & WarmRow>;

package/dist/index.d.ts

@@ -314,14 +314,22 @@ type MetricEvent = {
      * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
      * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
      *
-     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
-     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
-     * `skippedChunks: 0` is the expected shape, not a regression.
+     * For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
+     * every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
+     * carries `exclude` operands, because a suppression list may hold keys no include does and those are
+     * legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
      */
     readonly op?: 'intersect' | 'union' | 'andNot';
     /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
+    /**
+     * Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
+     *
+     * **Keys, not requests.** The actual cold GETs are roughly `fetchedChunks × operands`, plus one per
+     * `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
+     * charges for the same call. The two are different units by design; if you are reconciling a bill, the
+     * budget's accounting is the one that models requests.
+     */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -498,8 +506,8 @@ declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
  * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
  * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
  * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
- * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
- * counting a lost id or a surfaced error as a failure:
+ * jitter). Swept against a **modelled** backend — an in-process fake that throttles on concurrent requests in
+ * flight, not a real DynamoDB — 64 chunks, 5 trials each, counting a lost id or a surfaced error as a failure:
  *
  * ```text
  *   backend capacity:      1      2      4      8
@@ -609,6 +617,17 @@ declare class SegmentEngine {
     segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
     iterate(seg: SegmentRef): AsyncGenerator<number>;
     /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+     * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+     * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+     *
+     * ---
+     *
      * Chunk-skipping intersection. Computes
      * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
      * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -633,15 +652,6 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
-    /**
-     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-     *
-     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-     * intersected against a narrow audience costs a handful of reads, not 61,000.
-     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
@@ -730,8 +740,10 @@ declare class SegmentEngine {
      * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
      * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
      * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+     * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+     * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+     * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
      *
      * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
      * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -1055,8 +1067,16 @@ interface CompactionDeps {
     readonly cold: IColdDriver;
     readonly warm: IWarmDriver;
     readonly registry: IRegistryDriver;
-    /** Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time). */
-    readonly clock: Pick<Clock, 'now'>;
+    /**
+     * Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time).
+     *
+     * `now` is required; `sleep`/`yieldNow` are **optional and used only for cooperative yielding** while writing
+     * a generation. Supplying a full {@link Clock} (the facade does) makes compaction hand the event loop back
+     * periodically instead of blocking it for the whole write; supplying a bare `{ now }` keeps the previous
+     * uninterrupted behaviour. Optional rather than required because every existing caller passes `{ now }`, and
+     * requiring more would be a breaking change — the reason the yield below shipped unreachable the first time.
+     */
+    readonly clock: Pick<Clock, 'now'> & Partial<Pick<Clock, 'sleep' | 'yieldNow'>>;
     readonly maxBitmapBytes?: number;
     /**
      * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
@@ -1515,6 +1535,56 @@ declare class LocalFsRegistryDriver implements IRegistryDriver {
     private withRowLock;
 }
 
+/**
+ * Cooperative yielding for the long CPU-bound loops.
+ *
+ * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
+ * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
+ * to occupy. Measured on a 1M-id load, 62,455 chunks: **442 ms wall, and 450 ms during which the event loop did
+ * not turn at all**. Wire that to a request handler and every other in-flight request on the instance — health
+ * checks included — waits out the whole load.
+ *
+ * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
+ * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
+ * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
+ * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
+ * synchronous and periodically hand the loop back.
+ *
+ * TWO TRAPS, both of which make a plausible implementation measure as no change at all:
+ *
+ * 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
+ *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
+ *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
+ *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
+ *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    differ. The yield must be a **macrotask** — hence
+ *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
+ * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
+ *    large enough that its cost disappears into the batch it interrupts.
+ *
+ * At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
+ * baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
+ *
+ * **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
+ * 442 ms wall / 450 ms blocked before, 256 ms / 19 ms after. (b) This synthetic yield-primitive comparison:
+ * 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
+ * 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another is
+ * how this file previously published a 92 ms operation with 819 ms of starvation inside it.
+ */
+/**
+ * The only part of a `Clock` that yielding actually needs.
+ *
+ * Deliberately looser than `Clock` so the compaction path can opt in: `CompactionDeps.clock` is declared as
+ * `Pick<Clock, 'now'>` plus optional yield members, because its callers (including the `compact-segments` CLI)
+ * have always supplied a bare `{ now }`. Requiring a full `Clock` there would have been a breaking change for
+ * every one of them — which is exactly why the compaction yield shipped as dead code: the type could not carry
+ * a clock capable of yielding.
+ */
+interface Yielder {
+    yieldNow?(): Promise<void>;
+    sleep?(ms: number): Promise<void>;
+}
+
 interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
@@ -1704,7 +1774,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
-    clock?: Clock;
+    clock?: Yielder;
 }): Promise<{
     size: number;
     sha256: string;
@@ -2097,6 +2167,26 @@ declare class RetryingWarmDriver implements IWarmDriver {
         token: Token;
     }>;
     deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
+    /**
+     * Stream the inner driver's rows, retrying only until the first one arrives.
+     *
+     * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+     * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+     * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+     * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+     * after the entire segment was already in memory. Measured on a 500-row segment under
+     * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+     * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+     * distinguishing observable, the row count was.
+     *
+     * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+     * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+     * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+     * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+     * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+     * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+     * where re-reading from the start is safe.
+     */
     listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
         chunkKey: number;
     } & WarmRow>;

package/dist/index.js

@@ -142,7 +142,9 @@ async function collectWithinBudget(source, budget, op) {
 var YIELD_EVERY = 1024;
 function yieldEvery(clock, everyN = YIELD_EVERY) {
   if (clock === void 0) return () => null;
-  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : () => clock.sleep(1);
+  const sleep = clock.sleep;
+  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : sleep !== void 0 ? () => sleep.call(clock, 1) : void 0;
+  if (pause === void 0) return () => null;
   let n = 0;
   return () => {
     if (++n < everyN) return null;
@@ -339,6 +341,17 @@ var InstantClock = class {
   sleep() {
     return Promise.resolve();
   }
+  /**
+   * An explicit no-op, so the intent is stated rather than inferred.
+   *
+   * Without it, `yieldEvery` would fall back to `sleep(1)` — and this clock's `sleep` ignores its argument and
+   * resolves on a microtask, so the "yield" would silently do nothing while every doc comment claimed it did
+   * something. This wiring has no real event loop to be courteous to (it is the clock used when nobody injects
+   * one), so no-op is the right behaviour; it just has to be a decision rather than an accident.
+   */
+  yieldNow() {
+    return Promise.resolve();
+  }
 };
 var ZeroRng = class {
   next() {
@@ -456,6 +469,17 @@ var SegmentEngine = class {
     }
   }
   /**
+   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+   * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+   * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+   * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+   *
+   * ---
+   *
    * Chunk-skipping intersection. Computes
    * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
    * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -480,15 +504,6 @@ var SegmentEngine = class {
    * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
    * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
    */
-  /**
-   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-   *
-   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-   * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-   * intersected against a narrow audience costs a handful of reads, not 61,000.
-   */
   intersect(segs, options) {
     return this.combine(segs, options?.exclude ?? [], "all", "intersect", options);
   }
@@ -676,8 +691,10 @@ var SegmentEngine = class {
    * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
    * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
    * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-   * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+   * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+   * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+   * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
    *
    * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
    * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -720,8 +737,6 @@ var SegmentEngine = class {
       for (const id of ids) {
         stage(id);
         if (pending >= MUTATE_STAGE_IDS) compact();
-        const pause = tickId();
-        if (pause !== null) await pause;
       }
     }
     compact();
@@ -2312,6 +2327,8 @@ async function compactSegmentInner(ref, deps, options) {
     let tally;
     try {
       tally = await writeCrbmGenerationStream(deps.cold, gen0, stream, {
+        clock: deps.clock,
+        // cooperative if the caller's clock can yield; a no-op if it only tells the time
         crypto: cryptoAt(aead, 0)
       });
     } catch (err) {
@@ -2439,7 +2456,8 @@ async function compactSegmentInner(ref, deps, options) {
       deps.cold,
       { namespace: ref.namespace, segment: ref.segment, generation: newGen },
       stream,
-      cryptoAt(aead, newGen)
+      cryptoAt(aead, newGen),
+      deps.clock
     );
     try {
       await deps.registry.compareAndSwap(ref, leaseToken, {
@@ -2536,8 +2554,8 @@ async function* mergeChunksStream(ref, g, pinned, deps, maxBytes, readCrypto) {
     if (!merged.isEmpty) yield { chunkKey, bitmap: merged };
   }
 }
-async function stageGeneration(cold, key, chunks, crypto) {
-  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto });
+async function stageGeneration(cold, key, chunks, crypto, clock) {
+  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto, clock });
   await verifyGeneration(cold, key, tally, crypto);
 }
 async function verifyGeneration(cold, key, expected, crypto) {
@@ -3829,17 +3847,43 @@ var RetryingWarmDriver = class {
   deleteConditional(ref, expected) {
     return withRetry(() => this.inner.deleteConditional(ref, expected), this.policy, this.deps);
   }
+  /**
+   * Stream the inner driver's rows, retrying only until the first one arrives.
+   *
+   * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+   * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+   * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+   * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+   * after the entire segment was already in memory. Measured on a 500-row segment under
+   * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+   * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+   * distinguishing observable, the row count was.
+   *
+   * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+   * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+   * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+   * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+   * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+   * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+   * where re-reading from the start is safe.
+   */
   async *listChunks(ref, opts) {
-    const rows = await withRetry(
+    const started = await withRetry(
       async () => {
-        const out = [];
-        for await (const row of this.inner.listChunks(ref, opts)) out.push(row);
-        return out;
+        const iterator = this.inner.listChunks(ref, opts)[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        return { iterator, first };
       },
       this.policy,
       this.deps
     );
-    yield* rows;
+    if (started.first.done === true) return;
+    yield started.first.value;
+    for (; ; ) {
+      const next = await started.iterator.next();
+      if (next.done === true) return;
+      yield next.value;
+    }
   }
 };
 var RetryingColdChunkSource = class {