@cloudbitmaps/core 0.4.0 → 0.5.0

package/dist/index.d.cts

@@ -7,7 +7,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * `core/` is **codec-agnostic**: `SegmentEngine`, compaction, and the `.crbm` read/write helpers only ever
  * construct and combine bitmaps through the {@link CodecInterface} factory + the {@link CodecBitmap} value type
  * defined here — never a concrete implementation. The flagship codec is roaring (`roaringCodec`, today in
- * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitmap`
+ * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitset`
  * (plain bitset) and `@cloudbitmaps/soaring` plug in behind the same seam with zero engine or driver changes.
  *
  * **Homogeneity contract:** a single store uses a single codec, so every {@link CodecBitmap} an operation sees
@@ -18,7 +18,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * mints the cache per store, so this cannot arise in normal use.)
  *
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
- * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitmap` will add live on that
+ * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
  * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
  * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
@@ -314,14 +314,22 @@ type MetricEvent = {
      * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
      * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
      *
-     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
-     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
-     * `skippedChunks: 0` is the expected shape, not a regression.
+     * For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
+     * every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
+     * carries `exclude` operands, because a suppression list may hold keys no include does and those are
+     * legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
      */
     readonly op?: 'intersect' | 'union' | 'andNot';
     /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
+    /**
+     * Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
+     *
+     * **Keys, not requests.** The actual cold GETs are roughly `fetchedChunks × operands`, plus one per
+     * `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
+     * charges for the same call. The two are different units by design; if you are reconciling a bill, the
+     * budget's accounting is the one that models requests.
+     */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -498,8 +506,8 @@ declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
  * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
  * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
  * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
- * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
- * counting a lost id or a surfaced error as a failure:
+ * jitter). Swept against a **modelled** backend — an in-process fake that throttles on concurrent requests in
+ * flight, not a real DynamoDB — 64 chunks, 5 trials each, counting a lost id or a surfaced error as a failure:
  *
  * ```text
  *   backend capacity:      1      2      4      8
@@ -609,6 +617,17 @@ declare class SegmentEngine {
     segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
     iterate(seg: SegmentRef): AsyncGenerator<number>;
     /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+     * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+     * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+     *
+     * ---
+     *
      * Chunk-skipping intersection. Computes
      * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
      * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -633,15 +652,6 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
-    /**
-     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-     *
-     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-     * intersected against a narrow audience costs a handful of reads, not 61,000.
-     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
@@ -730,8 +740,10 @@ declare class SegmentEngine {
      * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
      * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
      * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+     * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+     * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+     * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
      *
      * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
      * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -1055,8 +1067,16 @@ interface CompactionDeps {
     readonly cold: IColdDriver;
     readonly warm: IWarmDriver;
     readonly registry: IRegistryDriver;
-    /** Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time). */
-    readonly clock: Pick<Clock, 'now'>;
+    /**
+     * Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time).
+     *
+     * `now` is required; `sleep`/`yieldNow` are **optional and used only for cooperative yielding** while writing
+     * a generation. Supplying a full {@link Clock} (the facade does) makes compaction hand the event loop back
+     * periodically instead of blocking it for the whole write; supplying a bare `{ now }` keeps the previous
+     * uninterrupted behaviour. Optional rather than required because every existing caller passes `{ now }`, and
+     * requiring more would be a breaking change — the reason the yield below shipped unreachable the first time.
+     */
+    readonly clock: Pick<Clock, 'now'> & Partial<Pick<Clock, 'sleep' | 'yieldNow'>>;
     readonly maxBitmapBytes?: number;
     /**
      * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
@@ -1515,6 +1535,56 @@ declare class LocalFsRegistryDriver implements IRegistryDriver {
     private withRowLock;
 }
 
+/**
+ * Cooperative yielding for the long CPU-bound loops.
+ *
+ * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
+ * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
+ * to occupy. Measured on a 1M-id load, 62,455 chunks: **442 ms wall, and 450 ms during which the event loop did
+ * not turn at all**. Wire that to a request handler and every other in-flight request on the instance — health
+ * checks included — waits out the whole load.
+ *
+ * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
+ * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
+ * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
+ * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
+ * synchronous and periodically hand the loop back.
+ *
+ * TWO TRAPS, both of which make a plausible implementation measure as no change at all:
+ *
+ * 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
+ *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
+ *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
+ *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
+ *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    differ. The yield must be a **macrotask** — hence
+ *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
+ * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
+ *    large enough that its cost disappears into the batch it interrupts.
+ *
+ * At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
+ * baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
+ *
+ * **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
+ * 442 ms wall / 450 ms blocked before, 256 ms / 19 ms after. (b) This synthetic yield-primitive comparison:
+ * 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
+ * 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another is
+ * how this file previously published a 92 ms operation with 819 ms of starvation inside it.
+ */
+/**
+ * The only part of a `Clock` that yielding actually needs.
+ *
+ * Deliberately looser than `Clock` so the compaction path can opt in: `CompactionDeps.clock` is declared as
+ * `Pick<Clock, 'now'>` plus optional yield members, because its callers (including the `compact-segments` CLI)
+ * have always supplied a bare `{ now }`. Requiring a full `Clock` there would have been a breaking change for
+ * every one of them — which is exactly why the compaction yield shipped as dead code: the type could not carry
+ * a clock capable of yielding.
+ */
+interface Yielder {
+    yieldNow?(): Promise<void>;
+    sleep?(ms: number): Promise<void>;
+}
+
 interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
@@ -1704,7 +1774,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
-    clock?: Clock;
+    clock?: Yielder;
 }): Promise<{
     size: number;
     sha256: string;
@@ -2061,11 +2131,23 @@ declare function runConsistencyCheck(deps: {
  * I/O of their own); the wrapped driver is responsible for *classifying* its transient faults (raising
  * {@link TransientError}); these decorators decide *whether and when* to retry.
  *
- * **Streaming methods** (`listChunks`, `list`) are retried by **re-enumerating from the start**, buffering
- * the result — a partially-consumed async iterator can't be safely resumed mid-stream (it would re-yield
- * earlier items). Their consumers in the engine already collect fully (the Warm dirty-set is small under
- * Topology-A; the cold `list` is a discovery scan), so buffering preserves current semantics. Point methods
- * are retried in place with no buffering.
+ * **Streaming methods split into two shapes**, and the difference is deliberate — a partially-consumed async
+ * iterator cannot be resumed mid-stream (it would re-yield earlier items), so each method picks between
+ * buffering and bounded memory:
+ *
+ *   · `RetryingColdDriver.list` / `RetryingRegistryDriver.list` **buffer**, inside `withRetry`, so the whole
+ *     enumeration is retried by re-running it from the start. Both are discovery scans over a generation or
+ *     namespace listing, where the result set is small and whole-scan retry is worth the memory.
+ *   · `RetryingWarmDriver.listChunks` does **not** buffer. It retries only until the first row arrives and
+ *     then streams live, because buffering it defeated the engine's resident-memory bound outright — see the
+ *     measured account on the method itself. A mid-stream fault therefore propagates rather than being
+ *     retried, which is the documented trade there.
+ *
+ * Point methods are retried in place, with nothing to buffer.
+ *
+ * (This paragraph previously said streaming methods buffer, full stop. That described `listChunks` as it was
+ * before the bound was fixed, and the stale wording had also been copied into the Postgres and MySQL warm
+ * drivers — so three comments promised a mid-stream resilience that the hot read path does not have.)
  *
  * What is **not** retried here: {@link WriteConflictError} (OCC — the engine's read-modify-write loop owns
  * that; a blind replay would re-apply against a stale token), and every deterministic error
@@ -2097,6 +2179,26 @@ declare class RetryingWarmDriver implements IWarmDriver {
         token: Token;
     }>;
     deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
+    /**
+     * Stream the inner driver's rows, retrying only until the first one arrives.
+     *
+     * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+     * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+     * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+     * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+     * after the entire segment was already in memory. Measured on a 500-row segment under
+     * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+     * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+     * distinguishing observable, the row count was.
+     *
+     * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+     * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+     * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+     * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+     * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+     * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+     * where re-reading from the start is safe.
+     */
     listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
         chunkKey: number;
     } & WarmRow>;

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * `core/` is **codec-agnostic**: `SegmentEngine`, compaction, and the `.crbm` read/write helpers only ever
  * construct and combine bitmaps through the {@link CodecInterface} factory + the {@link CodecBitmap} value type
  * defined here — never a concrete implementation. The flagship codec is roaring (`roaringCodec`, today in
- * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitmap`
+ * `core/bitmap.ts`; it moves to `@cloudbitmaps/roaring` when the package split lands); `@cloudbitmaps/bitset`
  * (plain bitset) and `@cloudbitmaps/soaring` plug in behind the same seam with zero engine or driver changes.
  *
  * **Homogeneity contract:** a single store uses a single codec, so every {@link CodecBitmap} an operation sees
@@ -18,7 +18,7 @@ export { o as BufferReader, p as BufferSink, q as GovernanceMeta, r as NO_ROW, s
  * mints the cache per store, so this cannot arise in normal use.)
  *
  * **Interface surface = exactly what the engine needs**, kept general enough for the known fast-follow codecs
- * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitmap` will add live on that
+ * (positional/rank-select access + raw-bitset interop that `@cloudbitmaps/bitset` will add live on that
  * package's own extended value type, not here — the engine never calls them). The one codec-linked concern
  * this seam intentionally leaves in the format layer is the `.crbm` **serialization id** (`ROARING_PORTABLE_ID`
  * in `crbm/format.ts`): it is stamped in the footer and validated on read, and generalizes to a per-codec id
@@ -314,14 +314,22 @@ type MetricEvent = {
      * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
      * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
      *
-     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
-     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
-     * `skippedChunks: 0` is the expected shape, not a regression.
+     * For `'union'` over its *include* operands alone, `skippedChunks` is 0 by construction — union reads
+     * every chunk of every operand, so there is nothing to prune. It can still be non-zero when a union
+     * carries `exclude` operands, because a suppression list may hold keys no include does and those are
+     * legitimately never fetched. So: 0 is the expected shape for a plain union, not a guarantee.
      */
     readonly op?: 'intersect' | 'union' | 'andNot';
     /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
+    /**
+     * Distinct chunk **keys** selected to fetch — for `'intersect'`, those present in every operand.
+     *
+     * **Keys, not requests.** The actual cold GETs are roughly `fetchedChunks × operands`, plus one per
+     * `exclude` that holds a given key, which is why this number is smaller than what the per-op budget
+     * charges for the same call. The two are different units by design; if you are reconciling a bill, the
+     * budget's accounting is the one that models requests.
+     */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -498,8 +506,8 @@ declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
  * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
  * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
  * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
- * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
- * counting a lost id or a surfaced error as a failure:
+ * jitter). Swept against a **modelled** backend — an in-process fake that throttles on concurrent requests in
+ * flight, not a real DynamoDB — 64 chunks, 5 trials each, counting a lost id or a surfaced error as a failure:
  *
  * ```text
  *   backend capacity:      1      2      4      8
@@ -609,6 +617,17 @@ declare class SegmentEngine {
     segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
     iterate(seg: SegmentRef): AsyncGenerator<number>;
     /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+     * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+     * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+     *
+     * ---
+     *
      * Chunk-skipping intersection. Computes
      * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
      * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -633,15 +652,6 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
-    /**
-     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-     *
-     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-     * intersected against a narrow audience costs a handful of reads, not 61,000.
-     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
@@ -730,8 +740,10 @@ declare class SegmentEngine {
      * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
      * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
      * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+     * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+     * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+     * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
      *
      * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
      * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -1055,8 +1067,16 @@ interface CompactionDeps {
     readonly cold: IColdDriver;
     readonly warm: IWarmDriver;
     readonly registry: IRegistryDriver;
-    /** Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time). */
-    readonly clock: Pick<Clock, 'now'>;
+    /**
+     * Injected time source for lease expiry (the determinism seam — `core/` never reads ambient time).
+     *
+     * `now` is required; `sleep`/`yieldNow` are **optional and used only for cooperative yielding** while writing
+     * a generation. Supplying a full {@link Clock} (the facade does) makes compaction hand the event loop back
+     * periodically instead of blocking it for the whole write; supplying a bare `{ now }` keeps the previous
+     * uninterrupted behaviour. Optional rather than required because every existing caller passes `{ now }`, and
+     * requiring more would be a breaking change — the reason the yield below shipped unreachable the first time.
+     */
+    readonly clock: Pick<Clock, 'now'> & Partial<Pick<Clock, 'sleep' | 'yieldNow'>>;
     readonly maxBitmapBytes?: number;
     /**
      * Bitmap codec — compaction only merges bitmaps through it. Optional in the type so this
@@ -1515,6 +1535,56 @@ declare class LocalFsRegistryDriver implements IRegistryDriver {
     private withRowLock;
 }
 
+/**
+ * Cooperative yielding for the long CPU-bound loops.
+ *
+ * Everything this library does on the hot read path is small. Bulk-load is the exception: it is the one place
+ * where a single call legitimately occupies the CPU for hundreds of milliseconds, and Node has exactly one thread
+ * to occupy. Measured on a 1M-id load, 62,455 chunks: **442 ms wall, and 450 ms during which the event loop did
+ * not turn at all**. Wire that to a request handler and every other in-flight request on the instance — health
+ * checks included — waits out the whole load.
+ *
+ * The fix is not to make the work asynchronous. That was measured too, and it is a 7x *regression*: handing each
+ * chunk's insert to the threadpool costs ~9 µs of dispatch against ~1.5 µs of actual work once ids are spread
+ * across ~62,000 chunks (636 ms versus 92 ms). The work is genuinely CPU-bound and genuinely small per chunk;
+ * the only thing wrong with it is that it never stops to let anyone else go. So the fix is to keep the work
+ * synchronous and periodically hand the loop back.
+ *
+ * TWO TRAPS, both of which make a plausible implementation measure as no change at all:
+ *
+ * 1. **`await Promise.resolve()` — and equally `Clock.sleep(0)` — do nothing.** Both resolve on a microtask, and
+ *    the microtask queue drains to empty before the event loop advances a single phase. A loop awaiting them
+ *    yields to other *microtasks* and to nothing else; no pending I/O callback runs. Measured: 555 ms of
+ *    starvation against a 568 ms unyielded baseline **on the synthetic yield-primitive benchmark below** —
+ *    a different, smaller workload than the 442 ms end-to-end load above, which is why the two baselines
+ *    differ. The yield must be a **macrotask** — hence
+ *    `Clock.yieldNow`, backed by `setImmediate` in production wiring.
+ * 2. **Yielding per unit of work costs more than the work.** The yield has to be periodic, and `everyN` has to be
+ *    large enough that its cost disappears into the batch it interrupts.
+ *
+ * At `everyN = 1024` over that same synthetic workload the trade is close to free: 569 ms wall against a 568 ms
+ * baseline, with the worst event-loop gap down from 568 ms to 13.8 ms.
+ *
+ * **Three experiments, three sets of numbers — do not mix them.** (a) The end-to-end 1M-id bulk load:
+ * 442 ms wall / 450 ms blocked before, 256 ms / 19 ms after. (b) This synthetic yield-primitive comparison:
+ * 568 ms / 568 ms unyielded, 569 ms / 13.8 ms with `setImmediate`. (c) A per-chunk insert microbenchmark:
+ * 92 ms sync versus 636 ms via the threadpool. Quoting a baseline from one against a result from another is
+ * how this file previously published a 92 ms operation with 819 ms of starvation inside it.
+ */
+/**
+ * The only part of a `Clock` that yielding actually needs.
+ *
+ * Deliberately looser than `Clock` so the compaction path can opt in: `CompactionDeps.clock` is declared as
+ * `Pick<Clock, 'now'>` plus optional yield members, because its callers (including the `compact-segments` CLI)
+ * have always supplied a bare `{ now }`. Requiring a full `Clock` there would have been a breaking change for
+ * every one of them — which is exactly why the compaction yield shipped as dead code: the type could not carry
+ * a clock capable of yielding.
+ */
+interface Yielder {
+    yieldNow?(): Promise<void>;
+    sleep?(ms: number): Promise<void>;
+}
+
 interface CrbmReaderOptions {
     /** Speculative tail size in bytes (default 256 KB; clamped up to at least the footer size). */
     readonly tailBytes?: number;
@@ -1704,7 +1774,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
-    clock?: Clock;
+    clock?: Yielder;
 }): Promise<{
     size: number;
     sha256: string;
@@ -2061,11 +2131,23 @@ declare function runConsistencyCheck(deps: {
  * I/O of their own); the wrapped driver is responsible for *classifying* its transient faults (raising
  * {@link TransientError}); these decorators decide *whether and when* to retry.
  *
- * **Streaming methods** (`listChunks`, `list`) are retried by **re-enumerating from the start**, buffering
- * the result — a partially-consumed async iterator can't be safely resumed mid-stream (it would re-yield
- * earlier items). Their consumers in the engine already collect fully (the Warm dirty-set is small under
- * Topology-A; the cold `list` is a discovery scan), so buffering preserves current semantics. Point methods
- * are retried in place with no buffering.
+ * **Streaming methods split into two shapes**, and the difference is deliberate — a partially-consumed async
+ * iterator cannot be resumed mid-stream (it would re-yield earlier items), so each method picks between
+ * buffering and bounded memory:
+ *
+ *   · `RetryingColdDriver.list` / `RetryingRegistryDriver.list` **buffer**, inside `withRetry`, so the whole
+ *     enumeration is retried by re-running it from the start. Both are discovery scans over a generation or
+ *     namespace listing, where the result set is small and whole-scan retry is worth the memory.
+ *   · `RetryingWarmDriver.listChunks` does **not** buffer. It retries only until the first row arrives and
+ *     then streams live, because buffering it defeated the engine's resident-memory bound outright — see the
+ *     measured account on the method itself. A mid-stream fault therefore propagates rather than being
+ *     retried, which is the documented trade there.
+ *
+ * Point methods are retried in place, with nothing to buffer.
+ *
+ * (This paragraph previously said streaming methods buffer, full stop. That described `listChunks` as it was
+ * before the bound was fixed, and the stale wording had also been copied into the Postgres and MySQL warm
+ * drivers — so three comments promised a mid-stream resilience that the hot read path does not have.)
  *
  * What is **not** retried here: {@link WriteConflictError} (OCC — the engine's read-modify-write loop owns
  * that; a blind replay would re-apply against a stale token), and every deterministic error
@@ -2097,6 +2179,26 @@ declare class RetryingWarmDriver implements IWarmDriver {
         token: Token;
     }>;
     deleteConditional(ref: ChunkRef, expected: Token): Promise<void>;
+    /**
+     * Stream the inner driver's rows, retrying only until the first one arrives.
+     *
+     * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+     * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+     * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+     * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+     * after the entire segment was already in memory. Measured on a 500-row segment under
+     * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+     * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+     * distinguishing observable, the row count was.
+     *
+     * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+     * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+     * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+     * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+     * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+     * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+     * where re-reading from the start is safe.
+     */
     listChunks(ref: SegmentRef, opts?: WarmReadOptions): AsyncIterable<{
         chunkKey: number;
     } & WarmRow>;