@cloudbitmaps/core 0.3.0 → 0.4.1

package/dist/index.cjs

@@ -91,6 +91,14 @@ function applyRemove(delta, remainder) {
   delta.removes.add(remainder);
   delta.adds.remove(remainder);
 }
+function applyAddAll(delta, ids) {
+  delta.adds.orInPlace(ids);
+  delta.removes.andNotInPlace(ids);
+}
+function applyRemoveAll(delta, ids) {
+  delta.removes.orInPlace(ids);
+  delta.adds.andNotInPlace(ids);
+}
 function effective(cold, delta) {
   const out = cold.clone();
   out.orInPlace(delta.adds);
@@ -214,6 +222,21 @@ async function mapWithConcurrency(items, limit, fn) {
   return results;
 }
 
+// src/core/cooperative.ts
+var YIELD_EVERY = 1024;
+function yieldEvery(clock, everyN = YIELD_EVERY) {
+  if (clock === void 0) return () => null;
+  const sleep = clock.sleep;
+  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : sleep !== void 0 ? () => sleep.call(clock, 1) : void 0;
+  if (pause === void 0) return () => null;
+  let n = 0;
+  return () => {
+    if (++n < everyN) return null;
+    n = 0;
+    return pause();
+  };
+}
+
 // src/core/keys.ts
 var FIELD = " ";
 var NO_NAMESPACE = "/";
@@ -396,6 +419,8 @@ async function withRetry(op, policy, deps) {
 var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
 var DEFAULT_MAX_RETRIES = 16;
 var DEFAULT_INTERSECT_CONCURRENCY = 8;
+var MUTATE_STAGE_IDS = 1 << 20;
+var YIELD_EVERY_IDS = 1 << 14;
 var InstantClock = class {
   now() {
     return 0;
@@ -403,6 +428,17 @@ var InstantClock = class {
   sleep() {
     return Promise.resolve();
   }
+  /**
+   * An explicit no-op, so the intent is stated rather than inferred.
+   *
+   * Without it, `yieldEvery` would fall back to `sleep(1)` — and this clock's `sleep` ignores its argument and
+   * resolves on a microtask, so the "yield" would silently do nothing while every doc comment claimed it did
+   * something. This wiring has no real event loop to be courteous to (it is the clock used when nobody injects
+   * one), so no-op is the right behaviour; it just has to be a decision rather than an accident.
+   */
+  yieldNow() {
+    return Promise.resolve();
+  }
 };
 var ZeroRng = class {
   next() {
@@ -410,6 +446,7 @@ var ZeroRng = class {
   }
 };
 var DEFAULT_MAX_WARM_SCAN_BYTES = 64 * 1024 * 1024;
+var DEFAULT_WRITE_CONCURRENCY = 4;
 var SegmentEngine = class {
   warm;
   cold;
@@ -439,7 +476,7 @@ var SegmentEngine = class {
     this.rng = deps.rng ?? new ZeroRng();
     this.occBackoff = deps.occBackoff ?? DEFAULT_OCC_BACKOFF;
     this.readOpts = { consistent: (deps.warmReadConsistency ?? "strong") !== "eventual" };
-    this.writeConcurrency = deps.writeConcurrency ?? 1;
+    this.writeConcurrency = deps.writeConcurrency ?? DEFAULT_WRITE_CONCURRENCY;
     this.maxWarmScanBytes = deps.maxWarmScanBytes ?? DEFAULT_MAX_WARM_SCAN_BYTES;
     if (!Number.isFinite(this.maxWarmScanBytes) || this.maxWarmScanBytes < 1) {
       throw new ValidationError(
@@ -459,10 +496,10 @@ var SegmentEngine = class {
     return this.readModifyWrite(seg, chunkKey, (d) => applyRemove(d, remainder));
   }
   addMany(seg, ids) {
-    return this.mutateMany(seg, ids, applyAdd);
+    return this.mutateMany(seg, ids, applyAddAll);
   }
   removeMany(seg, ids) {
-    return this.mutateMany(seg, ids, applyRemove);
+    return this.mutateMany(seg, ids, applyRemoveAll);
   }
   async has(seg, id, opts) {
     const { chunkKey, remainder } = splitId(id);
@@ -519,6 +556,17 @@ var SegmentEngine = class {
     }
   }
   /**
+   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+   * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+   * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+   * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+   *
+   * ---
+   *
    * Chunk-skipping intersection. Computes
    * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
    * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -543,9 +591,35 @@ var SegmentEngine = class {
    * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
    * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
    */
-  async *intersect(segs, options) {
+  intersect(segs, options) {
+    return this.combine(segs, options?.exclude ?? [], "all", "intersect", options);
+  }
+  /**
+   * `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
+   * than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
+   * has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
+   * budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
+   */
+  union(segs, options) {
+    return this.combine(segs, options?.exclude ?? [], "any", "union", options);
+  }
+  /**
+   * `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
+   *
+   * Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
+   * the cost scales with the segment being filtered rather than with the size of the suppression list.
+   */
+  andNot(seg, excludes, options) {
+    return this.combine([seg], excludes, "all", "andNot", options);
+  }
+  async *combine(segs, excludeSegs, mode, op, options) {
     if (segs.length === 0) {
-      throw new ValidationError("intersect requires at least one segment");
+      throw new ValidationError(`${op} requires at least one segment`);
+    }
+    if (op === "andNot" && excludeSegs.length === 0) {
+      throw new ValidationError("andNot requires at least one segment to subtract");
     }
     const limit = options?.concurrency ?? DEFAULT_INTERSECT_CONCURRENCY;
     if (!Number.isInteger(limit) || limit < 1) {
@@ -554,34 +628,52 @@ var SegmentEngine = class {
       );
     }
     const budget = resolvePerOpBudget(options?.budget, this.budget);
-    const operands = await Promise.all(
-      segs.map(async (seg) => {
-        const warmRows = await this.collectWarm(seg, "intersect", null);
-        const keys = await this.chunkKeys(seg, warmRows);
-        const gen = await this.currentGen(seg);
-        return { seg, warmRows, keys: new Set(keys), gen };
-      })
-    );
-    const pivot = operands.reduce((a, b) => a.keys.size <= b.keys.size ? a : b);
-    const common = [];
-    for (const k of pivot.keys) {
-      if (operands.every((op) => op.keys.has(k))) common.push(k);
-    }
-    common.sort((a, b) => a - b);
-    checkBudget(budget, common.length * operands.length, "intersect");
+    const extract = async (seg) => {
+      const warmRows = await this.collectWarm(seg, op, null);
+      const keys = await this.chunkKeys(seg, warmRows);
+      const gen = await this.currentGen(seg);
+      return { seg, warmRows, keys: new Set(keys), gen };
+    };
+    const [operands, excludes] = await Promise.all([
+      Promise.all(segs.map(extract)),
+      Promise.all(excludeSegs.map(extract))
+    ]);
+    const candidates = [];
+    if (mode === "all") {
+      const pivot = operands.reduce((a, b) => a.keys.size <= b.keys.size ? a : b);
+      for (const k of pivot.keys) {
+        if (operands.every((o) => o.keys.has(k))) candidates.push(k);
+      }
+    } else {
+      const seen = /* @__PURE__ */ new Set();
+      for (const o of operands) for (const k of o.keys) seen.add(k);
+      for (const k of seen) candidates.push(k);
+    }
+    candidates.sort((a, b) => a - b);
+    const common = candidates;
+    let units = 0;
+    for (const key of common) {
+      for (const o of operands) if (mode === "all" || o.keys.has(key)) units += 1;
+      for (const e of excludes) if (e.keys.has(key)) units += 1;
+    }
+    checkBudget(budget, units, op);
     if (this.metricsOn) {
       const distinctKeys = /* @__PURE__ */ new Set();
-      for (const op of operands) {
-        for (const k of op.keys) distinctKeys.add(k);
+      for (const op2 of operands) {
+        for (const k of op2.keys) distinctKeys.add(k);
       }
+      for (const e of excludes) for (const k of e.keys) distinctKeys.add(k);
       this.metrics.onEvent({
         kind: "intersect",
-        operands: segs.length,
+        op,
+        operands: segs.length + excludes.length,
         fetchedChunks: common.length,
+        // Never negative: under 'any' every candidate key came from some operand, so `distinctKeys` is a
+        // superset of `common`; excludes only ever add keys here.
         skippedChunks: distinctKeys.size - common.length
       });
     }
-    const startAt = (key) => this.intersectChunk(operands, key).then(
+    const startAt = (key) => this.combineChunk(operands, excludes, mode, key).then(
       (result) => ({ key, result }),
       (error) => ({ key, result: null, error })
     );
@@ -603,16 +695,33 @@ var SegmentEngine = class {
    * first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
    * a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
    */
-  async intersectChunk(operands, chunkKey) {
+  async combineChunk(operands, excludes, mode, chunkKey) {
+    const present = mode === "all" ? operands : operands.filter((o) => o.keys.has(chunkKey));
+    if (present.length === 0) return null;
     const chunks = await Promise.all(
-      operands.map((op) => this.effectiveChunk(op.seg, chunkKey, op.warmRows, op.gen))
+      present.map((op) => this.effectiveChunk(op.seg, chunkKey, op.warmRows, op.gen))
     );
     const acc = chunks[0];
     for (let i = 1; i < chunks.length; i++) {
-      acc.andInPlace(chunks[i]);
-      if (acc.isEmpty) return null;
+      if (mode === "all") {
+        acc.andInPlace(chunks[i]);
+        if (acc.isEmpty) return null;
+      } else {
+        acc.orInPlace(chunks[i]);
+      }
     }
-    return acc.isEmpty ? null : acc;
+    if (acc.isEmpty) return null;
+    const relevant = excludes.filter((e) => e.keys.has(chunkKey));
+    if (relevant.length > 0) {
+      const cuts = await Promise.all(
+        relevant.map((e) => this.effectiveChunk(e.seg, chunkKey, e.warmRows, e.gen))
+      );
+      for (const cut of cuts) {
+        acc.andNotInPlace(cut);
+        if (acc.isEmpty) return null;
+      }
+    }
+    return acc;
   }
   /**
    * Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
@@ -620,9 +729,30 @@ var SegmentEngine = class {
    * `dest` is added to, not replaced.
    */
   async intersectInto(dest, segs, options) {
-    const batchSize = Math.max(1, options?.batchSize ?? 4096);
+    await this.drainInto(dest, this.intersect(segs, options), options?.batchSize);
+  }
+  /**
+   * Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
+   * {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
+   */
+  async unionInto(dest, segs, options) {
+    await this.drainInto(dest, this.union(segs, options), options?.batchSize);
+  }
+  /** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
+  async andNotInto(dest, seg, excludes, options) {
+    await this.drainInto(dest, this.andNot(seg, excludes, options), options?.batchSize);
+  }
+  /**
+   * Stream a combine's results into `dest` in bounded batches.
+   *
+   * `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
+   * mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
+   * large the result is.
+   */
+  async drainInto(dest, source, batchSizeOption) {
+    const batchSize = Math.max(1, batchSizeOption ?? 4096);
     let batch = [];
-    for await (const id of this.intersect(segs, options)) {
+    for await (const id of source) {
       batch.push(id);
       if (batch.length >= batchSize) {
         await this.addMany(dest, batch);
@@ -631,19 +761,77 @@ var SegmentEngine = class {
     }
     if (batch.length > 0) await this.addMany(dest, batch);
   }
+  /**
+   * Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
+   *
+   * Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
+   * hand-batching `page → addMany(page)`.
+   *
+   * **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
+   * input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
+   * the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
+   * order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
+   * **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
+   * multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
+   * different bug.
+   *
+   * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
+   * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
+   * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
+   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+   * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+   * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+   * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
+   *
+   * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
+   * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
+   * large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
+   * thousands of warm rows. The crossover is documented in the guide.
+   */
   async mutateMany(seg, ids, apply) {
     const byChunk = /* @__PURE__ */ new Map();
-    for (const id of ids) {
+    const staged = /* @__PURE__ */ new Map();
+    let pending = 0;
+    const tickId = yieldEvery(this.clock, YIELD_EVERY_IDS);
+    const compact = () => {
+      for (const [chunkKey, rems] of staged) {
+        if (rems.length === 0) continue;
+        const existing = byChunk.get(chunkKey);
+        if (existing === void 0) byChunk.set(chunkKey, this.codec.fromValues(rems));
+        else existing.addMany(rems);
+        rems.length = 0;
+      }
+      pending = 0;
+    };
+    const stage = (id) => {
       const { chunkKey, remainder } = splitId(id);
-      const rems = byChunk.get(chunkKey) ?? [];
+      let rems = staged.get(chunkKey);
+      if (rems === void 0) {
+        rems = [];
+        staged.set(chunkKey, rems);
+      }
       rems.push(remainder);
-      byChunk.set(chunkKey, rems);
+      pending += 1;
+    };
+    if (Symbol.asyncIterator in ids) {
+      for await (const id of ids) {
+        stage(id);
+        if (pending >= MUTATE_STAGE_IDS) compact();
+        const pause = tickId();
+        if (pause !== null) await pause;
+      }
+    } else {
+      for (const id of ids) {
+        stage(id);
+        if (pending >= MUTATE_STAGE_IDS) compact();
+      }
     }
+    compact();
     await mapWithConcurrency(
       [...byChunk],
       this.writeConcurrency,
-      ([chunkKey, rems]) => this.readModifyWrite(seg, chunkKey, (d) => {
-        for (const r of rems) apply(d, r);
+      ([chunkKey, bitmap]) => this.readModifyWrite(seg, chunkKey, (d) => {
+        apply(d, bitmap);
       })
     );
   }
@@ -1772,6 +1960,7 @@ function coldBlobReader(driver, key) {
 }
 var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
 var BULK_FLUSH_IDS = 1 << 20;
+var YIELD_EVERY_IDS2 = 1 << 14;
 var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
 var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
 var CrbmColdChunkSource = class {
@@ -1990,11 +2179,14 @@ var CrbmColdChunkSource = class {
 };
 function writeCrbmGeneration(driver, key, chunks, options = {}) {
   const sorted = [...chunks].sort((a, b) => a.chunkKey - b.chunkKey);
+  const tick = yieldEvery(options.clock);
   return driver.putImmutable(key, async (sink) => {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for (const { chunkKey, bitmap } of sorted) {
       if (bitmap.isEmpty) continue;
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
+      const pause = tick();
+      if (pause !== null) await pause;
     }
     await writer.finish();
   });
@@ -2002,6 +2194,7 @@ function writeCrbmGeneration(driver, key, chunks, options = {}) {
 async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
   const chunkKeys = [];
   let cardinality = 0;
+  const tick = yieldEvery(options.clock);
   const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for await (const { chunkKey, bitmap } of chunks) {
@@ -2009,6 +2202,8 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       chunkKeys.push(chunkKey);
       cardinality += bitmap.size;
+      const pause = tick();
+      if (pause !== null) await pause;
     }
     await writer.finish();
   });
@@ -2051,17 +2246,21 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
   const byChunk = /* @__PURE__ */ new Map();
   const pendingByChunk = /* @__PURE__ */ new Map();
   let pending = 0;
-  const flushPending = () => {
+  const tickChunk = yieldEvery(options.clock);
+  const tickId = yieldEvery(options.clock, YIELD_EVERY_IDS2);
+  const flushPending = async () => {
     for (const [chunkKey, rems] of pendingByChunk) {
       if (rems.length === 0) continue;
       const existingBitmap = byChunk.get(chunkKey);
       if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
       else existingBitmap.addMany(rems);
       rems.length = 0;
+      const pause = tickChunk();
+      if (pause !== null) await pause;
     }
     pending = 0;
   };
-  for await (const id of toAsyncIterable(ids)) {
+  const ingest = (id) => {
     const { chunkKey, remainder } = splitId(id);
     let bucket = pendingByChunk.get(chunkKey);
     if (bucket === void 0) {
@@ -2069,14 +2268,29 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
       pendingByChunk.set(chunkKey, bucket);
     }
     bucket.push(remainder);
-    if (++pending >= BULK_FLUSH_IDS) flushPending();
+    return ++pending >= BULK_FLUSH_IDS;
+  };
+  if (Symbol.asyncIterator in ids) {
+    for await (const id of ids) {
+      if (ingest(id)) await flushPending();
+      const pause = tickId();
+      if (pause !== null) await pause;
+    }
+  } else {
+    for (const id of ids) {
+      if (ingest(id)) await flushPending();
+      const pause = tickId();
+      if (pause !== null) await pause;
+    }
   }
-  flushPending();
+  await flushPending();
   let cardinality = 0;
   const chunks = [];
   for (const [chunkKey, bitmap] of byChunk) {
     cardinality += bitmap.size;
     chunks.push({ chunkKey, bitmap });
+    const pause = tickChunk();
+    if (pause !== null) await pause;
   }
   if (options.keystore !== void 0 && options.registry === void 0) {
     throw new ValidationError("bulk-load encryption requires a registry to store the wrapped DEK");
@@ -2099,7 +2313,10 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
       crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
     }
   }
-  const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, { crypto });
+  const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, {
+    crypto,
+    clock: options.clock
+  });
   if (options.registry !== void 0) {
     const becameCurrent = await publishGeneration(options.registry, key, {
       wrappedDeks: newWrapped
@@ -2115,13 +2332,6 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
   }
   return { size, sha256, chunkCount: chunks.length, cardinality };
 }
-async function* toAsyncIterable(src) {
-  if (Symbol.asyncIterator in src) {
-    yield* src;
-  } else {
-    yield* src;
-  }
-}
 
 // src/core/compaction.ts
 var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
@@ -2227,6 +2437,8 @@ async function compactSegmentInner(ref, deps, options) {
     let tally;
     try {
       tally = await writeCrbmGenerationStream(deps.cold, gen0, stream, {
+        clock: deps.clock,
+        // cooperative if the caller's clock can yield; a no-op if it only tells the time
         crypto: cryptoAt(aead, 0)
       });
     } catch (err) {
@@ -2354,7 +2566,8 @@ async function compactSegmentInner(ref, deps, options) {
       deps.cold,
       { namespace: ref.namespace, segment: ref.segment, generation: newGen },
       stream,
-      cryptoAt(aead, newGen)
+      cryptoAt(aead, newGen),
+      deps.clock
     );
     try {
       await deps.registry.compareAndSwap(ref, leaseToken, {
@@ -2451,8 +2664,8 @@ async function* mergeChunksStream(ref, g, pinned, deps, maxBytes, readCrypto) {
     if (!merged.isEmpty) yield { chunkKey, bitmap: merged };
   }
 }
-async function stageGeneration(cold, key, chunks, crypto) {
-  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto });
+async function stageGeneration(cold, key, chunks, crypto, clock) {
+  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto, clock });
   await verifyGeneration(cold, key, tally, crypto);
 }
 async function verifyGeneration(cold, key, expected, crypto) {
@@ -3950,17 +4163,43 @@ var RetryingWarmDriver = class {
   deleteConditional(ref, expected) {
     return withRetry(() => this.inner.deleteConditional(ref, expected), this.policy, this.deps);
   }
+  /**
+   * Stream the inner driver's rows, retrying only until the first one arrives.
+   *
+   * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+   * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+   * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+   * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+   * after the entire segment was already in memory. Measured on a 500-row segment under
+   * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+   * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+   * distinguishing observable, the row count was.
+   *
+   * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+   * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+   * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+   * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+   * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+   * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+   * where re-reading from the start is safe.
+   */
   async *listChunks(ref, opts) {
-    const rows = await withRetry(
+    const started = await withRetry(
       async () => {
-        const out = [];
-        for await (const row of this.inner.listChunks(ref, opts)) out.push(row);
-        return out;
+        const iterator = this.inner.listChunks(ref, opts)[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        return { iterator, first };
       },
       this.policy,
       this.deps
     );
-    yield* rows;
+    if (started.first.done === true) return;
+    yield started.first.value;
+    for (; ; ) {
+      const next = await started.iterator.next();
+      if (next.done === true) return;
+      yield next.value;
+    }
   }
 };
 var RetryingColdChunkSource = class {
@@ -4090,6 +4329,7 @@ exports.DEFAULT_MAX_WARM_SCAN_BYTES = DEFAULT_MAX_WARM_SCAN_BYTES;
 exports.DEFAULT_OCC_BACKOFF = DEFAULT_OCC_BACKOFF;
 exports.DEFAULT_PRICING = DEFAULT_PRICING;
 exports.DEFAULT_RETRY_POLICY = DEFAULT_RETRY_POLICY;
+exports.DEFAULT_WRITE_CONCURRENCY = DEFAULT_WRITE_CONCURRENCY;
 exports.InProcessKeystore = InProcessKeystore;
 exports.IntegrityError = IntegrityError;
 exports.KeyUnavailableError = KeyUnavailableError;