@cloudbitmaps/core 0.2.0 → 0.4.0

package/dist/index.js

@@ -37,6 +37,14 @@ function applyRemove(delta, remainder) {
   delta.removes.add(remainder);
   delta.adds.remove(remainder);
 }
+function applyAddAll(delta, ids) {
+  delta.adds.orInPlace(ids);
+  delta.removes.andNotInPlace(ids);
+}
+function applyRemoveAll(delta, ids) {
+  delta.removes.orInPlace(ids);
+  delta.adds.andNotInPlace(ids);
+}
 function effective(cold, delta) {
   const out = cold.clone();
   out.orInPlace(delta.adds);
@@ -117,6 +125,31 @@ function checkBudget(budget, projectedRequests, op) {
     );
   }
 }
+async function collectWithinBudget(source, budget, op) {
+  const out = [];
+  for await (const item of source) {
+    out.push(item);
+    if (budget !== null && out.length > budget.maxRequests) {
+      throw new BudgetExceededError(
+        `${op} would fan out to more than ${budget.maxRequests} units, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``
+      );
+    }
+  }
+  return out;
+}
+
+// src/core/cooperative.ts
+var YIELD_EVERY = 1024;
+function yieldEvery(clock, everyN = YIELD_EVERY) {
+  if (clock === void 0) return () => null;
+  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : () => clock.sleep(1);
+  let n = 0;
+  return () => {
+    if (++n < everyN) return null;
+    n = 0;
+    return pause();
+  };
+}
 
 // src/core/keys.ts
 var FIELD = " ";
@@ -297,6 +330,8 @@ async function withRetry(op, policy, deps) {
 var DEFAULT_MAX_BITMAP_BYTES = 1 << 20;
 var DEFAULT_MAX_RETRIES = 16;
 var DEFAULT_INTERSECT_CONCURRENCY = 8;
+var MUTATE_STAGE_IDS = 1 << 20;
+var YIELD_EVERY_IDS = 1 << 14;
 var InstantClock = class {
   now() {
     return 0;
@@ -310,6 +345,8 @@ var ZeroRng = class {
     return 0;
   }
 };
+var DEFAULT_MAX_WARM_SCAN_BYTES = 64 * 1024 * 1024;
+var DEFAULT_WRITE_CONCURRENCY = 4;
 var SegmentEngine = class {
   warm;
   cold;
@@ -325,6 +362,7 @@ var SegmentEngine = class {
   /** Consistency for READ-path Warm fetches (precomputed; OCC RMW passes strong explicitly, see `warmGet`). */
   readOpts;
   writeConcurrency;
+  maxWarmScanBytes;
   /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. See {@link checkBudget}. */
   budget;
   constructor(deps) {
@@ -338,7 +376,13 @@ var SegmentEngine = class {
     this.rng = deps.rng ?? new ZeroRng();
     this.occBackoff = deps.occBackoff ?? DEFAULT_OCC_BACKOFF;
     this.readOpts = { consistent: (deps.warmReadConsistency ?? "strong") !== "eventual" };
-    this.writeConcurrency = deps.writeConcurrency ?? 1;
+    this.writeConcurrency = deps.writeConcurrency ?? DEFAULT_WRITE_CONCURRENCY;
+    this.maxWarmScanBytes = deps.maxWarmScanBytes ?? DEFAULT_MAX_WARM_SCAN_BYTES;
+    if (!Number.isFinite(this.maxWarmScanBytes) || this.maxWarmScanBytes < 1) {
+      throw new ValidationError(
+        `maxWarmScanBytes must be a finite number >= 1 (got ${String(deps.maxWarmScanBytes)})`
+      );
+    }
     this.budget = deps.budget === void 0 ? DEFAULT_BUDGET : deps.budget;
     this.metrics = safeMetrics(deps.metrics ?? NOOP_METRICS);
     this.metricsOn = this.metrics !== NOOP_METRICS;
@@ -352,10 +396,10 @@ var SegmentEngine = class {
     return this.readModifyWrite(seg, chunkKey, (d) => applyRemove(d, remainder));
   }
   addMany(seg, ids) {
-    return this.mutateMany(seg, ids, applyAdd);
+    return this.mutateMany(seg, ids, applyAddAll);
   }
   removeMany(seg, ids) {
-    return this.mutateMany(seg, ids, applyRemove);
+    return this.mutateMany(seg, ids, applyRemoveAll);
   }
   async has(seg, id, opts) {
     const { chunkKey, remainder } = splitId(id);
@@ -370,7 +414,7 @@ var SegmentEngine = class {
     return cold ? cold.has(remainder) : false;
   }
   async count(seg) {
-    const warmRows = await this.collectWarm(seg);
+    const warmRows = await this.collectWarm(seg, "count", this.budget?.maxRequests ?? null);
     const cardinalities = this.cold.cardinalities ? await this.cold.cardinalities(seg) : null;
     if (!cardinalities) {
       const chunkKeys = await this.chunkKeys(seg, warmRows);
@@ -402,7 +446,7 @@ var SegmentEngine = class {
     return this.cold.sizeOf ? this.cold.sizeOf(seg) : Promise.resolve(null);
   }
   async *iterate(seg) {
-    const warmRows = await this.collectWarm(seg);
+    const warmRows = await this.collectWarm(seg, "iterate", this.budget?.maxRequests ?? null);
     const chunkKeys = await this.chunkKeys(seg, warmRows);
     checkBudget(this.budget, chunkKeys.length, "iterate");
     const gen = await this.currentGen(seg);
@@ -436,9 +480,44 @@ var SegmentEngine = class {
    * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
    * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
    */
-  async *intersect(segs, options) {
+  /**
+   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+   * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
+   * intersected against a narrow audience costs a handful of reads, not 61,000.
+   */
+  intersect(segs, options) {
+    return this.combine(segs, options?.exclude ?? [], "all", "intersect", options);
+  }
+  /**
+   * `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
+   * than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
+   * has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
+   * budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
+   */
+  union(segs, options) {
+    return this.combine(segs, options?.exclude ?? [], "any", "union", options);
+  }
+  /**
+   * `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
+   *
+   * Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
+   * the cost scales with the segment being filtered rather than with the size of the suppression list.
+   */
+  andNot(seg, excludes, options) {
+    return this.combine([seg], excludes, "all", "andNot", options);
+  }
+  async *combine(segs, excludeSegs, mode, op, options) {
     if (segs.length === 0) {
-      throw new ValidationError("intersect requires at least one segment");
+      throw new ValidationError(`${op} requires at least one segment`);
+    }
+    if (op === "andNot" && excludeSegs.length === 0) {
+      throw new ValidationError("andNot requires at least one segment to subtract");
     }
     const limit = options?.concurrency ?? DEFAULT_INTERSECT_CONCURRENCY;
     if (!Number.isInteger(limit) || limit < 1) {
@@ -447,34 +526,52 @@ var SegmentEngine = class {
       );
     }
     const budget = resolvePerOpBudget(options?.budget, this.budget);
-    const operands = await Promise.all(
-      segs.map(async (seg) => {
-        const warmRows = await this.collectWarm(seg);
-        const keys = await this.chunkKeys(seg, warmRows);
-        const gen = await this.currentGen(seg);
-        return { seg, warmRows, keys: new Set(keys), gen };
-      })
-    );
-    const pivot = operands.reduce((a, b) => a.keys.size <= b.keys.size ? a : b);
-    const common = [];
-    for (const k of pivot.keys) {
-      if (operands.every((op) => op.keys.has(k))) common.push(k);
-    }
-    common.sort((a, b) => a - b);
-    checkBudget(budget, common.length * operands.length, "intersect");
+    const extract = async (seg) => {
+      const warmRows = await this.collectWarm(seg, op, null);
+      const keys = await this.chunkKeys(seg, warmRows);
+      const gen = await this.currentGen(seg);
+      return { seg, warmRows, keys: new Set(keys), gen };
+    };
+    const [operands, excludes] = await Promise.all([
+      Promise.all(segs.map(extract)),
+      Promise.all(excludeSegs.map(extract))
+    ]);
+    const candidates = [];
+    if (mode === "all") {
+      const pivot = operands.reduce((a, b) => a.keys.size <= b.keys.size ? a : b);
+      for (const k of pivot.keys) {
+        if (operands.every((o) => o.keys.has(k))) candidates.push(k);
+      }
+    } else {
+      const seen = /* @__PURE__ */ new Set();
+      for (const o of operands) for (const k of o.keys) seen.add(k);
+      for (const k of seen) candidates.push(k);
+    }
+    candidates.sort((a, b) => a - b);
+    const common = candidates;
+    let units = 0;
+    for (const key of common) {
+      for (const o of operands) if (mode === "all" || o.keys.has(key)) units += 1;
+      for (const e of excludes) if (e.keys.has(key)) units += 1;
+    }
+    checkBudget(budget, units, op);
     if (this.metricsOn) {
       const distinctKeys = /* @__PURE__ */ new Set();
-      for (const op of operands) {
-        for (const k of op.keys) distinctKeys.add(k);
+      for (const op2 of operands) {
+        for (const k of op2.keys) distinctKeys.add(k);
       }
+      for (const e of excludes) for (const k of e.keys) distinctKeys.add(k);
       this.metrics.onEvent({
         kind: "intersect",
-        operands: segs.length,
+        op,
+        operands: segs.length + excludes.length,
         fetchedChunks: common.length,
+        // Never negative: under 'any' every candidate key came from some operand, so `distinctKeys` is a
+        // superset of `common`; excludes only ever add keys here.
         skippedChunks: distinctKeys.size - common.length
       });
     }
-    const startAt = (key) => this.intersectChunk(operands, key).then(
+    const startAt = (key) => this.combineChunk(operands, excludes, mode, key).then(
       (result) => ({ key, result }),
       (error) => ({ key, result: null, error })
     );
@@ -496,16 +593,33 @@ var SegmentEngine = class {
    * first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
    * a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
    */
-  async intersectChunk(operands, chunkKey) {
+  async combineChunk(operands, excludes, mode, chunkKey) {
+    const present = mode === "all" ? operands : operands.filter((o) => o.keys.has(chunkKey));
+    if (present.length === 0) return null;
     const chunks = await Promise.all(
-      operands.map((op) => this.effectiveChunk(op.seg, chunkKey, op.warmRows, op.gen))
+      present.map((op) => this.effectiveChunk(op.seg, chunkKey, op.warmRows, op.gen))
     );
     const acc = chunks[0];
     for (let i = 1; i < chunks.length; i++) {
-      acc.andInPlace(chunks[i]);
-      if (acc.isEmpty) return null;
+      if (mode === "all") {
+        acc.andInPlace(chunks[i]);
+        if (acc.isEmpty) return null;
+      } else {
+        acc.orInPlace(chunks[i]);
+      }
+    }
+    if (acc.isEmpty) return null;
+    const relevant = excludes.filter((e) => e.keys.has(chunkKey));
+    if (relevant.length > 0) {
+      const cuts = await Promise.all(
+        relevant.map((e) => this.effectiveChunk(e.seg, chunkKey, e.warmRows, e.gen))
+      );
+      for (const cut of cuts) {
+        acc.andNotInPlace(cut);
+        if (acc.isEmpty) return null;
+      }
     }
-    return acc.isEmpty ? null : acc;
+    return acc;
   }
   /**
    * Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
@@ -513,9 +627,30 @@ var SegmentEngine = class {
    * `dest` is added to, not replaced.
    */
   async intersectInto(dest, segs, options) {
-    const batchSize = Math.max(1, options?.batchSize ?? 4096);
+    await this.drainInto(dest, this.intersect(segs, options), options?.batchSize);
+  }
+  /**
+   * Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
+   * {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
+   */
+  async unionInto(dest, segs, options) {
+    await this.drainInto(dest, this.union(segs, options), options?.batchSize);
+  }
+  /** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
+  async andNotInto(dest, seg, excludes, options) {
+    await this.drainInto(dest, this.andNot(seg, excludes, options), options?.batchSize);
+  }
+  /**
+   * Stream a combine's results into `dest` in bounded batches.
+   *
+   * `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
+   * mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
+   * large the result is.
+   */
+  async drainInto(dest, source, batchSizeOption) {
+    const batchSize = Math.max(1, batchSizeOption ?? 4096);
     let batch = [];
-    for await (const id of this.intersect(segs, options)) {
+    for await (const id of source) {
       batch.push(id);
       if (batch.length >= batchSize) {
         await this.addMany(dest, batch);
@@ -524,19 +659,77 @@ var SegmentEngine = class {
     }
     if (batch.length > 0) await this.addMany(dest, batch);
   }
+  /**
+   * Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
+   *
+   * Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
+   * hand-batching `page → addMany(page)`.
+   *
+   * **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
+   * input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
+   * the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
+   * order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
+   * **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
+   * multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
+   * different bug.
+   *
+   * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
+   * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
+   * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
+   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
+   * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+   *
+   * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
+   * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
+   * large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
+   * thousands of warm rows. The crossover is documented in the guide.
+   */
   async mutateMany(seg, ids, apply) {
     const byChunk = /* @__PURE__ */ new Map();
-    for (const id of ids) {
+    const staged = /* @__PURE__ */ new Map();
+    let pending = 0;
+    const tickId = yieldEvery(this.clock, YIELD_EVERY_IDS);
+    const compact = () => {
+      for (const [chunkKey, rems] of staged) {
+        if (rems.length === 0) continue;
+        const existing = byChunk.get(chunkKey);
+        if (existing === void 0) byChunk.set(chunkKey, this.codec.fromValues(rems));
+        else existing.addMany(rems);
+        rems.length = 0;
+      }
+      pending = 0;
+    };
+    const stage = (id) => {
       const { chunkKey, remainder } = splitId(id);
-      const rems = byChunk.get(chunkKey) ?? [];
+      let rems = staged.get(chunkKey);
+      if (rems === void 0) {
+        rems = [];
+        staged.set(chunkKey, rems);
+      }
       rems.push(remainder);
-      byChunk.set(chunkKey, rems);
+      pending += 1;
+    };
+    if (Symbol.asyncIterator in ids) {
+      for await (const id of ids) {
+        stage(id);
+        if (pending >= MUTATE_STAGE_IDS) compact();
+        const pause = tickId();
+        if (pause !== null) await pause;
+      }
+    } else {
+      for (const id of ids) {
+        stage(id);
+        if (pending >= MUTATE_STAGE_IDS) compact();
+        const pause = tickId();
+        if (pause !== null) await pause;
+      }
     }
+    compact();
     await mapWithConcurrency(
       [...byChunk],
       this.writeConcurrency,
-      ([chunkKey, rems]) => this.readModifyWrite(seg, chunkKey, (d) => {
-        for (const r of rems) apply(d, r);
+      ([chunkKey, bitmap]) => this.readModifyWrite(seg, chunkKey, (d) => {
+        apply(d, bitmap);
       })
     );
   }
@@ -590,12 +783,22 @@ var SegmentEngine = class {
     }
     return row;
   }
-  async collectWarm(seg) {
+  async collectWarm(seg, op, rowCeiling) {
     const rows = /* @__PURE__ */ new Map();
     let bytes = 0;
     for await (const row of this.warm.listChunks(seg, this.readOpts)) {
       rows.set(row.chunkKey, row.bytes);
       bytes += row.bytes.length;
+      if (bytes > this.maxWarmScanBytes) {
+        throw new BudgetExceededError(
+          `${op} would hold more than ${this.maxWarmScanBytes} bytes of warm deltas resident for one segment \u2014 the scan was abandoned there rather than completed. Raise \`maxWarmScanBytes\`, narrow the segment, or compact it so fewer chunks carry warm deltas. (This ceiling is independent of \`budget\` and stays in force when \`budget: false\`.)`
+        );
+      }
+      if (rowCeiling !== null && rows.size > rowCeiling) {
+        throw new BudgetExceededError(
+          `${op} would read more than ${rowCeiling} warm chunk rows, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``
+        );
+      }
     }
     if (this.metricsOn) {
       this.metrics.onEvent({
@@ -1083,22 +1286,44 @@ function aadFor(ref, generation, scope) {
 }
 
 // src/core/crbm/crc32c.ts
-var TABLE = (() => {
+var TABLES = (() => {
   const POLY = 2197175160;
-  const table = new Uint32Array(256);
+  const t0 = new Uint32Array(256);
   for (let n = 0; n < 256; n++) {
     let c = n;
-    for (let k = 0; k < 8; k++) {
-      c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
+    for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
+    t0[n] = c >>> 0;
+  }
+  const tables = [t0];
+  for (let k = 1; k < 8; k++) {
+    const prev = tables[k - 1];
+    const next = new Uint32Array(256);
+    for (let n = 0; n < 256; n++) {
+      const v = prev[n];
+      next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
     }
-    table[n] = c >>> 0;
+    tables.push(next);
   }
-  return table;
+  return tables;
 })();
+var T0 = TABLES[0];
+var T1 = TABLES[1];
+var T2 = TABLES[2];
+var T3 = TABLES[3];
+var T4 = TABLES[4];
+var T5 = TABLES[5];
+var T6 = TABLES[6];
+var T7 = TABLES[7];
 function crc32c(bytes, seed = 0) {
   let crc = (seed ^ 4294967295) >>> 0;
-  for (let i = 0; i < bytes.length; i++) {
-    crc = (TABLE[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
+  let i = 0;
+  const n8 = bytes.length - bytes.length % 8;
+  for (; i < n8; i += 8) {
+    crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
+    crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
+  }
+  for (; i < bytes.length; i++) {
+    crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
   }
   return (crc ^ 4294967295) >>> 0;
 }
@@ -1609,6 +1834,8 @@ function coldBlobReader(driver, key) {
   };
 }
 var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
+var BULK_FLUSH_IDS = 1 << 20;
+var YIELD_EVERY_IDS2 = 1 << 14;
 var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
 var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
 var CrbmColdChunkSource = class {
@@ -1827,11 +2054,14 @@ var CrbmColdChunkSource = class {
 };
 function writeCrbmGeneration(driver, key, chunks, options = {}) {
   const sorted = [...chunks].sort((a, b) => a.chunkKey - b.chunkKey);
+  const tick = yieldEvery(options.clock);
   return driver.putImmutable(key, async (sink) => {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for (const { chunkKey, bitmap } of sorted) {
       if (bitmap.isEmpty) continue;
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
+      const pause = tick();
+      if (pause !== null) await pause;
     }
     await writer.finish();
   });
@@ -1839,6 +2069,7 @@ function writeCrbmGeneration(driver, key, chunks, options = {}) {
 async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
   const chunkKeys = [];
   let cardinality = 0;
+  const tick = yieldEvery(options.clock);
   const { size, sha256 } = await driver.putImmutable(key, async (sink) => {
     const writer = new CrbmWriter(sink, { generation: key.generation, crypto: options.crypto });
     for await (const { chunkKey, bitmap } of chunks) {
@@ -1846,6 +2077,8 @@ async function writeCrbmGenerationStream(driver, key, chunks, options = {}) {
       await writer.addChunk(chunkKey, bitmap.serialize(), bitmap.size);
       chunkKeys.push(chunkKey);
       cardinality += bitmap.size;
+      const pause = tick();
+      if (pause !== null) await pause;
     }
     await writer.finish();
   });
@@ -1886,20 +2119,53 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
   }
   const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
   const byChunk = /* @__PURE__ */ new Map();
-  for await (const id of toAsyncIterable(ids)) {
+  const pendingByChunk = /* @__PURE__ */ new Map();
+  let pending = 0;
+  const tickChunk = yieldEvery(options.clock);
+  const tickId = yieldEvery(options.clock, YIELD_EVERY_IDS2);
+  const flushPending = async () => {
+    for (const [chunkKey, rems] of pendingByChunk) {
+      if (rems.length === 0) continue;
+      const existingBitmap = byChunk.get(chunkKey);
+      if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
+      else existingBitmap.addMany(rems);
+      rems.length = 0;
+      const pause = tickChunk();
+      if (pause !== null) await pause;
+    }
+    pending = 0;
+  };
+  const ingest = (id) => {
     const { chunkKey, remainder } = splitId(id);
-    let bitmap = byChunk.get(chunkKey);
-    if (bitmap === void 0) {
-      bitmap = codec.empty();
-      byChunk.set(chunkKey, bitmap);
+    let bucket = pendingByChunk.get(chunkKey);
+    if (bucket === void 0) {
+      bucket = [];
+      pendingByChunk.set(chunkKey, bucket);
+    }
+    bucket.push(remainder);
+    return ++pending >= BULK_FLUSH_IDS;
+  };
+  if (Symbol.asyncIterator in ids) {
+    for await (const id of ids) {
+      if (ingest(id)) await flushPending();
+      const pause = tickId();
+      if (pause !== null) await pause;
+    }
+  } else {
+    for (const id of ids) {
+      if (ingest(id)) await flushPending();
+      const pause = tickId();
+      if (pause !== null) await pause;
     }
-    bitmap.add(remainder);
   }
+  await flushPending();
   let cardinality = 0;
   const chunks = [];
   for (const [chunkKey, bitmap] of byChunk) {
     cardinality += bitmap.size;
     chunks.push({ chunkKey, bitmap });
+    const pause = tickChunk();
+    if (pause !== null) await pause;
   }
   if (options.keystore !== void 0 && options.registry === void 0) {
     throw new ValidationError("bulk-load encryption requires a registry to store the wrapped DEK");
@@ -1922,7 +2188,10 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
       crypto = { aead: minted.aead, aadFor: (scope) => aadFor(key, key.generation, scope) };
     }
   }
-  const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, { crypto });
+  const { size, sha256 } = await writeCrbmGeneration(driver, key, chunks, {
+    crypto,
+    clock: options.clock
+  });
   if (options.registry !== void 0) {
     const becameCurrent = await publishGeneration(options.registry, key, {
       wrappedDeks: newWrapped
@@ -1938,13 +2207,6 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
   }
   return { size, sha256, chunkCount: chunks.length, cardinality };
 }
-async function* toAsyncIterable(src) {
-  if (Symbol.asyncIterator in src) {
-    yield* src;
-  } else {
-    yield* src;
-  }
-}
 
 // src/core/compaction.ts
 var DEFAULT_MAX_BITMAP_BYTES2 = 1 << 20;
@@ -3472,6 +3734,7 @@ async function eraseWarm(warm, ref) {
 }
 
 // src/core/consistency.ts
+var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
 var DEFAULT_CHECK_CONCURRENCY = 8;
 async function listGenerations(cold, ref) {
   const present = /* @__PURE__ */ new Set();
@@ -3483,8 +3746,21 @@ async function runConsistencyCheck(deps, options = {}) {
   if (!Number.isInteger(concurrency) || concurrency < 1) {
     throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
   }
+  const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
+  if (!Number.isFinite(maxScanSegments) || maxScanSegments < 1) {
+    throw new ValidationError(
+      `maxScanSegments must be a finite number >= 1; got ${String(options.maxScanSegments)}`
+    );
+  }
   const recs = [];
-  for await (const rec of deps.registry.list(options.namespace)) recs.push(rec);
+  for await (const rec of deps.registry.list(options.namespace)) {
+    recs.push(rec);
+    if (recs.length > maxScanSegments) {
+      throw new BudgetExceededError(
+        `checkConsistency would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available.`
+      );
+    }
+  }
   const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
     if (rec.status === "destroyed") return { kind: "ok" };
     const ref = { segment: rec.segment, namespace: rec.namespace };
@@ -3676,6 +3952,6 @@ var RetryingRegistryDriver = class {
   }
 };
 
-export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
+export { AWS_US_EAST_1_ONDEMAND, BoundedLru, BufferReader, BufferSink, CountingMetricsSink, CrbmColdChunkSource, CrbmReader, CrbmWriter, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, InProcessKeystore, LocalFsColdDriver, LocalFsRegistryDriver, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, MemoryWarmDriver, NOOP_AUDIT, NOOP_METRICS, NodeAead, RecordingAuditSink, RetryingColdChunkSource, RetryingColdDriver, RetryingRegistryDriver, RetryingWarmDriver, SegmentEngine, aadFor, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isTransient, joinId, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, withRetry, writeCrbmGeneration };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map