@cloudbitmaps/core 0.4.0 → 0.5.0

package/dist/index.cjs

@@ -226,7 +226,9 @@ async function mapWithConcurrency(items, limit, fn) {
 var YIELD_EVERY = 1024;
 function yieldEvery(clock, everyN = YIELD_EVERY) {
   if (clock === void 0) return () => null;
-  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : () => clock.sleep(1);
+  const sleep = clock.sleep;
+  const pause = clock.yieldNow !== void 0 ? () => clock.yieldNow() : sleep !== void 0 ? () => sleep.call(clock, 1) : void 0;
+  if (pause === void 0) return () => null;
   let n = 0;
   return () => {
     if (++n < everyN) return null;
@@ -426,6 +428,17 @@ var InstantClock = class {
   sleep() {
     return Promise.resolve();
   }
+  /**
+   * An explicit no-op, so the intent is stated rather than inferred.
+   *
+   * Without it, `yieldEvery` would fall back to `sleep(1)` — and this clock's `sleep` ignores its argument and
+   * resolves on a microtask, so the "yield" would silently do nothing while every doc comment claimed it did
+   * something. This wiring has no real event loop to be courteous to (it is the clock used when nobody injects
+   * one), so no-op is the right behaviour; it just has to be a decision rather than an accident.
+   */
+  yieldNow() {
+    return Promise.resolve();
+  }
 };
 var ZeroRng = class {
   next() {
@@ -543,6 +556,17 @@ var SegmentEngine = class {
     }
   }
   /**
+   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+   *
+   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+   * operand is read **only at the keys that survived the intersection** — the cost is bounded by the
+   * audience's surviving key count and **never scales with the size of the suppression list**. Subtracting a
+   * 61,000-chunk global opt-out list costs at most one read per surviving key, not 61,000.
+   *
+   * ---
+   *
    * Chunk-skipping intersection. Computes
    * `seg[0] ∩ seg[1] ∩ …` and streams the result ids ascending. It aligns each segment's *effective*
    * chunk-key set (Warm ∪ Cold), keeps only keys present in **all** operands (a key missing from any
@@ -567,15 +591,6 @@ var SegmentEngine = class {
    * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
    * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
    */
-  /**
-   * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
-   *
-   * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
-   * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
-   * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
-   * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
-   * intersected against a narrow audience costs a handful of reads, not 61,000.
-   */
   intersect(segs, options) {
     return this.combine(segs, options?.exclude ?? [], "all", "intersect", options);
   }
@@ -763,8 +778,10 @@ var SegmentEngine = class {
    * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
    * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
    * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
-   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
-   * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+   * still sees exactly one read-modify-write per distinct chunk regardless of stream length. Above the staging cap
+   * this holds less than the previous sync-only version, which kept every remainder as an uncompressed JS
+   * number; below it — the common `addMany` — nothing is compacted until the end and the two are comparable.
+   * The guarantee is that a stream of *any* length stays bounded, not that every call got smaller.
    *
    * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
    * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
@@ -807,8 +824,6 @@ var SegmentEngine = class {
       for (const id of ids) {
         stage(id);
         if (pending >= MUTATE_STAGE_IDS) compact();
-        const pause = tickId();
-        if (pause !== null) await pause;
       }
     }
     compact();
@@ -2203,6 +2218,10 @@ async function publishGeneration(registry, key, options = {}) {
           currentGen: key.generation,
           wrappedDeks: options.wrappedDeks
         });
+      } else if (record.status === "destroyed") {
+        throw new ValidationError(
+          `segment "${key.segment}" was destroyed (crypto-shredded) while generation ${key.generation} was being written \u2014 refusing to publish it; the written object is unreadable. Use a new segment.`
+        );
       } else if (record.currentGen > key.generation) {
         return false;
       } else if (record.currentGen === key.generation) {
@@ -2422,6 +2441,8 @@ async function compactSegmentInner(ref, deps, options) {
     let tally;
     try {
       tally = await writeCrbmGenerationStream(deps.cold, gen0, stream, {
+        clock: deps.clock,
+        // cooperative if the caller's clock can yield; a no-op if it only tells the time
         crypto: cryptoAt(aead, 0)
       });
     } catch (err) {
@@ -2549,7 +2570,8 @@ async function compactSegmentInner(ref, deps, options) {
       deps.cold,
       { namespace: ref.namespace, segment: ref.segment, generation: newGen },
       stream,
-      cryptoAt(aead, newGen)
+      cryptoAt(aead, newGen),
+      deps.clock
     );
     try {
       await deps.registry.compareAndSwap(ref, leaseToken, {
@@ -2646,8 +2668,8 @@ async function* mergeChunksStream(ref, g, pinned, deps, maxBytes, readCrypto) {
     if (!merged.isEmpty) yield { chunkKey, bitmap: merged };
   }
 }
-async function stageGeneration(cold, key, chunks, crypto) {
-  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto });
+async function stageGeneration(cold, key, chunks, crypto, clock) {
+  const tally = await writeCrbmGenerationStream(cold, key, chunks, { crypto, clock });
   await verifyGeneration(cold, key, tally, crypto);
 }
 async function verifyGeneration(cold, key, expected, crypto) {
@@ -4028,12 +4050,13 @@ async function shredSegment(ref, deps, allowCleartext) {
 }
 async function eraseWarm(warm, ref) {
   let deleted = 0;
+  let conflicts = 0;
   for (let pass = 0; pass < MAX_WARM_PASSES; pass++) {
+    conflicts = 0;
     const rows = [];
     for await (const row of warm.listChunks(ref))
       rows.push({ chunkKey: row.chunkKey, token: row.token });
     if (rows.length === 0) break;
-    let conflicts = 0;
     for (const { chunkKey, token } of rows) {
       const chunkRef = { namespace: ref.namespace, segment: ref.segment, chunkKey };
       try {
@@ -4046,6 +4069,11 @@ async function eraseWarm(warm, ref) {
     }
     if (conflicts === 0) break;
   }
+  if (conflicts > 0) {
+    throw new WriteConflictError(
+      `destroySegment: ${conflicts} warm row(s) of "${ref.segment}" were rewritten during every one of ${MAX_WARM_PASSES} erase passes \u2014 the segment was NOT destroyed and its cleartext warm deltas are still readable. Stop writes to this segment and retry.`
+    );
+  }
   return deleted;
 }
 
@@ -4145,17 +4173,47 @@ var RetryingWarmDriver = class {
   deleteConditional(ref, expected) {
     return withRetry(() => this.inner.deleteConditional(ref, expected), this.policy, this.deps);
   }
+  /**
+   * Stream the inner driver's rows, retrying only until the first one arrives.
+   *
+   * **This used to buffer the whole scan** — `for await (…) out.push(row)` inside `withRetry`, then
+   * `yield* out` — which made the retry trivially safe and quietly defeated every bound above it. The engine's
+   * `collectWarm` refuses *during* enumeration precisely so resident memory is `O(ceiling)` rather than
+   * `O(segment)`; with the buffering wrapper in place (and it is wired by default) it saw its first row only
+   * after the entire segment was already in memory. Measured on a 500-row segment under
+   * `budget: { maxRequests: 3 }`: **500 rows materialised with the default wiring, 4 with `retry: false`**.
+   * Both paths threw the same `BudgetExceededError`, which is why no test caught it — the error was never the
+   * distinguishing observable, the row count was.
+   *
+   * **The trade this makes.** A mid-stream fault now propagates instead of being retried. It cannot be retried
+   * honestly: rows have already been handed to the consumer, so restarting the iterator would re-yield them,
+   * and the driver interface exposes no resumption token to continue from. Retry is kept where it pays —
+   * establishing the scan, which is where transient connect/throttle failures actually land — and bounded
+   * memory wins the conflict, because invariant 6 ("bounded memory & cost, always") is a hard invariant and
+   * mid-scan retry is not. A caller who needs whole-scan retry can wrap the operation at their own level,
+   * where re-reading from the start is safe.
+   */
   async *listChunks(ref, opts) {
-    const rows = await withRetry(
+    const started = await withRetry(
       async () => {
-        const out = [];
-        for await (const row of this.inner.listChunks(ref, opts)) out.push(row);
-        return out;
+        const iterator = this.inner.listChunks(ref, opts)[Symbol.asyncIterator]();
+        const first = await iterator.next();
+        return { iterator, first };
       },
       this.policy,
       this.deps
     );
-    yield* rows;
+    if (started.first.done === true) return;
+    try {
+      yield started.first.value;
+      for (; ; ) {
+        const next = await started.iterator.next();
+        if (next.done === true) return;
+        yield next.value;
+      }
+    } finally {
+      await started.iterator.return?.();
+    }
   }
 };
 var RetryingColdChunkSource = class {