@cloudbitmaps/core 0.3.0 → 0.4.0

package/dist/index.d.cts

@@ -166,6 +166,21 @@ interface Clock {
      * replayable. `ms <= 0` resolves on a microtask without scheduling a timer.
      */
     sleep(ms: number): Promise<void>;
+    /**
+     * Hand the event loop back once, without asking for any elapsed time.
+     *
+     * This is **not** `sleep(0)`, and the difference is the reason the member exists. `sleep(0)` resolves on a
+     * microtask, and microtasks drain before the loop advances a phase — so awaiting it inside a CPU-bound loop
+     * yields to nothing at all, and a co-resident server stays blocked for the loop's full duration. A real yield
+     * needs a macrotask. `sleep(1)` is one, but it buys the relief at ~1 ms of dead wall-clock per yield.
+     *
+     * **Optional**, so every `Clock` written before this member still satisfies the interface. Callers must
+     * therefore degrade rather than assume: `clock.yieldNow?.() ?? clock.sleep(1)` — correct on any clock, cheap
+     * on one that implements this. `core/` cannot supply it itself (it is timer-free, lint-enforced); production
+     * wiring backs it with `setImmediate`, and a simulator can make it a no-op so virtual time is not perturbed
+     * by what is purely a scheduling courtesy.
+     */
+    yieldNow?(): Promise<void>;
 }
 /** Injected randomness. `core/` must never call `Math.random()` directly. Seedable for simulation. */
 interface Rng {
@@ -253,7 +268,7 @@ declare class BoundedLru<K, V> {
  * "Observability" section.
  */
 /** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
-type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto';
+type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
 /**
  * One observability event. A discriminated union on `kind` — new variants can be added over time without
  * breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
@@ -295,8 +310,18 @@ type MetricEvent = {
     readonly delayMs: number;
 } | {
     readonly kind: 'intersect';
+    /**
+     * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
+     * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
+     *
+     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
+     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
+     * `skippedChunks: 0` is the expected shape, not a regression.
+     */
+    readonly op?: 'intersect' | 'union' | 'andNot';
+    /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Shared **distinct** chunk-keys selected to fetch (present in every operand). */
+    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -462,6 +487,37 @@ declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: R
  * caller shows up.
  */
 declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
+/**
+ * Default number of Warm chunk writes in flight per `addMany`/`removeMany`: **4**.
+ *
+ * The flusher was serial by default, which meant a 100-chunk `addMany` against a backend with ~10 ms of
+ * round-trip latency spent a full second doing nothing but waiting — one round-trip at a time, on work that has
+ * no ordering requirement between chunks. Distinct chunks are independent OCC rows.
+ *
+ * **Why 4 and not more.** The number is chosen against the *backend's* tolerance, not the client's appetite. A
+ * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
+ * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
+ * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
+ * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
+ * counting a lost id or a surfaced error as a failure:
+ *
+ * ```text
+ *   backend capacity:      1      2      4      8
+ *   writeConcurrency  4:  0/5    0/5    0/5    0/5
+ *   writeConcurrency  8:  0/5    0/5    0/5    0/5
+ *   writeConcurrency 16:  0/5    0/5    0/5    0/5
+ *   writeConcurrency 32:  3/5    0/5    0/5    0/5   ← retry budget exhausted
+ * ```
+ *
+ * So the retry machinery holds far past 4; 4 is deliberately conservative, sitting 8x below the first observed
+ * failure against the harshest backend modelled. The remaining headroom is spent on the multiplier this table
+ * does not capture — many concurrent `addMany` calls sharing one backend — rather than on more throughput per
+ * call. A caller who knows their capacity can raise it.
+ *
+ * Raise it if your backend is provisioned for it (on-demand DynamoDB, or a Postgres pool sized to match); set
+ * it to `1` for the previous strictly-serial behaviour.
+ */
+declare const DEFAULT_WRITE_CONCURRENCY = 4;
 interface EngineDeps {
     readonly warm: IWarmDriver;
     readonly cold: ColdChunkSource;
@@ -490,9 +546,16 @@ interface EngineDeps {
      */
     readonly warmReadConsistency?: 'strong' | 'eventual';
     /**
-     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher). Default **1** (serial —
-     * unchanged behavior). Higher values fan out distinct-chunk writes for throughput; each chunk is its own OCC
-     * row, so bounded concurrency is safe (`addMany` is already non-atomic on partial failure).
+     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
+     * Default {@link DEFAULT_WRITE_CONCURRENCY}.
+     *
+     * Each chunk is its own OCC row, so distinct chunks never conflict with each other and fanning them out is
+     * safe. Raise it for more throughput against a backend with capacity to spare; set it to `1` for strictly
+     * serial writes.
+     *
+     * As before, `addMany`/`removeMany` are **not atomic**: a mid-flush failure can leave a partial result. What
+     * concurrency changes is only *how much* of the batch may already have landed when the first error surfaces —
+     * the flusher stops scheduling on the first failure, but writes already in flight still settle.
      */
     readonly writeConcurrency?: number;
     /**
@@ -536,8 +599,8 @@ declare class SegmentEngine {
     constructor(deps: EngineDeps);
     add(seg: SegmentRef, id: number): Promise<void>;
     remove(seg: SegmentRef, id: number): Promise<void>;
-    addMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
-    removeMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
+    addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
+    removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
     has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
     count(seg: SegmentRef): Promise<number>;
     /** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
@@ -570,17 +633,51 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
+    /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
+     * intersected against a narrow audience costs a handful of reads, not 61,000.
+     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
+    }): AsyncGenerator<number>;
+    /**
+     * `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
+     * than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
+     * has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
+     * budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
+     */
+    union(segs: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
     }): AsyncGenerator<number>;
+    /**
+     * `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
+     *
+     * Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
+     * the cost scales with the segment being filtered rather than with the size of the suppression list.
+     */
+    andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        budget?: BudgetOption;
+    }): AsyncGenerator<number>;
+    private combine;
     /**
      * The AND of one chunk key across all operands (tier-merged per operand), or `null` if empty. Operand
      * chunks are fetched **in parallel** (the spec's parallel byte-range reads). `acc` takes ownership of the
      * first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
      * a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
      */
-    private intersectChunk;
+    private combineChunk;
     /**
      * Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
      * **Not atomic** — like `addMany`, ids land chunk-by-chunk; a failure mid-stream leaves a partial result.
@@ -589,7 +686,58 @@ declare class SegmentEngine {
     intersectInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         batchSize?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
+    }): Promise<void>;
+    /**
+     * Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
+     * {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
+     */
+    unionInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        batchSize?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
     }): Promise<void>;
+    /** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
+    andNotInto(dest: SegmentRef, seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        batchSize?: number;
+        budget?: BudgetOption;
+    }): Promise<void>;
+    /**
+     * Stream a combine's results into `dest` in bounded batches.
+     *
+     * `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
+     * mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
+     * large the result is.
+     */
+    private drainInto;
+    /**
+     * Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
+     *
+     * Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
+     * hand-batching `page → addMany(page)`.
+     *
+     * **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
+     * input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
+     * the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
+     * order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
+     * **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
+     * multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
+     * different bug.
+     *
+     * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
+     * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
+     * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
+     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     *
+     * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
+     * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
+     * large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
+     * thousands of warm rows. The crossover is documented in the guide.
+     */
     private mutateMany;
     /** OCC read-modify-write on a chunk's Warm delta, retrying on conflict (bounded). */
     private readModifyWrite;
@@ -1556,6 +1704,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
+    clock?: Clock;
 }): Promise<{
     size: number;
     sha256: string;
@@ -1610,6 +1759,16 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
     audit?: IAuditSink;
     /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
     codec?: CodecInterface;
+    /**
+     * Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
+     * long load does not stall everything else on the process. Without it the load still completes, just
+     * without yielding — which is the pre-existing behaviour, kept so this is purely additive.
+     *
+     * `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
+     * no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
+     * this seam rather than `setTimeout`.
+     */
+    clock?: Clock;
 }): Promise<BulkLoadResult>;
 
 /**
@@ -2024,4 +2183,4 @@ declare class CrbmWriter {
     private buildFooter;
 }
 
-export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
+export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };

package/dist/index.d.ts

@@ -166,6 +166,21 @@ interface Clock {
      * replayable. `ms <= 0` resolves on a microtask without scheduling a timer.
      */
     sleep(ms: number): Promise<void>;
+    /**
+     * Hand the event loop back once, without asking for any elapsed time.
+     *
+     * This is **not** `sleep(0)`, and the difference is the reason the member exists. `sleep(0)` resolves on a
+     * microtask, and microtasks drain before the loop advances a phase — so awaiting it inside a CPU-bound loop
+     * yields to nothing at all, and a co-resident server stays blocked for the loop's full duration. A real yield
+     * needs a macrotask. `sleep(1)` is one, but it buys the relief at ~1 ms of dead wall-clock per yield.
+     *
+     * **Optional**, so every `Clock` written before this member still satisfies the interface. Callers must
+     * therefore degrade rather than assume: `clock.yieldNow?.() ?? clock.sleep(1)` — correct on any clock, cheap
+     * on one that implements this. `core/` cannot supply it itself (it is timer-free, lint-enforced); production
+     * wiring backs it with `setImmediate`, and a simulator can make it a no-op so virtual time is not perturbed
+     * by what is purely a scheduling courtesy.
+     */
+    yieldNow?(): Promise<void>;
 }
 /** Injected randomness. `core/` must never call `Math.random()` directly. Seedable for simulation. */
 interface Rng {
@@ -253,7 +268,7 @@ declare class BoundedLru<K, V> {
  * "Observability" section.
  */
 /** The segment operations that emit an `op` latency event (timed with the injected clock, at the facade). */
-type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto';
+type MetricOpName = 'add' | 'remove' | 'addMany' | 'removeMany' | 'has' | 'count' | 'intersectInto' | 'unionInto' | 'andNotInto';
 /**
  * One observability event. A discriminated union on `kind` — new variants can be added over time without
  * breaking consumers (they switch on the kinds they care about). `namespace` is the segment's namespace
@@ -295,8 +310,18 @@ type MetricEvent = {
     readonly delayMs: number;
 } | {
     readonly kind: 'intersect';
+    /**
+     * Which chunk-aligned combine this was. **Optional for backward compatibility** — absent means
+     * `'intersect'`, which is all this event reported before `union`/`andNot` existed.
+     *
+     * `skippedChunks` is only ever non-zero for `'intersect'` and `'andNot'`: union has to read every chunk
+     * of every operand, so there is nothing it can prune. A steady stream of `op: 'union'` events with
+     * `skippedChunks: 0` is the expected shape, not a regression.
+     */
+    readonly op?: 'intersect' | 'union' | 'andNot';
+    /** Operand count, including any `exclude` (suppression) operands. */
     readonly operands: number;
-    /** Shared **distinct** chunk-keys selected to fetch (present in every operand). */
+    /** Distinct chunk-keys selected to fetch (for `'intersect'`, those present in every operand). */
     readonly fetchedChunks: number;
     /**
      * **Distinct** chunk-keys pruned — never fetched (the chunk-skipping saving). Counts distinct keys,
@@ -462,6 +487,37 @@ declare function withRetry<T>(op: () => Promise<T>, policy: RetryPolicy, deps: R
  * caller shows up.
  */
 declare const DEFAULT_MAX_WARM_SCAN_BYTES: number;
+/**
+ * Default number of Warm chunk writes in flight per `addMany`/`removeMany`: **4**.
+ *
+ * The flusher was serial by default, which meant a 100-chunk `addMany` against a backend with ~10 ms of
+ * round-trip latency spent a full second doing nothing but waiting — one round-trip at a time, on work that has
+ * no ordering requirement between chunks. Distinct chunks are independent OCC rows.
+ *
+ * **Why 4 and not more.** The number is chosen against the *backend's* tolerance, not the client's appetite. A
+ * write here is a read-modify-write, so 4 in flight is up to 8 concurrent requests per `addMany` call — and a
+ * server handling many concurrent calls multiplies that again. Provisioned-capacity backends answer a burst
+ * with throttling, which is free only while retries can absorb it (4 attempts, exponential backoff, full
+ * jitter). Swept against a backend that throttles on concurrent requests in flight, 64 chunks, 5 trials each,
+ * counting a lost id or a surfaced error as a failure:
+ *
+ * ```text
+ *   backend capacity:      1      2      4      8
+ *   writeConcurrency  4:  0/5    0/5    0/5    0/5
+ *   writeConcurrency  8:  0/5    0/5    0/5    0/5
+ *   writeConcurrency 16:  0/5    0/5    0/5    0/5
+ *   writeConcurrency 32:  3/5    0/5    0/5    0/5   ← retry budget exhausted
+ * ```
+ *
+ * So the retry machinery holds far past 4; 4 is deliberately conservative, sitting 8x below the first observed
+ * failure against the harshest backend modelled. The remaining headroom is spent on the multiplier this table
+ * does not capture — many concurrent `addMany` calls sharing one backend — rather than on more throughput per
+ * call. A caller who knows their capacity can raise it.
+ *
+ * Raise it if your backend is provisioned for it (on-demand DynamoDB, or a Postgres pool sized to match); set
+ * it to `1` for the previous strictly-serial behaviour.
+ */
+declare const DEFAULT_WRITE_CONCURRENCY = 4;
 interface EngineDeps {
     readonly warm: IWarmDriver;
     readonly cold: ColdChunkSource;
@@ -490,9 +546,16 @@ interface EngineDeps {
      */
     readonly warmReadConsistency?: 'strong' | 'eventual';
     /**
-     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher). Default **1** (serial —
-     * unchanged behavior). Higher values fan out distinct-chunk writes for throughput; each chunk is its own OCC
-     * row, so bounded concurrency is safe (`addMany` is already non-atomic on partial failure).
+     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher).
+     * Default {@link DEFAULT_WRITE_CONCURRENCY}.
+     *
+     * Each chunk is its own OCC row, so distinct chunks never conflict with each other and fanning them out is
+     * safe. Raise it for more throughput against a backend with capacity to spare; set it to `1` for strictly
+     * serial writes.
+     *
+     * As before, `addMany`/`removeMany` are **not atomic**: a mid-flush failure can leave a partial result. What
+     * concurrency changes is only *how much* of the batch may already have landed when the first error surfaces —
+     * the flusher stops scheduling on the first failure, but writes already in flight still settle.
      */
     readonly writeConcurrency?: number;
     /**
@@ -536,8 +599,8 @@ declare class SegmentEngine {
     constructor(deps: EngineDeps);
     add(seg: SegmentRef, id: number): Promise<void>;
     remove(seg: SegmentRef, id: number): Promise<void>;
-    addMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
-    removeMany(seg: SegmentRef, ids: Iterable<number>): Promise<void>;
+    addMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
+    removeMany(seg: SegmentRef, ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
     has(seg: SegmentRef, id: number, opts?: WarmReadOptions): Promise<boolean>;
     count(seg: SegmentRef): Promise<number>;
     /** Whether the Cold source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
@@ -570,17 +633,51 @@ declare class SegmentEngine {
      * re-resolves fresh (bypassing the TTL), which can hop generations even sub-TTL — still whole/immutable per
      * read, never torn. Full intra-op snapshot isolation is deferred (gap #4).
      */
+    /**
+     * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * The `exclude` operands are what make suppression **compose without materialising**. Applying suppression
+     * afterwards (intersect into a temp segment, then remove) writes an intermediate segment that nobody wants;
+     * applying it here folds it into the same chunk-aligned pass. It is also cheaper than it looks: an exclude
+     * operand is read **only at the keys that survived the intersection**, so a 61,000-chunk global opt-out list
+     * intersected against a narrow audience costs a handful of reads, not 61,000.
+     */
     intersect(segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
+    }): AsyncGenerator<number>;
+    /**
+     * `segs[0] ∪ segs[1] ∪ …`, minus every id in `options.exclude` — streamed ascending.
+     *
+     * **Union is the one composite read with no chunk-skipping**, and that is a property of the operation rather
+     * than of this implementation: an id in *any* operand belongs to the result, so every chunk of every operand
+     * has to be read. `intersect` can prune a key absent from any operand; union cannot prune anything. It is
+     * budgeted exactly like `intersect` so a wide union is refused rather than quietly billed.
+     */
+    union(segs: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
     }): AsyncGenerator<number>;
+    /**
+     * `seg \ (excludes[0] ∪ excludes[1] ∪ …)` — streamed ascending. Suppression on its own.
+     *
+     * Reads every chunk of `seg` (any of them may survive) but each exclude **only where it overlaps `seg`**, so
+     * the cost scales with the segment being filtered rather than with the size of the suppression list.
+     */
+    andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        budget?: BudgetOption;
+    }): AsyncGenerator<number>;
+    private combine;
     /**
      * The AND of one chunk key across all operands (tier-merged per operand), or `null` if empty. Operand
      * chunks are fetched **in parallel** (the spec's parallel byte-range reads). `acc` takes ownership of the
      * first operand's effective chunk and mutates it in place — safe because `effectiveChunk` always returns
      * a FRESH bitmap (never a cached/shared Cold instance), so this can't poison the HOT cache.
      */
-    private intersectChunk;
+    private combineChunk;
     /**
      * Materialize `seg[0] ∩ seg[1] ∩ …` into `dest` via batched `addMany` (streaming, bounded memory).
      * **Not atomic** — like `addMany`, ids land chunk-by-chunk; a failure mid-stream leaves a partial result.
@@ -589,7 +686,58 @@ declare class SegmentEngine {
     intersectInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
         concurrency?: number;
         batchSize?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
+    }): Promise<void>;
+    /**
+     * Materialize `segs[0] ∪ segs[1] ∪ …` (minus `options.exclude`) into `dest`. **Not atomic**, like
+     * {@link intersectInto}. See {@link union} for why this one cannot skip any chunk.
+     */
+    unionInto(dest: SegmentRef, segs: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        batchSize?: number;
+        budget?: BudgetOption;
+        exclude?: readonly SegmentRef[];
     }): Promise<void>;
+    /** Materialize `seg \ (excludes…)` into `dest`. **Not atomic**, like {@link intersectInto}. */
+    andNotInto(dest: SegmentRef, seg: SegmentRef, excludes: readonly SegmentRef[], options?: {
+        concurrency?: number;
+        batchSize?: number;
+        budget?: BudgetOption;
+    }): Promise<void>;
+    /**
+     * Stream a combine's results into `dest` in bounded batches.
+     *
+     * `dest` is **added to, not replaced**, and this is **not atomic** — ids land chunk-by-chunk, so a failure
+     * mid-stream leaves a partial result, exactly as `addMany` does. Batching keeps memory bounded no matter how
+     * large the result is.
+     */
+    private drainInto;
+    /**
+     * Group ids by chunk, then apply each chunk's whole batch in a single OCC read-modify-write.
+     *
+     * Accepts a **sync or async** iterable, so an Athena/BigQuery cursor streams straight in without the caller
+     * hand-batching `page → addMany(page)`.
+     *
+     * **The pending ids are held as BITMAPS, not as JS numbers, and that is the load-bearing choice.** Streaming
+     * input is unbounded by definition, so something has to give. The obvious answer — buffer N ids, flush them to
+     * the backend, repeat — bounds memory and is quietly catastrophic on cost: with ids arriving in arbitrary
+     * order, every flush touches nearly every chunk again, so an 11M-id stream at a 1M-id buffer would issue
+     * **11x** the OCC round-trips of a single pass. In a library whose headline claim is honest cost, silently
+     * multiplying someone's write bill by the size of their input is not a bounded-memory strategy; it is a
+     * different bug.
+     *
+     * So the staging buffer is capped, but it folds into per-chunk **compressed bitmaps** rather than being
+     * written out. Peak memory is the roaring representation of the ids seen so far — which is what the segment
+     * itself costs, and the floor for any implementation that writes each chunk exactly once — while the backend
+     * still sees exactly one read-modify-write per distinct chunk regardless of stream length. This is strictly
+     * less memory than the previous sync-only version, which held every remainder as an uncompressed JS number.
+     *
+     * There is no way to do better without a sorted stream: a chunk can only be written once it is known that no
+     * further ids for it will arrive, and unsorted input never gives that guarantee. Callers streaming *very*
+     * large sets want `bulkLoadCrbmGeneration` instead — it writes one immutable Cold object rather than tens of
+     * thousands of warm rows. The crossover is documented in the guide.
+     */
     private mutateMany;
     /** OCC read-modify-write on a chunk's Warm delta, retrying on conflict (bounded). */
     private readModifyWrite;
@@ -1556,6 +1704,7 @@ declare function writeCrbmGeneration(driver: IColdDriver, key: GenKey, chunks: I
     bitmap: CodecBitmap;
 }>, options?: {
     crypto?: CrbmCrypto;
+    clock?: Clock;
 }): Promise<{
     size: number;
     sha256: string;
@@ -1610,6 +1759,16 @@ declare function bulkLoadCrbmGeneration(driver: IColdDriver, key: GenKey, ids: I
     audit?: IAuditSink;
     /** Bitmap codec. Optional in the type; a **flavor** package binds it ({@link requireCodec}). */
     codec?: CodecInterface;
+    /**
+     * Injected clock. Supplying one makes bulk-load **cooperative**: it yields the event loop periodically so a
+     * long load does not stall everything else on the process. Without it the load still completes, just
+     * without yielding — which is the pre-existing behaviour, kept so this is purely additive.
+     *
+     * `@cloudbitmaps/roaring` supplies a real clock by default, so flavor users get cooperative behaviour with
+     * no wiring. `core/` cannot default it: it is timer-free by lint, which is exactly why waiting goes through
+     * this seam rather than `setTimeout`.
+     */
+    clock?: Clock;
 }): Promise<BulkLoadResult>;
 
 /**
@@ -2024,4 +2183,4 @@ declare class CrbmWriter {
     private buildFooter;
 }
 
-export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };
+export { AWS_US_EAST_1_ONDEMAND, Aead, AeadSealed, type AuditEvent, type AuditEventKind, BlobReader, BlobSink, BoundedLru, type Budget, BudgetExceededError, type BudgetOption, type BulkLoadResult, CapabilityError, ChunkRef, type Clock, CloudRoaringError, type CodecBitmap, type CodecInterface, ColdCaps, ColdChunkSource, type CompactionCandidate, type CompactionCycleResult, type CompactionDeps, type CompactionOptions, type CompactionResult, type ConsistencyErrorEntry, type ConsistencyIssue, type ConsistencyReport, type CostAdvisory, type CostReport, CountingMetricsSink, CrbmColdChunkSource, type CrbmColdChunkSourceOptions, CrbmCrypto, CrbmReader, type CrbmReaderOptions, CrbmWriter, type CrbmWriterOptions, DEFAULT_BUDGET, DEFAULT_MAX_SCAN_SEGMENTS, DEFAULT_MAX_WARM_SCAN_BYTES, DEFAULT_OCC_BACKOFF, DEFAULT_PRICING, DEFAULT_RETRY_POLICY, DEFAULT_WRITE_CONCURRENCY, type DestroyResult, type DiscoveryOptions, type EngineDeps, type EraseDeps, type EstimateInput, type ExportFailure, type ExportFormat, type ExportManifest, type ExportOptions, type ExportSink, type ExportWriter, type ExportedSegment, GenKey, type IAuditSink, IColdDriver, IKeystore, type IMetricsSink, IRegistryDriver, IWarmDriver, InProcessKeystore, type InProcessKeystoreOptions, IntegrityError, KeyUnavailableError, LocalFsColdDriver, LocalFsRegistryDriver, type LocalFsRegistryDriverOptions, LocalFsWarmDriver, MemoryColdChunkSource, MemoryColdDriver, MemoryRegistryDriver, type MemoryRegistryDriverOptions, MemoryWarmDriver, type MetricEvent, type MetricOpName, type MetricsSnapshot, NOOP_AUDIT, NOOP_METRICS, NewRegistryRecord, NoRow, NodeAead, NotFoundError, type PricingProfile, RecordingAuditSink, RegCaps, RegistryPatch, RegistryRecord, type RetryDeps, type RetryPolicy, RetryingColdChunkSource, RetryingColdDriver, type RetryingOptions, RetryingRegistryDriver, RetryingWarmDriver, type Rng, SegmentEngine, SegmentRef, SegmentSize, type SegmentSizing, TimeoutError, Token, type Topology, TransientError, UnsupportedError, ValidationError, WarmReadOptions, WarmRow, type Workload, WrappedDek, WriteConflictError, bulkLoadCrbmGeneration, checkBudget, chunkRefKey, collectWithinBudget, compactSegment, destroySegment, eraseNamespace, estimateCost, findCompactable, gcOrphanGenerations, groundedReport, isCloudRoaringError, isIntegrityError, isNotFoundError, isTransient, isTransientError, isValidationError, isWriteConflictError, joinId, mapWithConcurrency, publishGeneration, resolveBudget, resolvePerOpBudget, runCompactionCycle, runConsistencyCheck, runExport, safeMetrics, segmentKey, splitId, validateCompactionOptions, validateSegmentRef, withRetry, writeCrbmGeneration };