@cloudbitmaps/roaring 0.3.0 → 0.4.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, CodecBitmap, CodecInterface, IColdDriver, ColdChunkSource, IWarmDriver, IRegistryDriver, IKeystore, Clock, Rng, RetryPolicy, IMetricsSink, BudgetOption, SegmentEngine, SegmentRef, PricingProfile, Workload, Topology, CostReport, IAuditSink, CompactionOptions, CompactionResult, ConsistencyReport, ExportSink, ExportOptions, ExportManifest, EstimateInput } from '@cloudbitmaps/core';
+import { bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, compactSegment as compactSegment$1, runCompactionCycle as runCompactionCycle$1, runExport as runExport$1, CodecBitmap, CodecInterface, BudgetOption, IColdDriver, ColdChunkSource, IWarmDriver, IRegistryDriver, IKeystore, Clock, Rng, RetryPolicy, IMetricsSink, SegmentEngine, SegmentRef, PricingProfile, Workload, Topology, CostReport, IAuditSink, CompactionOptions, CompactionResult, ConsistencyReport, ExportSink, ExportOptions, ExportManifest, EstimateInput } from '@cloudbitmaps/core';
 export * from '@cloudbitmaps/core';
 
 /**
@@ -21,7 +21,15 @@ type BulkLoad = typeof bulkLoadCrbmGeneration$1;
 type CompactSegment = typeof compactSegment$1;
 type RunCompactionCycle = typeof runCompactionCycle$1;
 type RunExport = typeof runExport$1;
-/** {@link coreBulkLoad} with the roaring codec pre-bound. */
+/**
+ * {@link coreBulkLoad} with the roaring codec **and a real clock** pre-bound.
+ *
+ * The clock is what makes a large load **cooperative**: bulk-load yields the event loop periodically instead of
+ * stalling the process for its whole duration (measured at 819 ms for 1M ids before this). `core/` cannot
+ * default it — it is timer-free by lint, which is precisely why waiting goes through the `Clock` seam — so the
+ * flavor package supplies it, exactly as it supplies the codec. A caller who passes their own clock (a virtual
+ * one in a simulation, say) keeps it.
+ */
 declare const bulkLoadCrbmGeneration: BulkLoad;
 /** {@link coreCompactSegment} with the roaring codec pre-bound. */
 declare const compactSegment: CompactSegment;
@@ -191,9 +199,18 @@ interface CloudRoaringOptions {
      */
     readonly warmReadConsistency?: 'strong' | 'eventual';
     /**
-     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher). Default **1** (serial —
-     * unchanged). Raise it to fan distinct-chunk writes out for throughput on wide multi-chunk batches; each
-     * chunk is its own OCC row so this is safe, and (as before) a mid-flush failure can leave a partial result.
+     * Max Warm chunk writes in flight per `addMany`/`removeMany` (the bounded flusher). Default **4**
+     * (`DEFAULT_WRITE_CONCURRENCY`); set it to `1` for strictly serial writes.
+     *
+     * Distinct chunks are independent OCC rows, so fanning them out cannot make them conflict with each other.
+     * The bound exists for the *backend*: a provisioned-capacity store answers a burst by throttling, and 4 stays
+     * comfortably inside what the transient-retry path absorbs (measured 8x below the first observed failure).
+     * Raise it if your backend is provisioned for the fan-out — on-demand DynamoDB, or a connection pool sized to
+     * match. A pool *smaller* than this needs no action: `pg` and `mysql2` both queue rather than reject, so the
+     * flusher simply degrades toward serial.
+     *
+     * As before, a mid-flush failure can leave a **partial result**; `addMany`/`removeMany` are not atomic. What
+     * concurrency changes is only how much of the batch may already have landed when the first error surfaces.
      */
     readonly writeConcurrency?: number;
     /**
@@ -402,6 +419,29 @@ declare class CloudRoaring {
  * applied one chunk at a time, so if a later chunk fails (e.g. {@link WriteConflictError} after retries)
  * earlier chunks are already applied. Within a single chunk the update is atomic.
  */
+/** Options common to every chunk-aligned combine (`intersect` / `union` / `andNot`). */
+interface BaseCombineOptions {
+    /** Max chunk keys resolved concurrently — bounds the Cold footprint. A positive integer. */
+    readonly concurrency?: number;
+    /** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
+    readonly budget?: BudgetOption;
+}
+/**
+ * A combine that can also subtract.
+ *
+ * `exclude` is why suppression does not need an intermediate segment: applied here it folds into the same
+ * chunk-aligned pass, and each exclude is read **only at the keys that survived** — so a large global opt-out
+ * list costs reads proportional to the audience, not to itself.
+ */
+interface CombineOptions extends BaseCombineOptions {
+    /** Segments whose ids are subtracted from the result. */
+    readonly exclude?: Segment[];
+}
+/** {@link CombineOptions} plus the write batching used by the `*Into` variants. */
+interface CombineIntoOptions extends CombineOptions {
+    /** Ids buffered per `addMany` while materializing into `dest`. */
+    readonly batchSize?: number;
+}
 declare class Segment {
     private readonly engine;
     private readonly ref;
@@ -416,12 +456,24 @@ declare class Segment {
     private timed;
     /** Add an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
     add(id: number): Promise<void>;
-    /** Add many ids. Not atomic across chunks — see the class note. */
-    addMany(ids: Iterable<number>): Promise<void>;
+    /**
+     * Add many ids, from a **sync or async** iterable. Not atomic across chunks — see the class note.
+     *
+     * An async source means a database cursor streams straight in — `addMany(athenaCursor())` rather than
+     * hand-batching `page → addMany(page)`. Ids are grouped by chunk and each chunk is written **exactly once**,
+     * however long the stream: pending ids are held compressed, not buffered-and-flushed, so a long stream costs
+     * no more backend writes than a short one. Peak memory is the roaring representation of the ids you pass.
+     *
+     * **For a very large set, reach for `bulkLoadCrbmGeneration` instead.** `addMany` expresses a *delta* and
+     * writes one warm row per touched chunk — around 61,000 of them for a set spread across the whole id space.
+     * Bulk-load *replaces* the segment with one immutable Cold object. See the guide's cost comparison; picking
+     * the wrong one of these is the most expensive mistake available in this library.
+     */
+    addMany(ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
     /** Remove an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
     remove(id: number): Promise<void>;
-    /** Remove many ids. Not atomic across chunks — see the class note. */
-    removeMany(ids: Iterable<number>): Promise<void>;
+    /** Remove many ids, from a **sync or async** iterable. Not atomic across chunks — see {@link addMany}. */
+    removeMany(ids: Iterable<number> | AsyncIterable<number>): Promise<void>;
     has(id: number): Promise<boolean>;
     count(): Promise<number>;
     iterate(): AsyncIterable<number>;
@@ -434,16 +486,44 @@ declare class Segment {
      * up front, so total memory also carries each operand's Warm size — negligible under Topology-A.) Pass
      * `budget` to override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
-    intersect(others: Segment[], options?: {
-        concurrency?: number;
-        budget?: BudgetOption;
-    }): AsyncIterable<number>;
     /**
-     * Materialize `this ∩ others…` **into** `dest` (added, not replaced). Streaming + bounded-memory, but
-     * **not atomic** across chunks — see {@link addMany}.
+     * Map the facade's `Segment` handles in `exclude` down to the plain refs `core` takes. A method rather than
+     * a module function because `ref` is class-private — the encapsulation is worth more than the free function.
      */
-    intersectInto(dest: Segment, others: Segment[], options?: {
-        concurrency?: number;
+    private refsIn;
+    intersect(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    /**
+     * Materialize `this ∩ others…` (minus `exclude`) **into** `dest` (added, not replaced). Streaming +
+     * bounded-memory, but **not atomic** across chunks — see {@link addMany}.
+     */
+    intersectInto(dest: Segment, others: Segment[], options?: CombineIntoOptions): Promise<void>;
+    /**
+     * `this ∪ others…`, minus `options.exclude` — streamed ascending.
+     *
+     * **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
+     * here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
+     * `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
+     * same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
+     * deliberately. If you find yourself unioning the same segments on every read, materializing the combined
+     * segment once (`unionInto`, or a bulk-load rebuild) is the cheaper shape.
+     */
+    union(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    /** Materialize `this ∪ others…` (minus `exclude`) **into** `dest`. **Not atomic** — see {@link addMany}. */
+    unionInto(dest: Segment, others: Segment[], options?: CombineIntoOptions): Promise<void>;
+    /**
+     * `this \ (excludes…)` — streamed ascending. Suppression on its own.
+     *
+     * Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
+     * overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
+     * suppression list. Subtracting a 61,000-chunk global opt-out list from a narrow audience costs a handful of
+     * reads, not 61,000.
+     *
+     * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
+     * the suppression folds into the same pass rather than materializing an intermediate segment first.
+     */
+    andNot(excludes: Segment[], options?: BaseCombineOptions): AsyncIterable<number>;
+    /** Materialize `this \ (excludes…)` **into** `dest`. **Not atomic** — see {@link addMany}. */
+    andNotInto(dest: Segment, excludes: Segment[], options?: BaseCombineOptions & {
         batchSize?: number;
     }): Promise<void>;
     /**
@@ -459,6 +539,6 @@ declare class Segment {
 }
 
 /** Package version marker. Kept in sync with package.json at release. */
-declare const VERSION = "0.3.0";
+declare const VERSION = "0.4.0";
 
-export { CloudRoaring, type CloudRoaringOptions, type EraseSubjectResult, SafeBitmap, Segment, type SegmentOptions, type SubjectErasureEntry, type SubjectReport, type SubjectSegmentRef, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };
+export { type BaseCombineOptions, CloudRoaring, type CloudRoaringOptions, type CombineIntoOptions, type CombineOptions, type EraseSubjectResult, SafeBitmap, Segment, type SegmentOptions, type SubjectErasureEntry, type SubjectReport, type SubjectSegmentRef, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };

package/dist/index.js

@@ -92,12 +92,8 @@ var roaringCodec = {
   fromValues: (values) => SafeBitmap.fromValues(values),
   safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
 };
-var bulkLoadCrbmGeneration = (driver, key, ids, options = {}) => bulkLoadCrbmGeneration$1(driver, key, ids, { ...options, codec: options.codec ?? roaringCodec });
-var compactSegment = (ref, deps, options) => compactSegment$1(ref, { ...deps, codec: deps.codec ?? roaringCodec }, options);
-var runCompactionCycle = (deps, options) => runCompactionCycle$1({ ...deps, codec: deps.codec ?? roaringCodec }, options);
-var runExport = (reader, registry, sink, options = {}) => runExport$1(reader, registry, sink, { ...options, codec: options.codec ?? roaringCodec });
 
-// src/index.ts
+// src/system-clock.ts
 var SystemClock = class {
   now() {
     return Date.now();
@@ -108,7 +104,43 @@ var SystemClock = class {
       setTimeout(resolve, ms);
     });
   }
+  /**
+   * `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
+   *
+   * A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
+   * co-resident HTTP server keeps answering. Measured over a 61,035-chunk load, yielding every 1,024 chunks:
+   *
+   * ```text
+   *   no yield         568 ms wall   568.0 ms worst event-loop gap
+   *   sleep(0)         555 ms wall   555.2 ms   ← a microtask. Yields NOTHING.
+   *   setImmediate     569 ms wall    13.8 ms   ← free, and 41x less starvation
+   *   setTimeout(1)    625 ms wall    15.0 ms   ← +10% wall for no extra relief
+   * ```
+   *
+   * `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
+   * before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
+   * callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
+   * server wants — inbound requests are serviced before the loop resumes chewing.
+   *
+   * A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
+   * "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
+   */
+  yieldNow() {
+    return new Promise((resolve) => {
+      setImmediate(resolve);
+    });
+  }
 };
+var bulkLoadCrbmGeneration = (driver, key, ids, options = {}) => bulkLoadCrbmGeneration$1(driver, key, ids, {
+  ...options,
+  codec: options.codec ?? roaringCodec,
+  clock: options.clock ?? new SystemClock()
+});
+var compactSegment = (ref, deps, options) => compactSegment$1(ref, { ...deps, codec: deps.codec ?? roaringCodec }, options);
+var runCompactionCycle = (deps, options) => runCompactionCycle$1({ ...deps, codec: deps.codec ?? roaringCodec }, options);
+var runExport = (reader, registry, sink, options = {}) => runExport$1(reader, registry, sink, { ...options, codec: options.codec ?? roaringCodec });
+
+// src/index.ts
 var SystemRng = class {
   next() {
     return Math.random();
@@ -486,7 +518,19 @@ var Segment = class {
   add(id) {
     return this.timed("add", () => this.engine.add(this.ref, id));
   }
-  /** Add many ids. Not atomic across chunks — see the class note. */
+  /**
+   * Add many ids, from a **sync or async** iterable. Not atomic across chunks — see the class note.
+   *
+   * An async source means a database cursor streams straight in — `addMany(athenaCursor())` rather than
+   * hand-batching `page → addMany(page)`. Ids are grouped by chunk and each chunk is written **exactly once**,
+   * however long the stream: pending ids are held compressed, not buffered-and-flushed, so a long stream costs
+   * no more backend writes than a short one. Peak memory is the roaring representation of the ids you pass.
+   *
+   * **For a very large set, reach for `bulkLoadCrbmGeneration` instead.** `addMany` expresses a *delta* and
+   * writes one warm row per touched chunk — around 61,000 of them for a set spread across the whole id space.
+   * Bulk-load *replaces* the segment with one immutable Cold object. See the guide's cost comparison; picking
+   * the wrong one of these is the most expensive mistake available in this library.
+   */
   addMany(ids) {
     return this.timed("addMany", () => this.engine.addMany(this.ref, ids));
   }
@@ -494,7 +538,7 @@ var Segment = class {
   remove(id) {
     return this.timed("remove", () => this.engine.remove(this.ref, id));
   }
-  /** Remove many ids. Not atomic across chunks — see the class note. */
+  /** Remove many ids, from a **sync or async** iterable. Not atomic across chunks — see {@link addMany}. */
   removeMany(ids) {
     return this.timed("removeMany", () => this.engine.removeMany(this.ref, ids));
   }
@@ -516,17 +560,84 @@ var Segment = class {
    * up front, so total memory also carries each operand's Warm size — negligible under Topology-A.) Pass
    * `budget` to override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
+  /**
+   * Map the facade's `Segment` handles in `exclude` down to the plain refs `core` takes. A method rather than
+   * a module function because `ref` is class-private — the encapsulation is worth more than the free function.
+   */
+  refsIn(options) {
+    if (options === void 0) return void 0;
+    const { exclude, ...rest } = options;
+    return exclude === void 0 ? rest : { ...rest, exclude: exclude.map((o) => o.ref) };
+  }
   intersect(others, options) {
-    return this.engine.intersect([this.ref, ...others.map((o) => o.ref)], options);
+    return this.engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
   }
   /**
-   * Materialize `this ∩ others…` **into** `dest` (added, not replaced). Streaming + bounded-memory, but
-   * **not atomic** across chunks — see {@link addMany}.
+   * Materialize `this ∩ others…` (minus `exclude`) **into** `dest` (added, not replaced). Streaming +
+   * bounded-memory, but **not atomic** across chunks — see {@link addMany}.
    */
   intersectInto(dest, others, options) {
     return this.timed(
       "intersectInto",
-      () => this.engine.intersectInto(dest.ref, [this.ref, ...others.map((o) => o.ref)], options)
+      () => this.engine.intersectInto(
+        dest.ref,
+        [this.ref, ...others.map((o) => o.ref)],
+        this.refsIn(options)
+      )
+    );
+  }
+  /**
+   * `this ∪ others…`, minus `options.exclude` — streamed ascending.
+   *
+   * **The one composite read with no chunk-skipping,** and that is inherent to union rather than a limitation
+   * here: an id in *any* operand belongs to the result, so every chunk of every operand must be read.
+   * `intersect` prunes any key missing from any operand; union has nothing to prune. It is charged against the
+   * same per-op budget, so a wide union is refused rather than quietly billed — pass `budget` to raise it
+   * deliberately. If you find yourself unioning the same segments on every read, materializing the combined
+   * segment once (`unionInto`, or a bulk-load rebuild) is the cheaper shape.
+   */
+  union(others, options) {
+    return this.engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options));
+  }
+  /** Materialize `this ∪ others…` (minus `exclude`) **into** `dest`. **Not atomic** — see {@link addMany}. */
+  unionInto(dest, others, options) {
+    return this.timed(
+      "unionInto",
+      () => this.engine.unionInto(
+        dest.ref,
+        [this.ref, ...others.map((o) => o.ref)],
+        this.refsIn(options)
+      )
+    );
+  }
+  /**
+   * `this \ (excludes…)` — streamed ascending. Suppression on its own.
+   *
+   * Reads every chunk of `this` (any of them may survive the subtraction) but each exclude **only where it
+   * overlaps this segment**, so the cost tracks the segment being filtered rather than the size of the
+   * suppression list. Subtracting a 61,000-chunk global opt-out list from a narrow audience costs a handful of
+   * reads, not 61,000.
+   *
+   * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
+   * the suppression folds into the same pass rather than materializing an intermediate segment first.
+   */
+  andNot(excludes, options) {
+    return this.engine.andNot(
+      this.ref,
+      excludes.map((o) => o.ref),
+      options
+    );
+  }
+  /** Materialize `this \ (excludes…)` **into** `dest`. **Not atomic** — see {@link addMany}. */
+  andNotInto(dest, excludes, options) {
+    return this.timed(
+      "andNotInto",
+      () => this.engine.andNotInto(
+        dest.ref,
+        this.ref,
+        excludes.map((o) => o.ref),
+        options
+      )
     );
   }
   /**
@@ -547,7 +658,7 @@ var Segment = class {
     });
   }
 };
-var VERSION = "0.3.0";
+var VERSION = "0.4.0";
 
 export { CloudRoaring, SafeBitmap, Segment, VERSION, bulkLoadCrbmGeneration, compactSegment, roaringCodec, runCompactionCycle, runExport };
 //# sourceMappingURL=index.js.map