@cloudbitmaps/roaring 0.12.0 → 0.14.0

package/dist/index.d.ts

@@ -958,7 +958,13 @@ export declare class CloudRoaring {
  * in `(after, through]` and fetches only the chunks the range overlaps, as {@link Segment.iterate} does.
  */
 export interface BaseCombineOptions extends IdRange {
-    /** Max chunk keys resolved concurrently — bounds the Storage footprint. A positive integer. */
+    /**
+     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
+     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
+     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
+     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
+     * `concurrency` keys per operand past the last one it used.
+     */
     readonly concurrency?: number;
     /** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
     readonly budget?: BudgetOption;
@@ -1045,6 +1051,36 @@ export interface MaterializeOptions extends CombineOptions {
      */
     readonly keep?: number;
 }
+/**
+ * What a streaming read returns: the ids, ascending, one at a time under `for await`, or one chunk at a time from
+ * {@link IdStream.batches}.
+ *
+ * It is an `AsyncIterable<number>`, so every consumer of an id stream takes it as it always did. A live read is the
+ * engine's async generator itself, with `batches` attached: `for await` it, or drive it with `next()`, `return()` and
+ * `throw()`, exactly as before, and it is single-use (a second `for await` over it yields nothing). `batches()` is a
+ * separate, new read: it starts when called, fetches its chunks afresh and charges the per-op budget again, whether or
+ * not the per-id stream was read, and reading both is two reads (invariant 3 applies to each separately). Nothing is
+ * fetched until a read is first pulled.
+ */
+export interface IdStream extends AsyncIterable<number> {
+    /**
+     * The same ids as the per-id stream, in the same order, one `Uint32Array` per chunk: each array ascending, holding
+     * every id of one chunk the read yields (a range trimmed at its edges exactly as the per-id stream trims it), and
+     * no array is empty. Chunks arrive in ascending order, so the concatenation is the per-id stream.
+     *
+     * One `await` per chunk instead of one per id: measured locally, an in-memory read goes several times faster, and
+     * a large scan stops being bound by the event loop. The reads, the budget and the read-ahead window are the per-id
+     * stream's, and so is stopping early: leaving a `for await` over it (`break`, `return`, a throw) ends the read.
+     *
+     * **Memory:** one array per chunk, at most 65,536 ids (256 KiB), allocated for that chunk alone and yours to
+     * keep, change or hand on; the read holds no reference to it after yielding it.
+     *
+     * ```ts
+     * for await (const ids of seg.andNot([optOut]).batches()) await send(ids);
+     * ```
+     */
+    batches(): AsyncIterable<Uint32Array>;
+}
 /**
  * What `andNotInto` takes: the write options without `exclude`, because its `excludes` argument IS the
  * subtraction — a second one in the options would be two spellings of one thing.
@@ -1257,7 +1293,7 @@ export declare class Segment {
      * stream is first read. `after >= through` is an empty range, which reads nothing. An expired handle reads empty
      * without checking its options, as every read of one does.
      */
-    iterate(options?: IdRange): AsyncIterable<number>;
+    iterate(options?: IdRange): IdStream;
     /**
      * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
      * it was given, already reduced to its live handles ({@link liveExcludes}) and mapped down to the plain refs
@@ -1283,11 +1319,11 @@ export declare class Segment {
      * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
      * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
      * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-     * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
+     * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
      * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
      * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
-    intersect(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    intersect(others: Segment[], options?: CombineOptions): IdStream;
     /**
      * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
      * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
@@ -1319,7 +1355,7 @@ export declare class Segment {
      * deliberately. If you find yourself unioning the same segments on every read, materializing the combined
      * segment once (`unionInto`, or a load) is the cheaper shape.
      */
-    union(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    union(others: Segment[], options?: CombineOptions): IdStream;
     /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
     unionInto(dest: Segment, others: Segment[], options?: MaterializeOptions): Promise<MaterializeResult>;
     /**
@@ -1337,7 +1373,7 @@ export declare class Segment {
      * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
      * the suppression folds into the same pass rather than materializing an intermediate segment first.
      */
-    andNot(excludes: Segment[], options?: BaseCombineOptions): AsyncIterable<number>;
+    andNot(excludes: Segment[], options?: BaseCombineOptions): IdStream;
     /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
     andNotInto(dest: Segment, excludes: Segment[], options?: AndNotIntoOptions): Promise<MaterializeResult>;
     /**

package/dist/index.js

@@ -439,6 +439,10 @@ var SafeBitmap = class _SafeBitmap {
   toArray() {
     return this.bitmap.toArray();
   }
+  /** The values as one fresh ascending `Uint32Array`, exported natively. */
+  toUint32Array() {
+    return this.bitmap.toUint32Array();
+  }
 };
 function bitmapAsLoadInput(input) {
   return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
@@ -1658,9 +1662,14 @@ var CloudRoaring = class _CloudRoaring {
     return estimateCost(input);
   }
 };
-var failing = (err) => ({
-  [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
-});
+var withBatches = (gen, batches) => Object.assign(gen, { batches });
+var failing = (err) => {
+  const rejecting = () => ({ next: () => Promise.reject(err) });
+  return {
+    [Symbol.asyncIterator]: rejecting,
+    batches: () => ({ [Symbol.asyncIterator]: rejecting })
+  };
+};
 var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
 var twoPins = (a, b) => {
   const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
@@ -1679,6 +1688,8 @@ var readOptions = (options) => ({
 var NO_SEGMENTS = [];
 var EMPTY_IDS = {
   async *[Symbol.asyncIterator]() {
+  },
+  async *batches() {
   }
 };
 var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
@@ -1934,7 +1945,9 @@ var Segment = class _Segment {
     if (this.expired()) return EMPTY_IDS;
     const range = options == null ? void 0 : rangeOf(options);
     const none = range === void 0 || range.after === void 0 && range.through === void 0;
-    return this.engine.iterate(this.ref, none ? void 0 : range);
+    const { engine, ref } = this;
+    const bounds = none ? void 0 : range;
+    return withBatches(engine.iterate(ref, bounds), () => engine.iterateBatches(ref, bounds));
   }
   /**
    * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
@@ -1971,7 +1984,7 @@ var Segment = class _Segment {
    * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
    * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
    * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-   * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
+   * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
    * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
    * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
@@ -1984,7 +1997,9 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
+    const refs = [this.ref, ...others.map((o) => o.ref)];
+    const opts = this.refsIn(options, exclude);
+    return withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts));
   }
   /**
    * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
@@ -2042,7 +2057,9 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
+    const refs = [this.ref, ...others.map((o) => o.ref)];
+    const opts = this.refsIn(options, exclude);
+    return withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts));
   }
   /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
   async unionInto(dest, others, options) {
@@ -2081,10 +2098,12 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.andNot(
-      this.ref,
-      excludes.map((o) => o.ref),
-      options == null ? void 0 : readOptions(options)
+    const base = this.ref;
+    const refs = excludes.map((o) => o.ref);
+    const opts = options == null ? void 0 : readOptions(options);
+    return withBatches(
+      engine.andNot(base, refs, opts),
+      () => engine.andNotBatches(base, refs, opts)
     );
   }
   /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */