@cloudbitmaps/roaring 0.13.0 → 0.15.0

package/dist/index.d.ts

@@ -136,14 +136,17 @@ export interface CacheOptions {
     /**
      * Ceiling on how many segments' `.crbm` readers (each holding a parsed index) the store keeps open at once
      * (default 1024) — the steady-state memory bound for a long-running server that reads across many segments.
-     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET.
+     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET. A reader of a small generation, one whose whole object came with its tail read,
+     * also holds its chunk bytes, so a read of it makes no chunk request until the reader is evicted or the pointer refresh
+     * moves it on; a store with no timed refresh (`genTtlMs: 0`, or no registry) keeps none.
      * Applies whenever the store builds its own read path — a backend or a bare `IStorageDriver`. A pre-built
      * `StorageChunkSource` manages its own reader cache.
      */
     readonly readerMax?: number;
     /**
-     * Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices and the metadata a
-     * generation carries (default 64 MiB) — the byte half
+     * Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices, the metadata a
+     * generation carries, and the chunk bytes a reader keeps when its tail read returned a whole object whose chunks
+     * total at most this ceiling divided by {@link CacheOptions.readerMax} (default 64 MiB) — the byte half
      * of the memory bound, complementing the {@link CacheOptions.readerMax} *count* bound. A wide/dense segment's
      * parsed index can reach about 1.3 MB, so a count-only bound could let the open readers pin over a GB and blow a small
      * heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index footprint
@@ -959,11 +962,15 @@ export declare class CloudRoaring {
  */
 export interface BaseCombineOptions extends IdRange {
     /**
-     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
-     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
-     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
-     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
-     * `concurrency` keys per operand past the last one it used.
+     * How far ahead each operand is read: a positive integer, default 32. The library's own stores read each operand as
+     * one stream of coalesced ranges (chunks that sit near each other are one request), and `concurrency` is how many
+     * range requests a stream holds ahead of the read, in flight or landed and not yet used. So about
+     * `concurrency × operands` requests are in flight at most, and, since a range is at most 1 MiB (one chunk, when a chunk
+     * is larger), about `concurrency × operands` MiB are held, whatever the segment's size. The stream opens 4 ranges wide
+     * (or `concurrency`, if lower) and widens as ranges are taken, so a read that stops early has asked for little past
+     * where it stopped, and a read that fits one range makes one request however many chunks it needs. A source that
+     * reads chunk by chunk (a custom one) is read as before: `concurrency` chunk keys at once, opening 8 wide. A running
+     * stream holds its operand's reader outside the reader cache's bounds until it ends or the segment moves on.
      */
     readonly concurrency?: number;
     /** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
@@ -1051,6 +1058,36 @@ export interface MaterializeOptions extends CombineOptions {
      */
     readonly keep?: number;
 }
+/**
+ * What a streaming read returns: the ids, ascending, one at a time under `for await`, or one chunk at a time from
+ * {@link IdStream.batches}.
+ *
+ * It is an `AsyncIterable<number>`, so every consumer of an id stream takes it as it always did. A live read is the
+ * engine's async generator itself, with `batches` attached: `for await` it, or drive it with `next()`, `return()` and
+ * `throw()`, exactly as before, and it is single-use (a second `for await` over it yields nothing). `batches()` is a
+ * separate, new read: it starts when called, fetches its chunks afresh and charges the per-op budget again, whether or
+ * not the per-id stream was read, and reading both is two reads (invariant 3 applies to each separately). Nothing is
+ * fetched until a read is first pulled.
+ */
+export interface IdStream extends AsyncIterable<number> {
+    /**
+     * The same ids as the per-id stream, in the same order, one `Uint32Array` per chunk: each array ascending, holding
+     * every id of one chunk the read yields (a range trimmed at its edges exactly as the per-id stream trims it), and
+     * no array is empty. Chunks arrive in ascending order, so the concatenation is the per-id stream.
+     *
+     * One `await` per chunk instead of one per id: measured locally, an in-memory read goes several times faster, and
+     * a large scan stops being bound by the event loop. The reads, the budget and the read-ahead window are the per-id
+     * stream's, and so is stopping early: leaving a `for await` over it (`break`, `return`, a throw) ends the read.
+     *
+     * **Memory:** one array per chunk, at most 65,536 ids (256 KiB), allocated for that chunk alone and yours to
+     * keep, change or hand on; the read holds no reference to it after yielding it.
+     *
+     * ```ts
+     * for await (const ids of seg.andNot([optOut]).batches()) await send(ids);
+     * ```
+     */
+    batches(): AsyncIterable<Uint32Array>;
+}
 /**
  * What `andNotInto` takes: the write options without `exclude`, because its `excludes` argument IS the
  * subtraction — a second one in the options would be two spellings of one thing.
@@ -1263,7 +1300,7 @@ export declare class Segment {
      * stream is first read. `after >= through` is an empty range, which reads nothing. An expired handle reads empty
      * without checking its options, as every read of one does.
      */
-    iterate(options?: IdRange): AsyncIterable<number>;
+    iterate(options?: IdRange): IdStream;
     /**
      * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
      * it was given, already reduced to its live handles ({@link liveExcludes}) and mapped down to the plain refs
@@ -1288,12 +1325,13 @@ export declare class Segment {
     /**
      * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
      * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
-     * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-     * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
+     * and is never downloaded), reading each operand's as coalesced ranges under a bounded in-flight window — so the Storage
+     * footprint stays small (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (the range
+     * requests held ahead per operand: a positive integer, 32 by default).
      * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
      * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
-    intersect(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    intersect(others: Segment[], options?: CombineOptions): IdStream;
     /**
      * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
      * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
@@ -1325,7 +1363,7 @@ export declare class Segment {
      * deliberately. If you find yourself unioning the same segments on every read, materializing the combined
      * segment once (`unionInto`, or a load) is the cheaper shape.
      */
-    union(others: Segment[], options?: CombineOptions): AsyncIterable<number>;
+    union(others: Segment[], options?: CombineOptions): IdStream;
     /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
     unionInto(dest: Segment, others: Segment[], options?: MaterializeOptions): Promise<MaterializeResult>;
     /**
@@ -1343,7 +1381,7 @@ export declare class Segment {
      * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
      * the suppression folds into the same pass rather than materializing an intermediate segment first.
      */
-    andNot(excludes: Segment[], options?: BaseCombineOptions): AsyncIterable<number>;
+    andNot(excludes: Segment[], options?: BaseCombineOptions): IdStream;
     /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
     andNotInto(dest: Segment, excludes: Segment[], options?: AndNotIntoOptions): Promise<MaterializeResult>;
     /**

package/dist/index.js

@@ -439,6 +439,10 @@ var SafeBitmap = class _SafeBitmap {
   toArray() {
     return this.bitmap.toArray();
   }
+  /** The values as one fresh ascending `Uint32Array`, exported natively. */
+  toUint32Array() {
+    return this.bitmap.toUint32Array();
+  }
 };
 function bitmapAsLoadInput(input) {
   return input instanceof RoaringBitmap32 ? { bitmap: input } : input;
@@ -1658,9 +1662,14 @@ var CloudRoaring = class _CloudRoaring {
     return estimateCost(input);
   }
 };
-var failing = (err) => ({
-  [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
-});
+var withBatches = (gen, batches) => Object.assign(gen, { batches });
+var failing = (err) => {
+  const rejecting = () => ({ next: () => Promise.reject(err) });
+  return {
+    [Symbol.asyncIterator]: rejecting,
+    batches: () => ({ [Symbol.asyncIterator]: rejecting })
+  };
+};
 var samePin = (a, b) => a === void 0 || b === void 0 ? a === b : a.generation === b.generation && a.fingerprint === b.fingerprint;
 var twoPins = (a, b) => {
   const one = (p) => p === void 0 ? "live" : `pinned at generation ${p.generation ?? "none"}`;
@@ -1679,6 +1688,8 @@ var readOptions = (options) => ({
 var NO_SEGMENTS = [];
 var EMPTY_IDS = {
   async *[Symbol.asyncIterator]() {
+  },
+  async *batches() {
   }
 };
 var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
@@ -1934,7 +1945,9 @@ var Segment = class _Segment {
     if (this.expired()) return EMPTY_IDS;
     const range = options == null ? void 0 : rangeOf(options);
     const none = range === void 0 || range.after === void 0 && range.through === void 0;
-    return this.engine.iterate(this.ref, none ? void 0 : range);
+    const { engine, ref } = this;
+    const bounds = none ? void 0 : range;
+    return withBatches(engine.iterate(ref, bounds), () => engine.iterateBatches(ref, bounds));
   }
   /**
    * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
@@ -1970,8 +1983,9 @@ var Segment = class _Segment {
   /**
    * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
    * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
-   * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-   * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
+   * and is never downloaded), reading each operand's as coalesced ranges under a bounded in-flight window — so the Storage
+   * footprint stays small (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (the range
+   * requests held ahead per operand: a positive integer, 32 by default).
    * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
    * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
@@ -1984,7 +1998,9 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.intersect([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
+    const refs = [this.ref, ...others.map((o) => o.ref)];
+    const opts = this.refsIn(options, exclude);
+    return withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts));
   }
   /**
    * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
@@ -2042,7 +2058,9 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.union([this.ref, ...others.map((o) => o.ref)], this.refsIn(options, exclude));
+    const refs = [this.ref, ...others.map((o) => o.ref)];
+    const opts = this.refsIn(options, exclude);
+    return withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts));
   }
   /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
   async unionInto(dest, others, options) {
@@ -2081,10 +2099,12 @@ var Segment = class _Segment {
     } catch (err) {
       return failing(err);
     }
-    return engine.andNot(
-      this.ref,
-      excludes.map((o) => o.ref),
-      options == null ? void 0 : readOptions(options)
+    const base = this.ref;
+    const refs = excludes.map((o) => o.ref);
+    const opts = options == null ? void 0 : readOptions(options);
+    return withBatches(
+      engine.andNot(base, refs, opts),
+      () => engine.andNotBatches(base, refs, opts)
     );
   }
   /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */