@cloudbitmaps/roaring 0.14.0 → 0.16.0

package/dist/index.d.ts

@@ -136,14 +136,17 @@ export interface CacheOptions {
     /**
      * Ceiling on how many segments' `.crbm` readers (each holding a parsed index) the store keeps open at once
      * (default 1024) — the steady-state memory bound for a long-running server that reads across many segments.
-     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET.
+     * Past it the least-recently-used segment's reader is evicted; re-opening it later is one cheap tail GET. A reader of a small generation, one whose whole object came with its tail read,
+     * also holds its chunk bytes, so a read of it makes no chunk request until the reader is evicted or the pointer refresh
+     * moves it on; a store with no timed refresh (`genTtlMs: 0`, or no registry) keeps none.
      * Applies whenever the store builds its own read path — a backend or a bare `IStorageDriver`. A pre-built
      * `StorageChunkSource` manages its own reader cache.
      */
     readonly readerMax?: number;
     /**
-     * Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices and the metadata a
-     * generation carries (default 64 MiB) — the byte half
+     * Aggregate byte ceiling on what the open readers hold, their parsed `.crbm` indices, the metadata a
+     * generation carries, and the chunk bytes a reader keeps when its tail read returned a whole object whose chunks
+     * total at most this ceiling divided by {@link CacheOptions.readerMax} (default 64 MiB) — the byte half
      * of the memory bound, complementing the {@link CacheOptions.readerMax} *count* bound. A wide/dense segment's
      * parsed index can reach about 1.3 MB, so a count-only bound could let the open readers pin over a GB and blow a small
      * heap (e.g. a 128 MB Lambda); this evicts the least-recently-used reader once the summed index footprint
@@ -959,11 +962,15 @@ export declare class CloudRoaring {
  */
 export interface BaseCombineOptions extends IdRange {
     /**
-     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
-     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
-     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
-     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
-     * `concurrency` keys per operand past the last one it used.
+     * How far ahead each operand is read: a positive integer, default 32. The library's own stores read each operand as
+     * one stream of coalesced ranges (chunks that sit near each other are one request), and `concurrency` is how many
+     * range requests a stream holds ahead of the read, in flight or landed and not yet used. So about
+     * `concurrency × operands` requests are in flight at most, and, since a range is at most 1 MiB (one chunk, when a chunk
+     * is larger), about `concurrency × operands` MiB are held, whatever the segment's size. The stream opens 4 ranges wide
+     * (or `concurrency`, if lower) and widens as ranges are taken, so a read that stops early has asked for little past
+     * where it stopped, and a read that fits one range makes one request however many chunks it needs. A source that
+     * reads chunk by chunk (a custom one) is read as before: `concurrency` chunk keys at once, opening 8 wide. A running
+     * stream holds its operand's reader outside the reader cache's bounds until it ends or the segment moves on.
      */
     readonly concurrency?: number;
     /** Override the store's per-op denial-of-wallet budget for this call (`false` lifts it). */
@@ -1318,12 +1325,14 @@ export declare class Segment {
     /**
      * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
      * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
-     * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-     * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
+     * and is never downloaded), reading each operand's as coalesced ranges under a bounded in-flight window — so the Storage
+     * footprint stays small (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (the range
+     * requests held ahead per operand: a positive integer, 32 by default).
      * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
      * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
     intersect(others: Segment[], options?: CombineOptions): IdStream;
+    private intersectAs;
     /**
      * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
      * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
@@ -1356,6 +1365,7 @@ export declare class Segment {
      * segment once (`unionInto`, or a load) is the cheaper shape.
      */
     union(others: Segment[], options?: CombineOptions): IdStream;
+    private unionAs;
     /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
     unionInto(dest: Segment, others: Segment[], options?: MaterializeOptions): Promise<MaterializeResult>;
     /**
@@ -1374,6 +1384,7 @@ export declare class Segment {
      * the suppression folds into the same pass rather than materializing an intermediate segment first.
      */
     andNot(excludes: Segment[], options?: BaseCombineOptions): IdStream;
+    private andNotAs;
     /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
     andNotInto(dest: Segment, excludes: Segment[], options?: AndNotIntoOptions): Promise<MaterializeResult>;
     /**
@@ -1389,4 +1400,4 @@ export declare class Segment {
     }): Promise<CostReport>;
 }
 export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
-export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
+export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';

package/dist/index.js

@@ -23,6 +23,7 @@ import {
   estimateCost,
   groundedReport,
   loadSegment,
+  loadSegmentChunks,
   mapWithConcurrency,
   resolveBudget,
   resolvePerOpBudget,
@@ -450,7 +451,8 @@ function bitmapAsLoadInput(input) {
 var roaringCodec = {
   empty: () => SafeBitmap.empty(),
   fromValues: (values) => SafeBitmap.fromValues(values),
-  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options)
+  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
+  owns: (bitmap) => bitmap instanceof SafeBitmap
 };
 
 // src/system-clock.ts
@@ -875,7 +877,7 @@ var CloudRoaring = class _CloudRoaring {
     const metadata = options?.metadata;
     let result;
     try {
-      result = await loadSegment(dest, ids, deps, {
+      const loadOptions = {
         ...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
         ...options?.guard === void 0 ? {} : { guard: options.guard },
         ...metadata === void 0 ? {} : { metadata },
@@ -888,7 +890,8 @@ var CloudRoaring = class _CloudRoaring {
         // as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
         keep: options?.keep ?? KEEP_EVERY_GENERATION,
         ...options?.audit === void 0 ? {} : { audit: options.audit }
-      });
+      };
+      result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
     } finally {
       this.engine.invalidate(dest);
     }
@@ -1692,6 +1695,32 @@ var EMPTY_IDS = {
   async *batches() {
   }
 };
+var AS_IDS = {
+  none: EMPTY_IDS,
+  failing,
+  intersect: (engine, refs, opts) => withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts)),
+  union: (engine, refs, opts) => withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts)),
+  andNot: (engine, base, refs, opts) => withBatches(engine.andNot(base, refs, opts), () => engine.andNotBatches(base, refs, opts))
+};
+var NO_CHUNKS = {
+  async *[Symbol.asyncIterator]() {
+  }
+};
+var CombineChunks = class {
+  constructor(chunks) {
+    this.chunks = chunks;
+  }
+  chunks;
+};
+var AS_CHUNKS = {
+  none: NO_CHUNKS,
+  failing: (err) => ({
+    [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
+  }),
+  intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
+  union: (engine, refs, opts) => engine.unionChunks(refs, opts),
+  andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
+};
 var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
 var makeSegment;
 var minting = false;
@@ -1983,23 +2012,27 @@ var Segment = class _Segment {
   /**
    * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
    * Fetches only the Storage chunks present in *all* operands (a key absent from any operand contributes nothing
-   * and is never downloaded), streaming under a bounded in-flight window — so the Storage footprint stays small
-   * (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer, 32 by default).
+   * and is never downloaded), reading each operand's as coalesced ranges under a bounded in-flight window — so the Storage
+   * footprint stays small (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (the range
+   * requests held ahead per operand: a positive integer, 32 by default).
    * AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. Pass `budget` to
    * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
   intersect(others, options) {
-    if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
+    return this.intersectAs(AS_IDS, others, options);
+  }
+  intersectAs(out, others, options) {
+    if (this.expired() || others.some((o) => o.expired())) return out.none;
     const exclude = this.liveExcludes(options);
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts));
+    return out.intersect(engine, refs, opts);
   }
   /**
    * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
@@ -2026,7 +2059,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "intersectInto",
-      () => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
+        "intersectInto",
+        options
+      )
     );
   }
   /**
@@ -2040,26 +2078,29 @@ var Segment = class _Segment {
    * segment once (`unionInto`, or a load) is the cheaper shape.
    */
   union(others, options) {
+    return this.unionAs(AS_IDS, others, options);
+  }
+  unionAs(out, others, options) {
     const live = others.filter((o) => !o.expired());
     if (this.expired()) {
-      if (live.length === 0) return EMPTY_IDS;
-      return live[0].union(live.slice(1), options);
+      if (live.length === 0) return out.none;
+      return live[0].unionAs(out, live.slice(1), options);
     }
     if (live.length === 0 && others.length > 0) {
       const exclude2 = this.liveExcludes(options);
-      return exclude2.length > 0 ? this.andNot([...exclude2], options) : this.union([], options);
+      return exclude2.length > 0 ? this.andNotAs(out, [...exclude2], options) : this.unionAs(out, [], options);
     }
-    if (live.length !== others.length) return this.union(live, options);
+    if (live.length !== others.length) return this.unionAs(out, live, options);
     const exclude = this.liveExcludes(options);
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts));
+    return out.union(engine, refs, opts);
   }
   /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
   async unionInto(dest, others, options) {
@@ -2067,7 +2108,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "unionInto",
-      () => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
+        "unionInto",
+        options
+      )
     );
   }
   /**
@@ -2086,25 +2132,25 @@ var Segment = class _Segment {
    * the suppression folds into the same pass rather than materializing an intermediate segment first.
    */
   andNot(excludes, options) {
-    if (this.expired()) return EMPTY_IDS;
+    return this.andNotAs(AS_IDS, excludes, options);
+  }
+  andNotAs(out, excludes, options) {
+    if (this.expired()) return out.none;
     const liveExcludes = excludes.filter((e) => !e.expired());
     if (liveExcludes.length === 0 && excludes.length > 0) {
-      return this.union([], options == null ? void 0 : readOptions(options));
+      return this.unionAs(out, [], options == null ? void 0 : readOptions(options));
     }
-    if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
+    if (liveExcludes.length !== excludes.length) return this.andNotAs(out, liveExcludes, options);
     let engine;
     try {
       engine = this.combineEngine([this, ...excludes]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const base = this.ref;
     const refs = excludes.map((o) => o.ref);
     const opts = options == null ? void 0 : readOptions(options);
-    return withBatches(
-      engine.andNot(base, refs, opts),
-      () => engine.andNotBatches(base, refs, opts)
-    );
+    return out.andNot(engine, base, refs, opts);
   }
   /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
   async andNotInto(dest, excludes, options) {
@@ -2112,7 +2158,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...excludes]);
     return this.timed(
       "andNotInto",
-      () => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.andNotAs(AS_CHUNKS, excludes, options)),
+        "andNotInto",
+        options
+      )
     );
   }
   /**