@cloudbitmaps/roaring 0.15.0 → 0.16.0

package/dist/index.d.ts

@@ -1332,6 +1332,7 @@ export declare class Segment {
      * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
      */
     intersect(others: Segment[], options?: CombineOptions): IdStream;
+    private intersectAs;
     /**
      * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
      * are superseded, not added to. Streaming + bounded-memory; the result is one immutable object published
@@ -1364,6 +1365,7 @@ export declare class Segment {
      * segment once (`unionInto`, or a load) is the cheaper shape.
      */
     union(others: Segment[], options?: CombineOptions): IdStream;
+    private unionAs;
     /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
     unionInto(dest: Segment, others: Segment[], options?: MaterializeOptions): Promise<MaterializeResult>;
     /**
@@ -1382,6 +1384,7 @@ export declare class Segment {
      * the suppression folds into the same pass rather than materializing an intermediate segment first.
      */
     andNot(excludes: Segment[], options?: BaseCombineOptions): IdStream;
+    private andNotAs;
     /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
     andNotInto(dest: Segment, excludes: Segment[], options?: AndNotIntoOptions): Promise<MaterializeResult>;
     /**
@@ -1397,4 +1400,4 @@ export declare class Segment {
     }): Promise<CostReport>;
 }
 export { MemoryStorage, LocalFsStorage, CrbmStorageChunkSource, destroySegment, eraseNamespace, excludingReservedRows, readRetentionPolicy, MIN_EXPIRES_AT_MS, InProcessKeystore, NodeAead, aadFor, CloudRoaringError, ValidationError, WriteConflictError, IntegrityError, NotFoundError, UnsupportedError, CapabilityError, TransientError, KeyUnavailableError, BudgetExceededError, isCloudRoaringError, isWriteConflictError, isTransientError, isNotFoundError, isIntegrityError, isValidationError, CrbmReader, BufferReader, CountingMetricsSink, RecordingAuditSink, DEFAULT_RETRY_POLICY, AWS_US_EAST_1_ONDEMAND, ELASTICACHE_REDIS_US_EAST_1_ONDEMAND, ONE_REDIS_HA_CLUSTER, } from '@cloudbitmaps/core';
-export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';
+export type { Aead, AeadSealed, AuditEvent, BlobReader, BlobSink, Budget, BudgetOption, ChunkRef, Clock, CodecBitmap, CodecInterface, ConsistencyErrorEntry, ConsistencyIssue, ConsistencyReport, CostReport, CrbmCrypto, CrbmReaderOptions, CrbmStorageChunkSourceOptions, DestroyResult, ClearRegistrySummary, DropResult, EncodedChunk, EraseDeps, EstimateInput, ExportFailure, ExportFormat, ExportManifest, ExportOptions, ExportSink, ExportWriter, ExportedSegment, GenKey, GenerationEntry, GenerationMetadata, GovernanceMeta, IAuditSink, IKeystore, IMetricsSink, IRegistryDriver, IStorageDriver, IdRange, InProcessKeystoreOptions, LoadGuard, LoadInput, LoadOptions, LoadRefusal, LoadResult, LocalFsStorageOptions, MemoryStorageOptions, MetricEvent, MetricOpName, MetricsSnapshot, NewRegistryRecord, PinnedAt, PinnedObject, PortableBitmap, PricingProfile, RedisNodeType, RedisSizing, RegCaps, RegistryPatch, RegistryRecord, RegistryStatus, RegistrySummary, RegistryWriteOptions, RetentionPolicy, RetireEntry, RetireExpiredOptions, RetireExpiredResult, RetryPolicy, Rng, RollbackResult, SealedRegistrySummary, SegmentInfo, SegmentRef, SegmentSize, SegmentSizing, SetRetentionResult, StorageBackend, StorageCaps, StorageChunkSource, Token, Workload, WrappedDek, } from '@cloudbitmaps/core';

package/dist/index.js

@@ -23,6 +23,7 @@ import {
   estimateCost,
   groundedReport,
   loadSegment,
+  loadSegmentChunks,
   mapWithConcurrency,
   resolveBudget,
   resolvePerOpBudget,
@@ -450,7 +451,8 @@ function bitmapAsLoadInput(input) {
 var roaringCodec = {
   empty: () => SafeBitmap.empty(),
   fromValues: (values) => SafeBitmap.fromValues(values),
-  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options)
+  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
+  owns: (bitmap) => bitmap instanceof SafeBitmap
 };
 
 // src/system-clock.ts
@@ -875,7 +877,7 @@ var CloudRoaring = class _CloudRoaring {
     const metadata = options?.metadata;
     let result;
     try {
-      result = await loadSegment(dest, ids, deps, {
+      const loadOptions = {
         ...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
         ...options?.guard === void 0 ? {} : { guard: options.guard },
         ...metadata === void 0 ? {} : { metadata },
@@ -888,7 +890,8 @@ var CloudRoaring = class _CloudRoaring {
         // as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
         keep: options?.keep ?? KEEP_EVERY_GENERATION,
         ...options?.audit === void 0 ? {} : { audit: options.audit }
-      });
+      };
+      result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
     } finally {
       this.engine.invalidate(dest);
     }
@@ -1692,6 +1695,32 @@ var EMPTY_IDS = {
   async *batches() {
   }
 };
+var AS_IDS = {
+  none: EMPTY_IDS,
+  failing,
+  intersect: (engine, refs, opts) => withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts)),
+  union: (engine, refs, opts) => withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts)),
+  andNot: (engine, base, refs, opts) => withBatches(engine.andNot(base, refs, opts), () => engine.andNotBatches(base, refs, opts))
+};
+var NO_CHUNKS = {
+  async *[Symbol.asyncIterator]() {
+  }
+};
+var CombineChunks = class {
+  constructor(chunks) {
+    this.chunks = chunks;
+  }
+  chunks;
+};
+var AS_CHUNKS = {
+  none: NO_CHUNKS,
+  failing: (err) => ({
+    [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
+  }),
+  intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
+  union: (engine, refs, opts) => engine.unionChunks(refs, opts),
+  andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
+};
 var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
 var makeSegment;
 var minting = false;
@@ -1990,17 +2019,20 @@ var Segment = class _Segment {
    * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
   intersect(others, options) {
-    if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
+    return this.intersectAs(AS_IDS, others, options);
+  }
+  intersectAs(out, others, options) {
+    if (this.expired() || others.some((o) => o.expired())) return out.none;
     const exclude = this.liveExcludes(options);
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts));
+    return out.intersect(engine, refs, opts);
   }
   /**
    * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
@@ -2027,7 +2059,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "intersectInto",
-      () => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
+        "intersectInto",
+        options
+      )
     );
   }
   /**
@@ -2041,26 +2078,29 @@ var Segment = class _Segment {
    * segment once (`unionInto`, or a load) is the cheaper shape.
    */
   union(others, options) {
+    return this.unionAs(AS_IDS, others, options);
+  }
+  unionAs(out, others, options) {
     const live = others.filter((o) => !o.expired());
     if (this.expired()) {
-      if (live.length === 0) return EMPTY_IDS;
-      return live[0].union(live.slice(1), options);
+      if (live.length === 0) return out.none;
+      return live[0].unionAs(out, live.slice(1), options);
     }
     if (live.length === 0 && others.length > 0) {
       const exclude2 = this.liveExcludes(options);
-      return exclude2.length > 0 ? this.andNot([...exclude2], options) : this.union([], options);
+      return exclude2.length > 0 ? this.andNotAs(out, [...exclude2], options) : this.unionAs(out, [], options);
     }
-    if (live.length !== others.length) return this.union(live, options);
+    if (live.length !== others.length) return this.unionAs(out, live, options);
     const exclude = this.liveExcludes(options);
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts));
+    return out.union(engine, refs, opts);
   }
   /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
   async unionInto(dest, others, options) {
@@ -2068,7 +2108,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "unionInto",
-      () => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
+        "unionInto",
+        options
+      )
     );
   }
   /**
@@ -2087,25 +2132,25 @@ var Segment = class _Segment {
    * the suppression folds into the same pass rather than materializing an intermediate segment first.
    */
   andNot(excludes, options) {
-    if (this.expired()) return EMPTY_IDS;
+    return this.andNotAs(AS_IDS, excludes, options);
+  }
+  andNotAs(out, excludes, options) {
+    if (this.expired()) return out.none;
     const liveExcludes = excludes.filter((e) => !e.expired());
     if (liveExcludes.length === 0 && excludes.length > 0) {
-      return this.union([], options == null ? void 0 : readOptions(options));
+      return this.unionAs(out, [], options == null ? void 0 : readOptions(options));
     }
-    if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
+    if (liveExcludes.length !== excludes.length) return this.andNotAs(out, liveExcludes, options);
     let engine;
     try {
       engine = this.combineEngine([this, ...excludes]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const base = this.ref;
     const refs = excludes.map((o) => o.ref);
     const opts = options == null ? void 0 : readOptions(options);
-    return withBatches(
-      engine.andNot(base, refs, opts),
-      () => engine.andNotBatches(base, refs, opts)
-    );
+    return out.andNot(engine, base, refs, opts);
   }
   /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
   async andNotInto(dest, excludes, options) {
@@ -2113,7 +2158,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...excludes]);
     return this.timed(
       "andNotInto",
-      () => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.andNotAs(AS_CHUNKS, excludes, options)),
+        "andNotInto",
+        options
+      )
     );
   }
   /**