@cloudbitmaps/core 0.13.0 → 0.14.0

package/dist/core/codec.d.ts

@@ -52,6 +52,17 @@ export interface CodecBitmap {
     /** Ascending iterator over the set values. */
     [Symbol.iterator](): IterableIterator<number>;
     toArray(): number[];
+    /**
+     * The set's values as one ascending `Uint32Array` the caller owns: what a batch read converts a chunk with, in one
+     * native step instead of one iterator step per value.
+     *
+     * **Optional.** A codec without it is read through its iterator, which is correct and slower.
+     *
+     * **It must return a NEW array on every call, which the caller owns:** the engine adds the chunk's base to each
+     * value in place and hands the array on, so an array that is shared, cached or a view of the bitmap's own storage
+     * would be corrupted, and would corrupt the bitmap. It holds exactly what the iterator yields, ascending.
+     */
+    toUint32Array?(): Uint32Array;
     /**
      * The largest value in the set, or `undefined` when empty.
      *

package/dist/core/engine.d.ts

@@ -128,6 +128,13 @@ export declare class SegmentEngine {
     iterate(seg: SegmentRef, range?: IdRange): AsyncGenerator<number>;
     private iterateAll;
     private iterateRange;
+    /**
+     * {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
+     * the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
+     * reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
+     * the caller's to keep.
+     */
+    iterateBatches(seg: SegmentRef, range?: IdRange): AsyncGenerator<Uint32Array>;
     /**
      * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
      *
@@ -176,7 +183,20 @@ export declare class SegmentEngine {
      * the cost scales with the segment being filtered rather than with the size of the suppression list.
      */
     andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<number>;
+    /** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    intersectBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    unionBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    andNotBatches(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<Uint32Array>;
+    /**
+     * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
+     * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
+     * read and the batch read share it, so the two cannot disagree about what is read or charged.
+     */
+    private combinePlan;
     private combine;
+    private combineBatches;
     /**
      * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
      * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).

package/dist/index.js

@@ -372,6 +372,65 @@ function keysWithin(keys, w) {
   };
   return keys.slice(firstAtLeast(w.loKey), firstAtLeast(w.hiKey + 1));
 }
+var CombineWindow = class {
+  constructor(keys, limit, read) {
+    this.keys = keys;
+    this.limit = limit;
+    this.read = read;
+    this.fill();
+  }
+  keys;
+  limit;
+  read;
+  inFlight = [];
+  next_ = 0;
+  taken = 0;
+  fill() {
+    const width = Math.min(
+      this.limit,
+      Math.max(COMBINE_WINDOW_START, 2 ** Math.min(this.taken, 30))
+    );
+    while (this.next_ < this.keys.length && this.inFlight.length < width) {
+      const key = this.keys[this.next_++];
+      this.inFlight.push(
+        this.read(key).then(
+          (result) => ({ key, result }),
+          (error) => ({ key, result: null, error })
+        )
+      );
+    }
+  }
+  /** The next key's chunk in ascending key order, `undefined` when the keys are used up; throws that key's error. */
+  async next() {
+    if (this.inFlight.length === 0) return void 0;
+    const slot = await this.inFlight.shift();
+    this.taken += 1;
+    this.fill();
+    if (slot.error !== void 0) throw slot.error;
+    return slot;
+  }
+};
+function chunkIds(chunk, chunkKey, w) {
+  let rem = chunk.toUint32Array ? chunk.toUint32Array() : Uint32Array.from(chunk);
+  if (w !== null && isEdge(chunkKey, w)) {
+    const from = chunkKey === w.loKey ? w.loRem : 0;
+    const to = chunkKey === w.hiKey ? w.hiRem : MAX_REMAINDER;
+    rem = rem.slice(firstAbove(rem, from - 1), firstAbove(rem, to));
+  }
+  const base = chunkKey * CHUNK_COUNT;
+  for (let i = 0; i < rem.length; i++) rem[i] = base + (rem[i] & MAX_REMAINDER);
+  return rem;
+}
+function firstAbove(a, value) {
+  let lo = 0;
+  let hi = a.length;
+  while (lo < hi) {
+    const mid = lo + hi >>> 1;
+    if (a[mid] <= value) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo;
+}
 var SegmentEngine = class {
   storage;
   cache;
@@ -507,6 +566,26 @@ var SegmentEngine = class {
       }
     }
   }
+  /**
+   * {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
+   * the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
+   * reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
+   * the caller's to keep.
+   */
+  async *iterateBatches(seg, range) {
+    const w = windowOf(range);
+    if (w === "empty") return;
+    const chunkKeys = w === null ? await this.chunkKeys(seg) : keysWithin(await this.chunkKeys(seg), w);
+    checkBudget(this.budget, chunkKeys.length, "iterate");
+    const gen = await this.cacheVersion(seg);
+    const window = this.chunkWindow(seg, chunkKeys, gen, true);
+    for (const chunkKey of chunkKeys) {
+      const chunk = await window.take();
+      if (chunk === null) continue;
+      const ids = chunkIds(chunk, chunkKey, w);
+      if (ids.length > 0) yield ids;
+    }
+  }
   /**
    * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
    *
@@ -561,7 +640,24 @@ var SegmentEngine = class {
   andNot(seg, excludes, options) {
     return this.combine([seg], excludes, "all", "andNot", options);
   }
-  async *combine(segs, excludeSegs, mode, op, options) {
+  /** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+  intersectBatches(segs, options) {
+    return this.combineBatches(segs, options?.exclude ?? [], "all", "intersect", options);
+  }
+  /** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+  unionBatches(segs, options) {
+    return this.combineBatches(segs, options?.exclude ?? [], "any", "union", options);
+  }
+  /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+  andNotBatches(seg, excludes, options) {
+    return this.combineBatches([seg], excludes, "all", "andNot", options);
+  }
+  /**
+   * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
+   * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
+   * read and the batch read share it, so the two cannot disagree about what is read or charged.
+   */
+  async combinePlan(segs, excludeSegs, mode, op, options) {
     if (segs.length === 0) {
       throw new ValidationError(`${op} requires at least one segment`);
     }
@@ -576,7 +672,7 @@ var SegmentEngine = class {
     }
     const budget = resolvePerOpBudget(options?.budget, this.budget);
     const w = windowOf(options);
-    if (w === "empty") return;
+    if (w === "empty") return null;
     const extract = async (seg) => {
       const all = await this.chunkKeys(seg);
       const gen = await this.cacheVersion(seg);
@@ -621,24 +717,20 @@ var SegmentEngine = class {
         skippedChunks: distinctKeys.size - common.length
       });
     }
-    const startAt = (key) => this.combineChunk(operands, excludes, mode, key).then(
-      (result) => ({ key, result }),
-      (error) => ({ key, result: null, error })
-    );
-    const inFlight = [];
-    let next = 0;
-    let taken = 0;
-    const fill = () => {
-      const width = Math.min(limit, Math.max(COMBINE_WINDOW_START, 2 ** Math.min(taken, 30)));
-      while (next < common.length && inFlight.length < width)
-        inFlight.push(startAt(common[next++]));
+    return {
+      w,
+      window: new CombineWindow(
+        common,
+        limit,
+        (key) => this.combineChunk(operands, excludes, mode, key)
+      )
     };
-    fill();
-    while (inFlight.length > 0) {
-      const slot = await inFlight.shift();
-      taken += 1;
-      fill();
-      if (slot.error !== void 0) throw slot.error;
+  }
+  async *combine(segs, excludeSegs, mode, op, options) {
+    const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
+    if (plan === null) return;
+    const { w, window } = plan;
+    for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
       if (slot.result && !slot.result.isEmpty) {
         if (w !== null && isEdge(slot.key, w)) {
           for (const id of edgeIds(slot.result, slot.key, w)) yield id;
@@ -648,6 +740,17 @@ var SegmentEngine = class {
       }
     }
   }
+  async *combineBatches(segs, excludeSegs, mode, op, options) {
+    const plan = await this.combinePlan(segs, excludeSegs, mode, op, options);
+    if (plan === null) return;
+    const { w, window } = plan;
+    for (let slot = await window.next(); slot !== void 0; slot = await window.next()) {
+      if (slot.result) {
+        const ids = chunkIds(slot.result, slot.key, w);
+        if (ids.length > 0) yield ids;
+      }
+    }
+  }
   /**
    * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
    * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).