@cloudbitmaps/core 0.25.0 → 0.26.0

package/dist/core/crbm-storage-source.d.ts

@@ -405,9 +405,15 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
      * ({@link replacedPins}), and a check under way is shared by the reads that ask at once; neither "gone" nor "the
      * same" is kept, since either can change. A transient fault is rethrown, for the store's retries to see for what it
      * is. A footer that cannot be read otherwise says nothing about which object is there, so this returns, and the
-     * caller's own error stands.
+     * caller's own error stands; a caller with no error of its own passes `strict`, and that fault is raised.
      */
     private throwIfReplaced;
+    /**
+     * Throws the pin's `NotFoundError` when the object under `generation`'s key is no longer the one `held` names: one
+     * footer read, shared with a check already under way for the same pin. It fails closed: a footer that cannot be read,
+     * or an object that is not a generation object of this store, raises its own fault rather than passing.
+     */
+    verifyHeld(ref: SegmentRef, generation: number, held: PinnedObject): Promise<void>;
     /** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
     listChunkKeysAt(ref: SegmentRef, generation: number, held?: PinnedObject): Promise<number[]>;
     /**

package/dist/core/engine.d.ts

@@ -101,6 +101,14 @@ export interface CombinedChunk {
     readonly chunkKey: number;
     readonly bitmap: CodecBitmap;
 }
+/** Options of {@link SegmentEngine.combineCount}: a combine's, and a check the count runs at its start and at each look. */
+export interface CountOptions extends CombineOptions {
+    /**
+     * Runs before the first key and at each look, and may throw to stop the count (a lease that has ended). A count
+     * answered from the index fetches no chunk, so this is where such a handle is held to its terms.
+     */
+    readonly guard?: () => void;
+}
 export declare class SegmentEngine {
     private readonly storage;
     private readonly cache;
@@ -397,6 +405,51 @@ export declare class SegmentEngine {
     private combineBatches;
     /** The chunk read: the plan every combine shares, then each key's result, cut to the range at its edges. */
     private combineChunks;
+    /**
+     * How many ids `segs[0] ∩ …` (`mode` 'all') or `segs[0] ∪ …` ('any') holds, minus `excludeSegs`, without streaming
+     * them: the size of what {@link combine} would stream. Each chunk key is taken one of two ways. A key exactly one
+     * include holds, that no exclude holds and the range does not cut, is answered from the index's count for it, with
+     * no chunk read. Every other key is decoded and counted as the combine reads it, with the same plan, the same ranges
+     * and the same window, over those keys alone. So an intersection reads what its stream reads, and a union or a
+     * difference reads no more than its stream.
+     *
+     * The keys answered from the index are trusted as {@link count} trusts the index, and checked no further than the
+     * reader checks it when it opens the object; a decoded key is exact. A source that reports no per-chunk counts has
+     * every key decoded. A union or a difference of one operand with no exclude and no range is {@link count}.
+     *
+     * A running union or difference, or an intersection with an `exclude`, looks at its operands' versions every
+     * `concurrency` keys, a key answered from the index being one taken, and any count looks before the first key after
+     * an invalidation, and runs `options.guard` at each look. On a move it plans again above the last key taken, as a
+     * combine does, so the bound on what it serves of an earlier generation is the combine's. A pinned operand's object
+     * is looked up before the first key and at each look there is (one footer read each, a fault other than a transient
+     * one raised as it is), so an erasure of what it pinned stops the count there with the pin's `NotFoundError`. An
+     * intersection of two or more operands with no exclude decodes every key and so shows a move without a version
+     * look; its guard still runs every `concurrency` keys.
+     */
+    combineCount(segs: readonly SegmentRef[], excludeSegs: readonly SegmentRef[], mode: 'all' | 'any', op: 'intersect' | 'union' | 'andNot', options?: Omit<CountOptions, 'exclude'>): Promise<number>;
+    /** Take every step of `run`: the index's figure for one, the size of the next decoded chunk for another. */
+    private runCount;
+    /** Run the guard every `limit` keys when the version looks are off (an AND of two or more includes). */
+    private guardAtCadence;
+    /**
+     * Look up each pinned operand's generation object, as a pinned read finds it: one footer read per pinned operand, and
+     * the pin's `NotFoundError` when the object is gone or is another. An operand that is not pinned costs nothing.
+     */
+    private verifyPinned;
+    /** Look at every operand's version, as {@link nextSlotAfterLook} does, and plan the keys above `last` again on a move. */
+    private countLook;
+    /**
+     * {@link combineOperands} for a count: each include also carries the per-chunk counts of the generation its keys were
+     * listed from. The counts are asked for after the keys, and the two must name the same keys, or the operand is listed
+     * again (a move between the two reads), and refused if a source keeps disagreeing with itself. A source with no
+     * per-chunk counts gives none, and every key of that operand is decoded.
+     */
+    private countOperands;
+    /**
+     * The per-chunk counts of `seg`, checked as untrusted tier data; `undefined` when the source reports none, and `'moved'`
+     * when they name other keys than `listed`, which the segment's generation changing between the two reads explains.
+     */
+    private chunkCounts;
     /**
      * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
      * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).

package/dist/core/pinned-storage-source.d.ts

@@ -112,5 +112,7 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
      */
     currentVersion(ref: SegmentRef): Promise<string | null>;
     exists(ref: SegmentRef): Promise<boolean>;
+    /** Whether the object this segment's pin holds is still there: see {@link HeldObjectLookup.verifyPinned}. */
+    verifyPinned(ref: SegmentRef): Promise<void>;
     invalidate(ref: SegmentRef): void;
 }

package/dist/core/ports.d.ts

@@ -147,7 +147,9 @@ export interface StorageChunkSource {
      * already-parsed `.crbm` index with **no payload reads**, or `null` if the segment has no Storage generation.
      * Powers the free `count()` of a source with no {@link StorageChunkSource.summary} — the engine sums the index
      * instead of fetching a single chunk. A source with no
-     * index (e.g. the in-memory source) omits this, and `count()` falls back to fetching every chunk.
+     * index (e.g. the in-memory source) omits this, and `count()` falls back to fetching every chunk. A count of a
+     * combine also asks for it, and takes its keys to be the ones `listChunkKeys` names for the same generation (an emptied
+     * chunk is named by both or by neither), listing again when they differ and refusing a source that keeps disagreeing.
      */
     cardinalities?(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
     /**
@@ -227,6 +229,19 @@ export interface StorageChunkSource {
      */
     currentVersion?(ref: SegmentRef): Promise<string | null>;
 }
+/**
+ * What a source that reads pinned generations also offers the engine, apart from {@link StorageChunkSource}: a lookup of
+ * the object a pin holds. Not part of the port a source implements; the engine asks for it when it is there.
+ */
+export interface HeldObjectLookup {
+    /**
+     * Resolves when the segment is not pinned here, or when the object its pin holds is still under its key; rejects with
+     * the pin's `NotFoundError` when it is gone or is another object, and with the fault itself when the footer cannot be
+     * read or is not a generation object's (a transient fault is for the caller's retries). One footer read for a pinned
+     * segment, none for any other.
+     */
+    verifyPinned(ref: SegmentRef): Promise<void>;
+}
 /** Capabilities a Storage driver advertises; validated at wiring time, fail-fast. */
 export interface StorageCaps {
     /** REQUIRED — the format relies on byte-range reads. */

package/dist/driver-kit.js

@@ -1,6 +1,6 @@
 import {
   mapWithConcurrency
-} from "./chunk-BUDAVGBI.js";
+} from "./chunk-7VNEBKC4.js";
 import {
   DEFAULT_NAMESPACE,
   applyRegistryPatch,
@@ -18,7 +18,7 @@ import {
   validateNewRegistryRecord,
   validateRegistryPatch,
   webCryptoEntropy
-} from "./chunk-45ML6YU4.js";
+} from "./chunk-LJLZRVYN.js";
 import {
   CloudRoaringError,
   IntegrityError,
@@ -45,7 +45,7 @@ import {
   requireNonEmptyString,
   unsettledConflict,
   validateSegmentRef
-} from "./chunk-2B2QF4FU.js";
+} from "./chunk-UFOMYONZ.js";
 
 // src/drivers/_shared/object-registry-keys.ts
 var REGISTRY_SUFFIX = ".reg";

package/dist/drivers/retry/retrying-chunk-source.d.ts

@@ -51,6 +51,7 @@ export declare class RetryingStorageChunkSource implements StorageChunkSource {
     readonly invalidate?: (ref: SegmentRef) => void;
     readonly exists?: (ref: SegmentRef) => Promise<boolean>;
     readonly currentVersion?: (ref: SegmentRef) => Promise<string | null>;
+    readonly verifyPinned?: (ref: SegmentRef) => Promise<void>;
     constructor(inner: StorageChunkSource, opts: RetryingOptions);
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;

package/dist/index.d.ts

@@ -19,7 +19,7 @@
  * publishable on its own and every codec reuses one driver set with zero duplication.
  */
 export { SegmentEngine } from './core/engine.js';
-export type { EngineDeps, CombineOptions as EngineCombineOptions, IdRange } from './core/engine.js';
+export type { EngineDeps, CombineOptions as EngineCombineOptions, CountOptions as EngineCountOptions, IdRange, } from './core/engine.js';
 export { BoundedLru } from './core/lru.js';
 export { safeMetrics } from './core/metrics.js';
 export { runExport } from './export/index.js';

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   boundedAll,
   mapWithConcurrency
-} from "./chunk-BUDAVGBI.js";
+} from "./chunk-7VNEBKC4.js";
 import {
   BufferReader,
   KeptSegmentKeys,
@@ -9,7 +9,7 @@ import {
   segmentKey,
   segmentPrefix,
   shardOf
-} from "./chunk-WPER7R2R.js";
+} from "./chunk-MNXTZJZ6.js";
 import {
   BudgetExceededError,
   CHUNK_COUNT,
@@ -116,7 +116,7 @@ import {
   validateUserRef,
   windowOf,
   withRetry
-} from "./chunk-2B2QF4FU.js";
+} from "./chunk-UFOMYONZ.js";
 
 // src/core/budget.ts
 var DEFAULT_BUDGET = { maxRequests: 1e6 };
@@ -674,6 +674,7 @@ var COMBINE_WINDOW_START = 8;
 var COMBINE_RANGE_START = 4;
 var OPERANDS_AT_ONCE = 16;
 var BATCH_QUIET_HOPS = 16;
+var COUNT_RELIST_ATTEMPTS = 4;
 var ZERO_CLOCK = { now: () => 0 };
 var CombineWindow = class {
   constructor(keys, limit, read) {
@@ -797,6 +798,19 @@ function combineReads(keys, operands, excludes, mode, through) {
   }
   return units;
 }
+function countSteps(operands, excludes, mode, w) {
+  const steps = [];
+  const decode = [];
+  for (const key of alignKeys(operands, mode)) {
+    const holders = mode === "all" ? operands : operands.filter((o) => o.keys.has(key));
+    const only = holders.length === 1 ? holders[0] : void 0;
+    const n = only?.cards?.get(key);
+    const viaIndex = n !== void 0 && !excludes.some((e) => e.keys.has(key)) && !(w !== null && isEdge(key, w));
+    steps.push({ key, n: viaIndex ? n : -1 });
+    if (!viaIndex) decode.push(key);
+  }
+  return { steps, decode };
+}
 var SegmentEngine = class _SegmentEngine {
   storage;
   cache;
@@ -1514,6 +1528,209 @@ var SegmentEngine = class _SegmentEngine {
       this.closeCombine(run.plan);
     }
   }
+  /**
+   * How many ids `segs[0] ∩ …` (`mode` 'all') or `segs[0] ∪ …` ('any') holds, minus `excludeSegs`, without streaming
+   * them: the size of what {@link combine} would stream. Each chunk key is taken one of two ways. A key exactly one
+   * include holds, that no exclude holds and the range does not cut, is answered from the index's count for it, with
+   * no chunk read. Every other key is decoded and counted as the combine reads it, with the same plan, the same ranges
+   * and the same window, over those keys alone. So an intersection reads what its stream reads, and a union or a
+   * difference reads no more than its stream.
+   *
+   * The keys answered from the index are trusted as {@link count} trusts the index, and checked no further than the
+   * reader checks it when it opens the object; a decoded key is exact. A source that reports no per-chunk counts has
+   * every key decoded. A union or a difference of one operand with no exclude and no range is {@link count}.
+   *
+   * A running union or difference, or an intersection with an `exclude`, looks at its operands' versions every
+   * `concurrency` keys, a key answered from the index being one taken, and any count looks before the first key after
+   * an invalidation, and runs `options.guard` at each look. On a move it plans again above the last key taken, as a
+   * combine does, so the bound on what it serves of an earlier generation is the combine's. A pinned operand's object
+   * is looked up before the first key and at each look there is (one footer read each, a fault other than a transient
+   * one raised as it is), so an erasure of what it pinned stops the count there with the pin's `NotFoundError`. An
+   * intersection of two or more operands with no exclude decodes every key and so shows a move without a version
+   * look; its guard still runs every `concurrency` keys.
+   */
+  async combineCount(segs, excludeSegs, mode, op, options) {
+    const label = `${op}Count`;
+    if (segs.length === 0) throw new ValidationError(`${label} requires at least one segment`);
+    const limit = options?.concurrency === void 0 ? DEFAULT_INTERSECT_CONCURRENCY : options.concurrency;
+    requirePositiveInteger(limit, `${label}: concurrency`);
+    const budget = resolvePerOpBudget(options?.budget, this.budget);
+    const w = windowOf2(options, label);
+    options?.guard?.();
+    if (w === "empty") return 0;
+    if (op !== "intersect" && segs.length === 1 && excludeSegs.length === 0 && w === null) {
+      const n = await this.count(segs[0]);
+      if (n === 0) {
+        const none = {
+          seg: segs[0],
+          keys: /* @__PURE__ */ new Set(),
+          chunkless: true,
+          gen: void 0,
+          epoch: 0
+        };
+        await this.refuseAbsentOperands([none], options?.allowAbsentOperands, label);
+      }
+      return n;
+    }
+    const epoch = this.scope.epoch;
+    const [operands, excludes] = await this.countOperands(segs, excludeSegs, mode, w, -1);
+    await this.refuseAbsentOperands(
+      [...operands, ...excludes],
+      options?.allowAbsentOperands,
+      label
+    );
+    await this.verifyPinned([...operands, ...excludes]);
+    const { steps, decode } = countSteps(operands, excludes, mode, w);
+    checkBudget(budget, combineReads(decode, operands, excludes, mode, Infinity), label);
+    if (this.metricsOn) {
+      const distinct = /* @__PURE__ */ new Set();
+      for (const o of [...operands, ...excludes]) for (const k of o.keys) distinct.add(k);
+      this.metrics.onEvent({
+        kind: "intersect",
+        op,
+        operands: segs.length + excludes.length,
+        fetchedChunks: decode.length,
+        skippedChunks: distinct.size - decode.length
+      });
+    }
+    const everyKey = mode === "all" && segs.length > 1 && excludeSegs.length === 0;
+    const run = {
+      segs,
+      excludeSegs,
+      mode,
+      op,
+      limit,
+      budget,
+      w,
+      plan: this.openCombine(operands, excludes, mode, decode, limit),
+      looks: this.looks(everyKey ? Infinity : limit, epoch),
+      spent: 0,
+      steps,
+      at: 0,
+      guard: options?.guard,
+      ...everyKey && options?.guard !== void 0 ? { guardEvery: { since: 0 } } : {}
+    };
+    try {
+      return await this.runCount(run);
+    } finally {
+      this.closeCombine(run.plan);
+    }
+  }
+  /** Take every step of `run`: the index's figure for one, the size of the next decoded chunk for another. */
+  async runCount(run) {
+    let total = 0;
+    let last = -1;
+    while (run.at < run.steps.length) {
+      run.looks.left -= 1;
+      if (run.guardEvery !== void 0) this.guardAtCadence(run);
+      if (run.plan.moves.moved || this.lookDue(run.looks)) {
+        await this.countLook(run, last);
+        if (run.at >= run.steps.length) break;
+      }
+      const step = run.steps[run.at++];
+      last = step.key;
+      if (step.n >= 0) {
+        total += step.n;
+        continue;
+      }
+      const slot = await run.plan.window.next();
+      const chunk = slot?.result;
+      if (chunk === null || chunk === void 0 || chunk.isEmpty) continue;
+      total += run.w !== null && isEdge(step.key, run.w) ? cutToWindow(this.codec, chunk, step.key, run.w).size : chunk.size;
+    }
+    return total;
+  }
+  /** Run the guard every `limit` keys when the version looks are off (an AND of two or more includes). */
+  guardAtCadence(run) {
+    const cadence = run.guardEvery;
+    if (cadence.since >= run.limit) {
+      cadence.since = 0;
+      run.guard();
+    }
+    cadence.since += 1;
+  }
+  /**
+   * Look up each pinned operand's generation object, as a pinned read finds it: one footer read per pinned operand, and
+   * the pin's `NotFoundError` when the object is gone or is another. An operand that is not pinned costs nothing.
+   */
+  async verifyPinned(operands) {
+    const verify = this.storage.verifyPinned;
+    if (verify === void 0) return;
+    await boundedAll(operands, OPERANDS_AT_ONCE, (o) => verify.call(this.storage, o.seg));
+  }
+  /** Look at every operand's version, as {@link nextSlotAfterLook} does, and plan the keys above `last` again on a move. */
+  async countLook(run, last) {
+    const plan = run.plan;
+    plan.moves.moved = false;
+    this.looked(run.looks);
+    run.guard?.();
+    const all = [...plan.operands, ...plan.excludes];
+    await this.verifyPinned(all);
+    const now = await boundedAll(all, OPERANDS_AT_ONCE, (o) => this.cacheVersion(o.seg));
+    if (now.every((version, i) => version === all[i].gen)) return;
+    this.closeCombine(plan);
+    const spent = run.spent + combineReads(plan.keys, plan.operands, plan.excludes, run.mode, last);
+    const [operands, excludes] = await this.countOperands(
+      run.segs,
+      run.excludeSegs,
+      run.mode,
+      run.w,
+      last
+    );
+    const { steps, decode } = countSteps(operands, excludes, run.mode, run.w);
+    checkBudget(
+      run.budget,
+      spent + combineReads(decode, operands, excludes, run.mode, Infinity),
+      `${run.op}Count`
+    );
+    run.plan = this.openCombine(operands, excludes, run.mode, decode, run.limit);
+    run.spent = spent;
+    run.steps = steps;
+    run.at = 0;
+  }
+  /**
+   * {@link combineOperands} for a count: each include also carries the per-chunk counts of the generation its keys were
+   * listed from. The counts are asked for after the keys, and the two must name the same keys, or the operand is listed
+   * again (a move between the two reads), and refused if a source keeps disagreeing with itself. A source with no
+   * per-chunk counts gives none, and every key of that operand is decoded.
+   */
+  async countOperands(segs, excludeSegs, mode, w, after) {
+    const wantCounts = !(mode === "all" && segs.length > 1);
+    const extract = async (seg, withCounts) => {
+      for (let attempt = 0; attempt < COUNT_RELIST_ATTEMPTS; attempt++) {
+        const { listed: listed2, gen, epoch } = await this.listKeys(seg);
+        const cards = withCounts ? await this.chunkCounts(seg, listed2) : void 0;
+        if (cards === "moved") continue;
+        const keys = after < 0 && w === null ? listed2 : keysAbove(listed2, w, after);
+        return { seg, keys: new Set(keys), chunkless: listed2.length === 0, gen, epoch, cards };
+      }
+      throw new IntegrityError("a segment's index and its chunk keys name different chunks");
+    };
+    const all = await boundedAll(
+      [
+        ...segs.map((s) => [s, wantCounts]),
+        ...excludeSegs.map((s) => [s, false])
+      ],
+      OPERANDS_AT_ONCE,
+      ([seg, withCounts]) => extract(seg, withCounts)
+    );
+    return [all.slice(0, segs.length), all.slice(segs.length)];
+  }
+  /**
+   * The per-chunk counts of `seg`, checked as untrusted tier data; `undefined` when the source reports none, and `'moved'`
+   * when they name other keys than `listed`, which the segment's generation changing between the two reads explains.
+   */
+  async chunkCounts(seg, listed2) {
+    const counts = this.storage.cardinalities ? await this.storage.cardinalities(seg) : null;
+    if (counts === null) return void 0;
+    if (counts.size !== listed2.length) return "moved";
+    for (const key of listed2) {
+      const n = counts.get(key);
+      if (n === void 0) return "moved";
+      if (n !== 0) assertChunkCardinalityInRange(n);
+    }
+    return counts;
+  }
   /**
    * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
    * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
@@ -4461,9 +4678,9 @@ var CrbmStorageChunkSource = class {
    * ({@link replacedPins}), and a check under way is shared by the reads that ask at once; neither "gone" nor "the
    * same" is kept, since either can change. A transient fault is rethrown, for the store's retries to see for what it
    * is. A footer that cannot be read otherwise says nothing about which object is there, so this returns, and the
-   * caller's own error stands.
+   * caller's own error stands; a caller with no error of its own passes `strict`, and that fault is raised.
    */
-  async throwIfReplaced(ref, generation, version, fingerprint) {
+  async throwIfReplaced(ref, generation, version, fingerprint, strict = false) {
     const held = this.heldKey(ref, version, fingerprint);
     if (this.replacedPins.get(held) !== void 0) throw notThePinned(ref, generation);
     let check = this.checking.get(held);
@@ -4491,11 +4708,20 @@ var CrbmStorageChunkSource = class {
     try {
       replaced = await check;
     } catch (err) {
-      if (isTransientError(err)) throw err;
+      if (strict || isTransientError(err)) throw err;
       return;
     }
     if (replaced) throw notThePinned(ref, generation);
   }
+  /**
+   * Throws the pin's `NotFoundError` when the object under `generation`'s key is no longer the one `held` names: one
+   * footer read, shared with a check already under way for the same pin. It fails closed: a footer that cannot be read,
+   * or an object that is not a generation object of this store, raises its own fault rather than passing.
+   */
+  async verifyHeld(ref, generation, held) {
+    validateUserRef(ref, "read", PORT_KEYS);
+    await this.throwIfReplaced(ref, generation, held.version, held.fingerprint, true);
+  }
   /** Chunk keys of a specific generation — the pinned shape read. `held` as for {@link getChunkAt}. */
   async listChunkKeysAt(ref, generation, held) {
     validateUserRef(ref, "read", PORT_KEYS);
@@ -5171,6 +5397,14 @@ var PinnedStorageChunkSource = class {
   exists(ref) {
     return this.inner.exists(ref);
   }
+  /** Whether the object this segment's pin holds is still there: see {@link HeldObjectLookup.verifyPinned}. */
+  verifyPinned(ref) {
+    const pin = this.pinFor(ref);
+    const held = pin === void 0 ? void 0 : heldBy(pin);
+    if (pin === void 0 || held === void 0 || pin.generation === null)
+      return Promise.resolve();
+    return this.inner.verifyHeld(ref, pin.generation, held);
+  }
   invalidate(ref) {
     this.inner.invalidate(ref);
   }
@@ -12023,6 +12257,7 @@ var RetryingStorageChunkSource = class {
   invalidate;
   exists;
   currentVersion;
+  verifyPinned;
   constructor(inner, opts) {
     this.inner = inner;
     const r = toRetry(opts);
@@ -12068,6 +12303,10 @@ var RetryingStorageChunkSource = class {
     if (innerExists) {
       this.exists = (ref) => withRetry(() => innerExists.call(inner, ref), this.policy, this.deps);
     }
+    const innerVerify = inner.verifyPinned;
+    if (innerVerify) {
+      this.verifyPinned = (ref) => withRetry(() => innerVerify.call(inner, ref), this.policy, this.deps);
+    }
     const innerCurrentVersion = inner.currentVersion;
     if (innerCurrentVersion) {
       this.currentVersion = (ref) => withRetry(() => innerCurrentVersion.call(inner, ref), this.policy, this.deps);