@cloudbitmaps/core 0.17.0 → 0.18.0

package/dist/core/combine-expr.d.ts

@@ -0,0 +1,75 @@
+/** A string names an operand; a node holds exactly one operator over a non-empty list of expressions. */
+export type CombineExpr = string | {
+    and: CombineExpr[];
+} | {
+    or: CombineExpr[];
+} | {
+    andNot: CombineExpr[];
+};
+/** The most operator nodes one expression may nest. A literal that contains itself is refused by it. */
+export declare const MAX_EXPR_DEPTH = 64;
+/** The most nodes (operators and names) one expression may hold, so a shared sub-object cannot expand without bound. */
+export declare const MAX_EXPR_NODES = 4096;
+declare const OPERATORS: readonly ["and", "or", "andNot"];
+type Operator = (typeof OPERATORS)[number];
+/** A checked expression: operand indexes at the leaves, operators inside. */
+export type ExprNode = {
+    readonly kind: 'leaf';
+    readonly operand: number;
+} | {
+    readonly kind: Operator;
+    readonly kids: readonly ExprNode[];
+};
+/** How many nodes one {@link compileExpr} call may still allocate; shared across the expressions of one output. */
+export interface NodeBudget {
+    remaining: number;
+}
+/**
+ * Check `expr` and lower it. `path` names where it sits (`outputs[2].expr`), and every failure is a
+ * {@link ValidationError} that carries it plus the path inside the expression.
+ */
+export declare function compileExpr(expr: unknown, path: string, operands: ReadonlyMap<string, number>, budget?: NodeBudget, depth?: number): ExprNode;
+/** The deepest operator nesting in `node`: 0 for a name. */
+export declare function depthOf(node: ExprNode): number;
+/** The operand indexes `node` names, each once, ascending. */
+export declare function operandsOf(node: ExprNode, into?: Set<number>): number[];
+/** Ascending chunk keys, and an upper bound on the ids each holds. */
+export interface KeyBounds {
+    readonly keys: Uint16Array;
+    readonly card: Uint32Array;
+}
+/** The keys of an operand the plan cannot know (a fed one): every key. Compared by identity, never by content. */
+export declare const ANY_KEYS: Uint16Array;
+/** The bounds of an operand the plan cannot know. */
+export declare const ANY_BOUNDS: KeyBounds;
+/** What {@link planKeys} keeps of each operator node: the keys it can be non-empty at. */
+export type NodeKeys = Map<ExprNode, Uint16Array>;
+/**
+ * Bottom up: the keys at which each node of `node` can be non-empty, with an upper bound on its ids at each. The
+ * keys of every operator node are kept in `into` for {@link collectFetch}; the bounds are returned for the root alone,
+ * so memory is the keys of one expression, not their bounds.
+ */
+export declare function planKeys(node: ExprNode, operands: readonly KeyBounds[], into?: NodeKeys): KeyBounds;
+/**
+ * Top down: the keys of each operand an output needs, given the keys `demanded` of its root. A leaf is fetched where
+ * it is demanded and holds a chunk; `and` and `or` pass down what they can be non-empty at; `andNot` passes its left
+ * child the demand it can meet and each subtrahend that, so an exclude is read only where the left side can overlap it.
+ * `fetch(operand, keys)` is called once per operand occurrence.
+ */
+export declare function collectFetch(node: ExprNode, demanded: Uint16Array, operands: readonly KeyBounds[], planned: NodeKeys, fetch: (operand: number, keys: Uint16Array) => void): void;
+/**
+ * Whether `node` can be non-empty at one key, given which operands hold it: an `and` needs every child, an `or` any,
+ * and an `andNot` its left child. Exact for the key, where {@link planKeys} is a bound over all of them.
+ */
+export declare function canHold(node: ExprNode, holds: (operand: number) => boolean): boolean;
+/** A set of chunk keys as bits, for the keys an operand is fetched at across the outputs of a group. */
+export declare class KeyBits {
+    private readonly words;
+    private count;
+    add(keys: Uint16Array): void;
+    get size(): number;
+    has(key: number): boolean;
+    /** The keys, ascending. */
+    toKeys(): number[];
+}
+export {};

package/dist/core/combine-feed.d.ts

@@ -0,0 +1,142 @@
+/**
+ * The chunk-ordered feed of a batch combine: operands that arrive as records, one chunk key at a time, and are checked
+ * and converted as they arrive. Pure: it reads through the iterable it is given and knows no store, clock or storage.
+ *
+ * ```
+ * record { key, operands: { name: Uint32Array } }
+ *   check    key in range and not below the last; every name declared; every value a real Uint32Array; ids strictly
+ *            ascending and inside the key; each operand at most once per key        (a refusal, never a repair)
+ *   convert  each operand's ids into one compressed bitmap, charged to the ledger as it is built
+ *   group    the records of one key, held until a record with a higher key arrives or the feed ends
+ * end of feed   counts equal what was seen; a declared operand that never appeared is refused unless allowed
+ * ```
+ *
+ * Nothing here ever reads a bad feed as fewer members: every check is a refusal, and a refusal ends the feed.
+ */
+import type { CodecBitmap, CodecInterface } from './codec.js';
+/** One record of a feed: the ids each named operand holds at one chunk key. */
+export interface CombineManyFeedRecord {
+    /** The chunk key, `id >>> 16`, from 0 to 65,535. Non-decreasing over the feed; a key may arrive as several records. */
+    readonly key: number;
+    /** Ids by operand name: ascending, unique, every one inside `key`. An operand is named at most once per key. */
+    readonly operands: Readonly<Record<string, Uint32Array>>;
+}
+/** The fed operands of a batch combine. */
+export interface CombineManyFeed {
+    /** The fed operand names, declared up front. */
+    readonly names: readonly string[];
+    /** The records, read once, in key order. */
+    readonly records: AsyncIterable<CombineManyFeedRecord>;
+    /**
+     * The ids each fed operand holds over the whole feed, keyed by exactly `names`: an object, or a function called once
+     * after the last record. A count that differs from what the feed held refuses every fed output.
+     */
+    readonly counts: Readonly<Record<string, number>> | (() => Readonly<Record<string, number>> | Promise<Readonly<Record<string, number>>>);
+    /** Fed names allowed to hold no id anywhere in the feed. */
+    readonly mayBeEmpty?: readonly string[];
+    /** Whether an erasure ran in the store since the call began, or is running: read at every record and before each fed publish. */
+    readonly epoch?: {
+        readonly moved: () => boolean;
+    };
+}
+/** A feed that passed {@link checkFeed}: read once, so what was checked is what runs. */
+export interface CheckedFeed {
+    readonly names: readonly string[];
+    readonly records: AsyncIterable<unknown>;
+    readonly counts: CombineManyFeed['counts'];
+    readonly mayBeEmpty: ReadonlySet<string>;
+    readonly epoch: CombineManyFeed['epoch'];
+}
+/** Check a feed's shape, before any record is read. `stored` are the stored operand names, which a fed one may not repeat. */
+export declare function checkFeed(feed: unknown, stored: ReadonlySet<string>): CheckedFeed;
+/**
+ * The view a real `Uint32Array` holds, as a plain `Uint32Array` over the same bytes, or `undefined` for anything else.
+ * The brand is the `%TypedArray%` `@@toStringTag` getter, which reads the internal slot: it names only a real typed array
+ * of that kind (a subclass or another realm's included) and throws for a spoof, a proxy or any other object. The view's
+ * extent is then read through the intrinsic getters, so a subclass that overrides `length` or the iterator is never asked.
+ */
+export declare function plainUint32(value: unknown): Uint32Array | 'detached' | 'out of bounds' | undefined;
+/** Thrown inside the cursor when it was closed under an operation in flight; never reaches a caller. */
+export declare class FeedClosed extends Error {
+}
+/** Thrown when the store's erasure counter moved: the run turns it into one typed error per output. */
+export declare class FeedStale extends Error {
+}
+/** What a {@link FeedCursor} needs from the pass that runs it. */
+export interface FeedCursorDeps {
+    readonly feed: CheckedFeed;
+    readonly codec: CodecInterface;
+    /** The index the first fed operand has among the call's operands. */
+    readonly base: number;
+    /** Charge `bytes` to the ledger, making room as the pass does, or throw when they do not fit. */
+    readonly charge: (bytes: number) => void;
+    readonly release: (bytes: number) => void;
+    /** One call per operand converted: the pass's cooperative yield. */
+    readonly tick: () => Promise<void> | null;
+    /** Whether a record at `key` is inside the call's id range; one outside is checked and counted but not kept. */
+    readonly inWindow: (key: number) => boolean;
+}
+/**
+ * Reads a feed one record at a time: checks it, converts it, and hands the records of one key to the pass as a group.
+ * At most two records are resident, the group being built and the one that proved it complete.
+ */
+export declare class FeedCursor {
+    private readonly deps;
+    private iterator;
+    private group;
+    private look;
+    private current;
+    private ended;
+    private closed;
+    private finished;
+    private held;
+    private returned;
+    private prevKey;
+    private readonly index;
+    private readonly seen;
+    private readonly touched;
+    private readonly counts;
+    private readonly scratch;
+    private recordCount;
+    private keyCount;
+    private idCount;
+    constructor(deps: FeedCursorDeps);
+    /** Whether the feed can still yield a key: not closed, and neither ended nor drained. */
+    get active(): boolean;
+    /** Whether the whole feed was read and every end-of-feed check passed. */
+    get complete(): boolean;
+    /** The ledger bytes the cursor holds. */
+    get heldBytes(): number;
+    get stats(): {
+        readonly records: number;
+        readonly keys: number;
+        readonly ids: number;
+    };
+    /** The key the feed holds next, pulling the first record of it if none is held; `undefined` once nothing is left. */
+    nextKey(): Promise<number | undefined>;
+    /** Read until a record with a higher key arrives, or the feed ends: the group of {@link nextKey} is then whole. */
+    completeKey(): Promise<void>;
+    /** Take the whole group of {@link nextKey}: its bitmaps by operand index, held until {@link release}. */
+    take(): ReadonlyMap<number, CodecBitmap>;
+    /** Give back what the taken group holds. */
+    release(): void;
+    /** Stop reading: release everything held and return the iterator. Safe to call at any time, and again. */
+    close(): void;
+    /** Resolves once the iterator's `return` has settled. */
+    drained(): Promise<void>;
+    private fill;
+    /** The next record, checked and converted (`ops` is `null` when its key is outside the range); `undefined` at the end. */
+    private pull;
+    private end;
+    private refuse;
+    private charge;
+    private giveBack;
+    private convert;
+    /**
+     * Check `view` and copy its low 16 bits into the scratch array in one pass, each id read once: a buffer that changes
+     * under the check (a shared one) cannot make the checked value differ from the converted one.
+     */
+    private scan;
+    /** The counts and the empty rule, run once, after the iterator has ended and before anything is published. */
+    private finish;
+}

package/dist/core/combine-held.d.ts

@@ -0,0 +1,43 @@
+/**
+ * An operand held in memory, for a batch combine: ids checked and encoded as a load checks and encodes them, then kept
+ * as the per-chunk payload bytes a stored generation would hold. Pure: it knows no store and no storage.
+ *
+ * ```
+ * input   ids | { bitmap } | { serialized }
+ *   check    what a load checks: the size cap, the structure, the safe deserializer and each id's range; only a real
+ *            Uint32Array skips the per-id range check, by a brand a spoof cannot carry
+ *   encode   the load's own encoder, so a chunk is the bytes a load would store
+ * held    keys, cardinalities and one payload per chunk, read by the pass as a stored operand's chunks are
+ * ```
+ */
+import type { CodecInterface, EncodedChunk } from './codec.js';
+import type { Clock } from './determinism.js';
+/** A held operand's chunks: ascending keys, each with its exact cardinality and its payload bytes. */
+export declare class HeldChunks {
+    /** The chunk keys, ascending. */
+    readonly keys: Uint16Array;
+    /** How many ids each chunk holds. */
+    readonly cardinalities: Uint32Array;
+    /** What it holds in memory, for a call's resident-bytes ledger. */
+    readonly residentBytes: number;
+    private payloads;
+    constructor(chunks: Iterable<EncodedChunk>);
+    /** Whether it holds no id. */
+    get empty(): boolean;
+    /** Whether {@link release} ran. */
+    get released(): boolean;
+    /**
+     * The payload of the chunk at `key`, `null` when it holds none there. A view into what is held: not to be written to.
+     * @throws {ValidationError} once released.
+     */
+    payload(key: number): Uint8Array | null;
+    /** Zero and drop the bytes. Every later read throws; a second call does nothing. */
+    release(): void;
+}
+/**
+ * Check a held operand's input as a load checks it, and hold its chunks.
+ *
+ * @throws {ValidationError} where a load of the same input would, and for an object with an `ascending` key, which
+ * belongs to a feed.
+ */
+export declare function prepareHeld(input: unknown, codec: CodecInterface, clock: Clock | undefined): Promise<HeldChunks>;

package/dist/core/combine-ledger.d.ts

@@ -0,0 +1,43 @@
+/**
+ * The resident-bytes ledger of a batch combine. Pure: no I/O, time or randomness.
+ *
+ * It counts what is **resident**, not what a bitmap serializes to: a native bitmap costs a fixed amount over its
+ * serialized bytes, so a million small chunks hold far more than their serialized size says. Measured on the shipped
+ * codec with one bitmap per chunk, the cost over serialized size was about 440 bytes for a chunk of up to 50 ids,
+ * rising to about 1,700 bytes around 3,000 ids (an array container's spare capacity) and falling to about 300 bytes for
+ * a bitset. {@link residentBytes} is a model that stays above every measured point.
+ */
+/**
+ * The fixed resident cost of one native bitmap beyond its serialized bytes. Measured on the shipped codec, over 50,000
+ * bitmaps of one chunk each, it was 430 to 450 bytes for a chunk of up to 50 ids on macOS (arm64) and 357 to 373 on Linux
+ * (arm64, glibc 2.36), and 279 to 433 for a bitset; the constant sits above the highest of those.
+ */
+export declare const RESIDENT_BITMAP_OVERHEAD = 512;
+/** An upper bound on the serialized size of a chunk of `cardinality` ids: an array container, capped at a bitset. */
+export declare function serializedBound(cardinality: number): number;
+/** The resident bytes of one bitmap whose serialized size is `serialized`: above every measured point. */
+export declare function residentBytes(serialized: number): number;
+/** The resident bytes of one chunk of at most `cardinality` ids. */
+export declare function residentBound(cardinality: number): number;
+/**
+ * A byte budget with a high-water mark. Everything the pass keeps resident is charged to it and released when it is
+ * let go, so `used` is the pass's own count of what it holds, and `maxBufferedBytes` is a bound on it.
+ */
+export declare class ResidentLedger {
+    readonly limit: number;
+    private held;
+    private peak;
+    constructor(limit: number);
+    get used(): number;
+    get highWater(): number;
+    /** Bytes still free. */
+    get room(): number;
+    /** Charge `bytes` if they fit. */
+    tryCharge(bytes: number): boolean;
+    /**
+     * Charge `bytes`, asking `makeRoom` to free some while they do not fit. `makeRoom` releases what it frees (through
+     * {@link release}) and answers whether it freed anything; the charge is refused when it cannot.
+     */
+    reserve(bytes: number, makeRoom: () => boolean): boolean;
+    release(bytes: number): void;
+}

package/dist/core/combine-many.d.ts

@@ -0,0 +1,319 @@
+import type { CodecBitmap, CodecInterface } from './codec.js';
+import type { HeldChunks } from './combine-held.js';
+import type { CombineManyFeed } from './combine-feed.js';
+import type { CombineExpr } from './combine-expr.js';
+import type { Clock } from './determinism.js';
+import type { Budget } from './budget.js';
+import type { LoadGuard } from './load.js';
+import type { GenerationMetadata, SegmentRef, StorageChunkSource } from './ports.js';
+export type { CombineExpr } from './combine-expr.js';
+export { MAX_EXPR_DEPTH, MAX_EXPR_NODES } from './combine-expr.js';
+export type { CombineManyFeed, CombineManyFeedRecord } from './combine-feed.js';
+/** The range requests a pass keeps in flight at once across all its operands, inside a client's socket pool. */
+export declare const MAX_RANGES_IN_FLIGHT = 64;
+/** The largest `publishConcurrency` and `concurrency` a call may ask for. */
+export declare const MAX_BATCH_PARALLELISM = 1024;
+/** One named stored operand, as the pass reads it. */
+export interface CombineManyOperand {
+    readonly name: string;
+    readonly ref: SegmentRef;
+    /**
+     * Throws the typed error when this operand's handle can no longer be read: a lease that ended or was released, or a
+     * deadline that passed. Called before each chunk key the operand is read at, so a lapse fails the outputs that read
+     * it and never reads empty.
+     */
+    readonly check?: () => void;
+    /** The generation the operand is pinned to: `null` when pinned to a segment with none, `undefined` when not pinned. */
+    readonly pinnedGeneration?: number | null;
+    /**
+     * The pin's version: the generation and the registry row's token it was taken under (`<generation>:<token>`), or just
+     * the generation where there is no row. It is what tells a name deleted and created again, which restarts at
+     * generation 0, from the one that was pinned.
+     */
+    readonly pinnedVersion?: string | null;
+    /** The fingerprint of the object the pin holds: the identity of the generation's bytes, where a number can be taken again. */
+    readonly pinnedFingerprint?: string | null;
+    /**
+     * A fresh read of the segment's registry row: its current generation and token, or `null` when it has no row. Read
+     * for every subtracted pinned operand just before the publishes, and once more at the end of the call for the rest.
+     */
+    readonly current?: (options?: {
+        readonly fingerprint?: boolean;
+    }) => Promise<{
+        readonly generation: number | null;
+        readonly token: string;
+        readonly fingerprint?: string | null;
+    } | null>;
+    /**
+     * Set for an operand held in memory: its chunks are read from here, never from the source, and `ref`, `pinned*` and
+     * `current` are not used. It has no registry row, so nothing about it can move; what can invalidate it is an erasure in
+     * the store that made it ({@link CombineManyHeld.moved}) or its release.
+     */
+    readonly held?: CombineManyHeld;
+}
+/** An operand held in memory, as the pass reads it. */
+export interface CombineManyHeld {
+    /** The operand's chunks. Once released, reading one throws. */
+    readonly chunks: HeldChunks;
+    /**
+     * Whether an erasure has run in the store since the operand began to be made, one still running included, so what it holds may include an erased id. Checked
+     * before the call's first request, before each chunk key is read and immediately before each publish of an output that
+     * reads it.
+     */
+    readonly moved: () => boolean;
+    /** Whether holding no id is accepted: an include then empties an AND and an exclude subtracts nothing. */
+    readonly mayBeEmpty: boolean;
+}
+/** What one output asks for. */
+export interface CombineManyOutput<R> {
+    readonly expr: CombineExpr;
+    /** Subtracted from the result: a root `andNot`. */
+    readonly exclude?: readonly CombineExpr[];
+    readonly allowEmpty?: boolean;
+    readonly guard?: LoadGuard;
+    readonly metadata?: GenerationMetadata;
+    /** Overrides the call's `keep`. */
+    readonly keep?: number;
+    /** Throws to refuse the publish: the destination's lease or deadline, checked again just before it. */
+    readonly beforePublish?: () => void;
+    /**
+     * The requests a settled publish made that its value proves, for the call's totals by class; absent, a publish adds
+     * none. A publish that threw is not counted: what it sent is unknown.
+     */
+    readonly requestsOf?: (value: R) => {
+        readonly get: number;
+        readonly put: number;
+    };
+    /** Publish the finished chunks, ascending by key. What it returns or throws is the output's outcome. */
+    readonly publish: (chunks: AsyncIterable<{
+        readonly chunkKey: number;
+        readonly bitmap: CodecBitmap;
+    }>, write: CombineManyWrite) => Promise<R>;
+}
+/** The write options of one output, checked and defaulted: what its publish passes to the load. */
+export interface CombineManyWrite {
+    readonly keep: number;
+    readonly allowEmpty?: boolean;
+    readonly guard?: LoadGuard;
+    readonly metadata?: GenerationMetadata;
+}
+export interface CombineManyRequest<R> {
+    readonly operands: readonly CombineManyOperand[];
+    readonly outputs: readonly CombineManyOutput<R>[];
+    /** The call's `keep`; required. */
+    readonly keep: number;
+    readonly after?: number;
+    readonly through?: number;
+    /** Required with a feed: the feed's records and the fed outputs are counted against it as they arrive. */
+    readonly maxBufferedBytes: number;
+    readonly publishConcurrency: number;
+    readonly concurrency: number;
+    /** `undefined`: the plan's chunk reads with headroom. `null`: none. */
+    readonly budget?: Budget | null;
+    readonly allowAbsentOperands?: boolean;
+    /**
+     * Operands fed as records in chunk-key order, beside the stored ones. A feed is read once, so a call with one runs its
+     * outputs as one group; a fed output publishes only after the whole feed was read and checked.
+     */
+    readonly feed?: CombineManyFeed;
+}
+export interface CombineManyDeps {
+    /** Reads the operands: already pinned where they are, and retrying as the store's reads do. */
+    readonly source: StorageChunkSource;
+    readonly codec: CodecInterface;
+    readonly clock: Clock;
+    readonly maxBitmapBytes?: number;
+}
+/** What one output came to. */
+export type CombineManyOutcome<R> = {
+    readonly ok: true;
+    readonly value: R;
+} | {
+    readonly ok: false;
+    readonly error: unknown;
+};
+/** What the call did to one operand. */
+export interface CombineManyOperandStats {
+    /** Whether an output of the call read it. An operand no output names is not read. */
+    readonly read: boolean;
+    /** Whether the operand was read at a pinned generation. */
+    readonly pinned: boolean;
+    /** The generation pinned, `null` when the pin holds none, `undefined` when not pinned. */
+    readonly pinnedGeneration: number | null | undefined;
+    /** The generation the call began reading it at: the pin's, or the one live when its index was read. */
+    readonly startGeneration: number | null | undefined;
+    /**
+     * The generation current when the call ended, from one registry read of the operand's row: the re-read made before
+     * the publishes for an operand an output subtracts, one more at the end of the call for the rest. `undefined` where the
+     * operand was not read, or its row could not be.
+     */
+    readonly endGeneration: number | null | undefined;
+    /**
+     * Whether the operand was replaced while the call ran: for a pinned operand, its pin no longer names what the
+     * registry holds (another generation, the name deleted and created again, or the generation's number taken again by
+     * other bytes, which an operand an output subtracts is compared for by the object's fingerprint); for one read live, its
+     * generation or its incarnation changed from where the call began. `undefined` where `endGeneration` is, and for a live
+     * operand the call never opened, whose start is unknown.
+     */
+    readonly moved: boolean | undefined;
+    /** Chunk keys in its index, inside the range. */
+    readonly keys: number;
+    /** Chunk reads made of it, over all groups. */
+    readonly chunkReads: number;
+}
+/** What the call did to one output. */
+export interface CombineManyOutputStats {
+    /** The names of the operands its expression reads: the stored ones in call order, then the fed ones in `feed.names` order. */
+    readonly operands: readonly string[];
+    /** The group that computed it (from 0), or `null` when it never ran. */
+    readonly group: number | null;
+    /** Clock time the group's pass began and the output's publish settled (or it failed), when it ran. */
+    readonly startedAt: number | null;
+    readonly endedAt: number | null;
+    /** Chunks the output holds. */
+    readonly chunks: number;
+    /** What the index said its buffer could reach resident, which is what groups are planned on. */
+    readonly boundBytes: number;
+    /** The most its buffer held resident, in the ledger's own measure; compare with `boundBytes` to see how tight a plan was. */
+    readonly peakBytes: number;
+}
+export interface CombineManyStats {
+    /** Groups run, the re-runs of outputs a lying index made too big included. */
+    readonly groups: number;
+    readonly requests: {
+        /** Range requests sent for operand chunks (retries of one request count once). */
+        readonly rangeReads: number;
+        readonly rangeBytes: number;
+        /** (operand, key) chunk reads, the unit `budget` is counted in. */
+        readonly chunkReads: number;
+        /** Registry reads of the call itself: the re-checks, the end-of-call reads and the existence checks. */
+        readonly registryReads: number;
+        /**
+         * Operands whose index the call opened. An open is one tail read of the operand's object plus at most one registry
+         * row read, made by the store's readers, which a pin taken before the call may already have made.
+         */
+        readonly opens: number;
+        /**
+         * Outputs handed to a publish. A publish is one object write and one pointer write where it succeeds, plus the row and
+         * tail reads the load makes for its guard, which the call does not see.
+         */
+        readonly publishes: number;
+        /**
+         * The requests the call itself issues or can prove, a lower bound on what the drivers saw: `get` is range reads and
+         * registry reads, `put` the object write and pointer write of each published output and the object write of each
+         * refused one. To reconstruct the total GET-class requests add two reads (a tail and a row) for each operand a pin took,
+         * every named operand when `pin` is on, or for each of `opens` when it is off, and for each publish the reads of its load:
+         * two for a destination with a generation, three for one with none, and one more when its guard refuses it. Unknown to the call: a multipart upload's extra requests, a listing, and what a publish
+         * that threw sent.
+         */
+        readonly attributed: {
+            readonly get: number;
+            readonly put: number;
+        };
+    };
+    readonly budget: {
+        /** The limit used: `null` when lifted. */
+        readonly maxRequests: number | null;
+        /** Chunk reads the plan needed. */
+        readonly planned: number;
+        /** Chunk reads made. */
+        readonly used: number;
+    };
+    readonly memory: {
+        readonly maxBufferedBytes: number;
+        /** The most the ledger held at once. */
+        readonly highWaterBytes: number;
+        /** What the plan holds for the whole call, charged to the ledger: operand indexes, and each output's keys and expression. */
+        readonly planBytes: number;
+    };
+    readonly chunks: {
+        /** Index keys not read because no output needed them there, summed over groups. */
+        readonly pruned: number;
+    };
+    /** The most range requests in flight at once. */
+    readonly maxRangesInFlight: number;
+    /** What the feed delivered, when the call had one: records read, chunk keys evaluated from it, ids checked. */
+    readonly feed?: {
+        readonly records: number;
+        readonly keys: number;
+        readonly ids: number;
+    };
+    /** By operand name, an object with no prototype, so any name is a key. */
+    readonly operands: Readonly<Record<string, CombineManyOperandStats>>;
+    readonly outputs: readonly CombineManyOutputStats[];
+}
+export interface CombineManyRun<R> {
+    readonly outputs: readonly CombineManyOutcome<R>[];
+    readonly stats: CombineManyStats;
+}
+/** A request that passed every check that needs no I/O: what {@link runCombineMany} runs. */
+export interface CompiledCombineMany<R> {
+    readonly request: CombineManyRequest<R>;
+}
+/** The most nodes the expressions of one call may hold in all, so that a thousand large outputs cannot be planned at once. */
+export declare const MAX_CALL_NODES = 200000;
+/**
+ * Check a request: every expression, option and number, before any request is made. Throws {@link ValidationError}
+ * naming the output index and the path. Pure.
+ */
+export declare function compileCombineMany<R>(request: CombineManyRequest<R>): CompiledCombineMany<R>;
+/**
+ * `compiled` with its operands replaced, the same names in the same order: what a flavor does once it has pinned them,
+ * without checking the expressions again.
+ */
+export declare function rebindCombineMany<R>(compiled: CompiledCombineMany<R>, operands: readonly CombineManyOperand[]): CompiledCombineMany<R>;
+/**
+ * The resident bytes a group needs, kept as outputs are added so that one is priced in work proportional to its own
+ * operands, not to the group's size: its buffered outputs, its operand streams, the chunks being evaluated, the objects
+ * being published beside the buffers, and the key lists of its own plan.
+ */
+/** What {@link GroupCost} reads of an output. */
+export interface GroupCostItem {
+    readonly bound: number;
+    readonly serializedBound: number;
+    readonly rootKeys: Uint16Array;
+    readonly compiled: {
+        readonly operands: readonly number[];
+        readonly depth: number;
+    };
+}
+/**
+ * Work counters of the planner, for the tests that hold it to its cost: how many times a group was priced, how many
+ * operands those prices walked, and how many group plans were built. Plain numbers, never read by the planner.
+ */
+export declare const combineWork: {
+    adds: number;
+    withs: number;
+    operandSteps: number;
+    planBuilds: number;
+};
+export declare class GroupCost {
+    private readonly streamBytes;
+    private readonly publishConcurrency;
+    private outputs;
+    private depth;
+    /** The serialized bounds of the largest outputs, descending: at most `publishConcurrency` of them are written at once. */
+    private top;
+    private count;
+    private streams;
+    private planWork;
+    private readonly operands;
+    constructor(streamBytes: (operand: number) => number, publishConcurrency: number);
+    /** The cost of the group with `o` added. */
+    with(o: GroupCostItem): number;
+    add(o: GroupCostItem): void;
+    get size(): number;
+    private withLargest;
+    /**
+     * The bytes of objects being written beside the buffers, at most: the `publishConcurrency` largest outputs, each as
+     * large as its bound, with `bound` among the candidates.
+     */
+    private written;
+}
+/**
+ * Run a compiled request: read what its outputs need, evaluate them key by key and publish each. Never throws for a
+ * fault of one output or one operand: that is the output's outcome. Throws {@link ValidationError} for an operand that
+ * names no segment (unless allowed), and {@link BudgetExceededError} when its own plan needs more chunk reads than the
+ * budget, both before any chunk is read.
+ */
+export declare function runCombineMany<R>(compiled: CompiledCombineMany<R>, deps: CombineManyDeps): Promise<CombineManyRun<R>>;

package/dist/core/crbm-storage-source.d.ts

@@ -6,7 +6,7 @@ import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
 import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
 import { CrbmReader } from './crbm/reader.js';
 import type { CrbmReaderOptions } from './crbm/reader.js';
-import type { CodecBitmap, CodecInterface } from './codec.js';
+import type { CodecBitmap, CodecInterface, EncodedChunk } from './codec.js';
 import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
 export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
     /**
@@ -445,6 +445,13 @@ export declare function writeCrbmGeneration(driver: IStorageDriver, key: GenKey,
     sha256: string;
     fingerprint: string;
 }>;
+/**
+ * The chunks a load of ids, or of a decoded bitmap, writes: ascending by key, each non-empty and run-optimized. A decoded
+ * bitmap writes its own chunks, which never touches an id. Ids, and the bitmap of a codec that cannot encode its own
+ * chunks, are bucketed per chunk instead; both give the same bytes. Exported for a caller that holds the chunks itself
+ * instead of writing them.
+ */
+export declare function encodedChunksOfInput(ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput, codec: CodecInterface, clock: Clock | undefined): Promise<Iterable<EncodedChunk>>;
 /**
  * Whether the object under `key` is provably the one `fingerprint` names ({@link CrbmReader.fingerprint}), from one
  * read of its footer. False when it is another object, when it is gone, and when the read fails in any way: a caller