@cloudbitmaps/core 0.14.0 → 0.16.0

package/dist/core/crbm/reader.d.ts

@@ -1,6 +1,6 @@
 import type { BlobReader } from '../blob.js';
 import type { CrbmCrypto } from '../crypto.js';
-import type { GenerationMetadata } from '../ports.js';
+import type { ChunkRead, GenerationMetadata, ReadChunksOptions } from '../ports.js';
 /**
  * A parsed index as parallel typed arrays, one slot per entry, ascending by key. Typed arrays hold each field at
  * its own width with no per-entry object, so the retained heap is the arrays' byte length and nothing else.
@@ -58,6 +58,11 @@ export declare const fingerprintFor: (size: number, footerCrc: number) => string
  * checking that its target is what the row says.
  */
 export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
+/**
+ * {@link CrbmReader.open} for a reader that keeps a copy of the chunk region when the tail read returned the whole
+ * object and that region is at most `keepChunkBytesUpTo` bytes. Not exported from a package entry.
+ */
+export declare function openCrbmReaderKeeping(blob: BlobReader, options: CrbmReaderOptions, keepChunkBytesUpTo: number | undefined): Promise<CrbmReader>;
 export declare class CrbmReader {
     private readonly blob;
     private readonly objectSize;
@@ -82,6 +87,14 @@ export declare class CrbmReader {
     readonly metadata: GenerationMetadata | undefined;
     /** What that metadata adds to {@link retainedBytes}; 0 when there is none. */
     private readonly metadataWeight;
+    /** Where the chunk payloads end: the extension block's start when there is one, else the index's. */
+    private readonly payloadEnd;
+    /**
+     * A copy of the chunk region, `[PAYLOAD_START, payloadEnd)`, when `open()` kept one (see
+     * {@link openCrbmReaderKeeping}); `undefined` otherwise. Never handed out: a read of it is a copy or is
+     * consumed before it returns.
+     */
+    private readonly kept;
     private constructor();
     /**
      * Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
@@ -100,8 +113,9 @@ export declare class CrbmReader {
     /**
      * What this reader holds, the weight the storage reader cache bounds on: its parsed index
      * ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
-     * of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
-     * how the metadata is weighed may change in any release.
+     * of keys, plus the chunk bytes kept from the open, when it kept any. Equal to {@link retainedIndexBytes} for a
+     * generation with no metadata and nothing kept. A cache heuristic, not a contract: how the metadata is weighed may
+     * change in any release.
      */
     get retainedBytes(): number;
     /** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
@@ -130,7 +144,74 @@ export declare class CrbmReader {
      * not mutate it.
      */
     getChunk(chunkKey: number): Promise<Uint8Array | null>;
+    /**
+     * `[offset, offset + length)` of the object from the kept chunk region, or `undefined` when it is not wholly in it
+     * (or nothing was kept). The bounds checks are defence on untrusted bytes: the index's validation at open keeps every
+     * extent inside the region, so they do not fire for a reader that opened.
+     *
+     * The range is returned as a copy, so nothing a caller or an AEAD does with it changes the next read.
+     */
+    private keptRange;
+    /**
+     * Check one chunk's stored bytes against the index, and open them: the CRC32C first, then, on an encrypted object,
+     * the AEAD with the chunk's own associated data. The one check every read of a chunk goes through, whether the
+     * bytes came from a range of their own or from a slice of a larger one.
+     */
+    private openChunk;
+    /**
+     * The next needed chunk of the range in `cur`, checked and opened, as a stream hands it out; the range is let go of
+     * when this is its last chunk.
+     */
+    private nextChunk;
+    /**
+     * Read several chunks as a stream, one storage request per merged range ({@link planChunkReads}) rather than one per
+     * chunk. `chunkKeys` are in ascending order, a key may repeat, and anything else is a {@link ValidationError}. One
+     * item comes out per position of `chunkKeys`, in order: the chunk's bytes, or `null` where this generation has none,
+     * as {@link getChunk} answers for an absent key. A key given twice is read once and answered at both positions with
+     * the same view.
+     *
+     * The whole key list is planned once, up front, against this one generation's index. The ranges then run through a
+     * window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by default) that counts a range until the
+     * consumer has taken it, so a stream never holds more than that many ranges, in flight or landed, however many keys
+     * it was given or how slowly it is read; with `options.ramp` the window opens 1, 2, 4 … wide. A range is at most 1 MiB
+     * unless it is one chunk, which is at most the payload cap, so the bound is that many times the per-chunk cap
+     * (1 MiB, and 28 bytes more for an encrypted chunk). Nothing is read until the first item is asked for, and a
+     * consumer that stops launches no more: ranges already in flight finish and are dropped, and a failure of one of
+     * those is never raised.
+     *
+     * Every chunk is held to the check a read of it alone gets: its slice of the range must match its index entry's
+     * CRC32C, and on an encrypted object it opens under its own associated data, so a chunk moved to another place in
+     * the object, or another key's bytes, is refused. The bytes in a gap between needed chunks are never parsed. A
+     * range that comes back shorter than asked is an {@link IntegrityError}, and a chunk that fails its check ends the
+     * stream with that error when its turn comes, as a read of it alone would fail.
+     *
+     * `options.readRange`, when given, runs each range read, so a caller can retry one request without repeating the
+     * others; `options.now` times each request for `options.onRequest`, which hears of every range request that was sent, once, when
+     * it settles: taken, abandoned after the consumer stopped, or failed (a reader has no clock of its own, so `ms` is 0
+     * without one).
+     * A plain chunk is a writable view into the range it was read in: copy a chunk to keep it.
+     *
+     * A stream that fails (a range that errors, a chunk that fails its check) raises at once and sends nothing further:
+     * the ranges it still had in flight finish in the background, are not retried and never raise. Its `settled()`
+     * resolves when those have finished, for a caller that is about to read again and does not want them beside its next
+     * window (the source awaits it before a heal, and only then). It waits as long as the slowest of them takes to
+     * answer, which only the store's own read timeout bounds, so a read that never answers delays a heal and never an
+     * error.
+     */
+    readChunks(chunkKeys: readonly number[], options?: {
+        readonly concurrency?: number;
+        readonly ramp?: boolean | number;
+        readonly readRange?: <T>(read: () => Promise<T>) => Promise<T>;
+        readonly now?: () => number;
+        readonly onRequest?: ReadChunksOptions['onRequest'];
+    }): ChunkStream;
+    private streamChunks;
 }
+/** What {@link CrbmReader.readChunks} returns: the stream, and a way to wait for the reads it left in flight. */
+export type ChunkStream = AsyncGenerator<Omit<ChunkRead, 'version'>> & {
+    /** Resolves, never rejects, once every range the stream launched and did not hand out has finished. */
+    settled(): Promise<void>;
+};
 /**
  * Read the sections of an extension block, its bytes before the trailer, once their CRC has passed. Each section is
  * `u8 type ‖ u32 length ‖ bytes`, in strictly ascending type order, and together they fill the region exactly. Type 1

package/dist/core/crbm-storage-source.d.ts

@@ -1,12 +1,12 @@
 import { type IAuditSink } from './audit.js';
 import type { Yielder } from './cooperative.js';
 import type { Clock, Rng } from './determinism.js';
-import type { CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
-import type { ChunkRef, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
+import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
+import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
 import { CrbmReader } from './crbm/reader.js';
 import type { CrbmReaderOptions } from './crbm/reader.js';
 import type { CodecBitmap, CodecInterface } from './codec.js';
-import { DecodedLoadInput } from './load-input.js';
+import { ChunkLoadInput, DecodedLoadInput } from './load-input.js';
 export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
     /**
      * Optional {@link IRegistryDriver}. When provided, the current generation is resolved via the registry's
@@ -55,12 +55,16 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
      */
     readonly maxOpenSegments?: number;
     /**
-     * Aggregate byte ceiling on the parsed `.crbm` indices resident in the reader cache (default 64 MiB) — the
+     * Aggregate byte ceiling on what the reader cache holds (default 64 MiB): the parsed `.crbm` indices, and the chunk bytes a
+     * reader keeps when its first read returned a whole small object (see below) — the
      * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
      * parsed index can reach about 1.3 MB, so 1024 wide indices could pin over a GB and blow a small heap (e.g. a 128 MB
      * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedBytes} (the
-     * parsed index, and any metadata a generation carries) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
-     * of the count/byte bounds binds first. Lower it for memory-tight deployments with wide segments; a single
+     * parsed index, any metadata a generation carries, and any kept chunk bytes) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
+     * of the count/byte bounds binds first. A reader whose open read the whole object keeps a copy of its chunk region when
+     * that region is at most this ceiling divided by {@link maxOpenSegments} (64 KiB by default) — its fair share — so a small generation's
+     * chunks are served from memory with no further request, and a bigger one is read by range. Only a source with a timed
+     * pointer refresh ({@link currentGenTtlMs} above 0, a registry and a clock) keeps chunk bytes. Lower it for memory-tight deployments with wide segments; a single
      * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
      */
     readonly maxOpenIndexBytes?: number;
@@ -122,6 +126,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
     private readonly keystore;
     private readonly requireEncryption;
     private readonly readerOptions;
+    /** The most chunk bytes a reader keeps from its open: the reader cache's byte bound over its count bound, per reader; 0 without a timed pointer refresh. */
+    private readonly keepChunkBytesUpTo;
     private readonly clock;
     private readonly currentGenTtlMs;
     constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
@@ -317,6 +323,22 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
     /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
     private cryptoForRead;
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    /**
+     * Several chunks as a stream, in as few storage requests as {@link CrbmReader.readChunks} can make them (see
+     * {@link StorageChunkSource.getChunks}). The stream is served from one resolved generation, and every chunk says
+     * which version it came from. A generation swept, or an object replaced under its number, while the stream runs
+     * re-resolves the segment and carries on with the keys not yet yielded from whatever is current then, so each chunk
+     * is whole and from one generation and the stream may describe two; a second failure with no chunk yielded in between
+     * propagates. `options.retry` runs the resolution, each range request and the check that a failed read's object was
+     * replaced, so a transient fault repeats that step alone. Nothing is resolved until the first chunk is asked for.
+     */
+    getChunks(ref: SegmentRef, keys: readonly number[], options?: ReadChunksOptions): AsyncGenerator<ChunkRead>;
+    /**
+     * {@link getChunks} of a specific generation, the pinned shape read. It yields the chunks without a version: the
+     * caller that pinned the generation knows it. It never moves to another generation; `held` as for
+     * {@link getChunkAt}.
+     */
+    getChunksAt(ref: SegmentRef, generation: number, keys: readonly number[], held?: PinnedObject, options?: ReadChunksOptions): AsyncGenerator<Omit<ChunkRead, 'version'>>;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
@@ -633,7 +655,7 @@ export interface BulkLoadResult {
  * {@link WriteConflictError} (write-once). Without a registry a `StorageChunkSource` serves the **highest**
  * generation present, so a too-high number silently shadows real data; with one, `publishGeneration` decides.
  */
-export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput, options?: {
+export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenKey, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput | ChunkLoadInput, options?: {
     registry?: IRegistryDriver;
     keystore?: IKeystore;
     requireEncryption?: boolean;
@@ -681,6 +703,28 @@ export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenK
      */
     row?: RegistryRecord | null;
 }): Promise<BulkLoadResult>;
+/**
+ * What a load has already started when it calls {@link bulkLoadAhead}: its number, and the segment's data key, each
+ * still in flight so their round trips overlap the encoding. Both are joined where the write needs them, and neither
+ * may be left to reject unobserved.
+ */
+export interface LoadAhead {
+    /** The number the generation takes (see {@link nextLoadGeneration}); joined just before the object is written. */
+    readonly generation: PromiseLike<number>;
+    /**
+     * The key a caller already asked the keystore to unwrap, with the wrappings it asked about. Used only when the
+     * row the write reads after the ids carries exactly these wrappings; otherwise the write unwraps for itself.
+     */
+    readonly unwrapped?: {
+        readonly wrapped: readonly WrappedDek[];
+        readonly aead: PromiseLike<Aead>;
+    };
+}
+/**
+ * {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
+ * options, with the same meaning, and the same writes. The number is joined only where the object is written.
+ */
+export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, ahead: LoadAhead, ids: Parameters<typeof bulkLoadCrbmGeneration>[2], options?: NonNullable<Parameters<typeof bulkLoadCrbmGeneration>[3]>): Promise<BulkLoadResult>;
 /**
  * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
  * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old

package/dist/core/engine.d.ts

@@ -4,6 +4,13 @@ import type { Clock } from './determinism.js';
 import type { BoundedLru } from './lru.js';
 import type { IMetricsSink } from './metrics.js';
 import type { GenerationMetadata, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+/**
+ * Range requests a stream holds ahead of a combine or `iterate` by default (chunk keys, on a source that reads chunk by
+ * chunk, and for `count`'s fallback) — bounds memory and concurrent reads (invariant 6). A read that does not fit
+ * one range of a source's stream takes about one sequential round trip per this many ranges, so this, not the network,
+ * is what sets how long a long read takes.
+ */
+export declare const DEFAULT_INTERSECT_CONCURRENCY = 32;
 export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
@@ -49,11 +56,15 @@ export interface IdRange {
 /** Options common to the chunk-aligned combines. */
 export interface CombineOptions extends IdRange {
     /**
-     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
-     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
-     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
-     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
-     * `concurrency` keys per operand past the last one it used.
+     * How far ahead each operand is read: a positive integer, default 32. The library's own stores read each operand as
+     * one stream of coalesced ranges (chunks that sit near each other are one request), and `concurrency` is how many
+     * range requests a stream holds ahead of the read, in flight or landed and not yet used. So about
+     * `concurrency × operands` requests are in flight at most, and, since a range is at most 1 MiB (one chunk, when a chunk
+     * is larger), about `concurrency × operands` MiB are held, whatever the segment's size. The stream opens 4 ranges wide
+     * (or `concurrency`, if lower) and widens as ranges are taken, so a read that stops early has asked for little past
+     * where it stopped, and a read that fits one range makes one request however many chunks it needs. A source that
+     * reads chunk by chunk (a custom one) is read as before: `concurrency` chunk keys at once, opening 8 wide. A running
+     * stream holds its operand's reader outside the reader cache's bounds until it ends or the segment moves on.
      */
     readonly concurrency?: number;
     /** Override the store's per-op budget for this call (`false` lifts it). */
@@ -67,6 +78,11 @@ export interface CombineOptions extends IdRange {
      */
     readonly allowAbsentOperands?: boolean;
 }
+/** One chunk of a combine's result: the bitmap of its remainders, owned by whoever reads it. */
+export interface CombinedChunk {
+    readonly chunkKey: number;
+    readonly bitmap: CodecBitmap;
+}
 export declare class SegmentEngine {
     private readonly storage;
     private readonly cache;
@@ -83,6 +99,13 @@ export declare class SegmentEngine {
      * promise per distinct key in flight.
      */
     private readonly openReads;
+    /**
+     * The chunk streams open right now. An {@link invalidate} of a segment marks its streams, which then cache no chunk
+     * they deliver, as a read whose entry an invalidation dropped does not (streams hold no `openReads` entry).
+     */
+    private readonly openStreams;
+    /** How many times {@link invalidate} has been called: a read compares it with the count it began under. */
+    private invalidations;
     constructor(deps: EngineDeps);
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
@@ -117,9 +140,17 @@ export declare class SegmentEngine {
     /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
     private chunkWindow;
     /**
-     * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
-     * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
-     * chunks the range overlaps (see {@link IdRange}).
+     * The chunks of `seg` at `chunkKeys`, taken in ascending key order by a read that may stop at any point: one stream of
+     * coalesced ranges when the source has `getChunks`, opening 1, 2, 4 … ranges wide up to {@link DEFAULT_INTERSECT_CONCURRENCY},
+     * else the chunk-by-chunk window of that many reads. `close` stops what is left of it.
+     */
+    private chunkSequence;
+    /**
+     * Every id, ascending, reading the chunks as one stream of coalesced ranges (chunks that sit near each other are one
+     * request) that opens 1, 2, 4 … 32 ranges wide, so a read that stops early has asked for only a few ranges past the
+     * one it stopped in (each at most 1 MiB); with `range`, only the ids in `(after, through]`, fetching only the
+     * chunks the range overlaps (see {@link IdRange}). A source that reads chunk by chunk is read through a window of up to
+     * 32 chunk fetches that opens 1, 2, 4 … 32 wide.
      *
      * Two generators, not one with a branch: a second `yield` site in the full read's generator grows its frame, and
      * every id of a read that asked for no range paid for it (measured at about 5% per id). The full read keeps the
@@ -149,11 +180,11 @@ export declare class SegmentEngine {
      *
      * Chunk-skipping intersection. Aligns each segment's chunk-key set (read from the `.crbm` index, no payload),
      * keeps only keys present in **all** operands (a key missing from any operand can't contribute → its Storage
-     * chunks are never fetched — the core saving), then for each surviving key fetches the operands' chunks in
-     * parallel and hands them to the codec for the AND, streaming results through a bounded in-flight window.
+     * chunks are never fetched — the core saving), then reads each operand's surviving chunks as one stream of coalesced
+     * ranges and hands them, key by key, to the codec for the AND, streaming results through a bounded window.
      *
-     * **Memory:** the Storage payload footprint is bounded by the window (`concurrency × operands × chunk`), not by
-     * segment size — that's the Lambda-friendly property.
+     * **Memory:** the Storage payload footprint is bounded by the window (`concurrency × operands × chunk`, a range being
+     * at most a chunk's cap of 1 MiB), not by segment size — that's the Lambda-friendly property.
      *
      * Generation-consistent within the call (normal case): each operand's current generation is resolved **once**
      * up front (before the fan-out) and threaded into every chunk read, so a concurrent load can't corrupt or tear
@@ -164,7 +195,9 @@ export declare class SegmentEngine {
      * waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
      * (`maxOpenSegments`), whose re-read re-resolves fresh; a sweep deleting the generation it was reading, which
      * heals the read forward; and an invalidation, which this store's own `load`, `rollback` and `eraseSubject` make
-     * and `invalidate()` makes on request. Still whole/immutable per read, never torn.
+     * and `invalidate()` makes on request. Still whole/immutable per read, never torn. An operand's stream resolves its
+     * segment again before it serves each chunk, so each of those moves a running read as it would a read of one chunk, and
+     * the ranges it had requested of the earlier generation are dropped, not served.
      */
     intersect(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<number>;
     /**
@@ -189,6 +222,20 @@ export declare class SegmentEngine {
     unionBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
     /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
     andNotBatches(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<Uint32Array>;
+    /**
+     * {@link intersect}, as the chunks the result is made of: `{ chunkKey, bitmap }` in ascending key order, each a
+     * non-empty bitmap of the chunk's 16-bit remainders, cut to the read's range. What a materialisation writes
+     * straight into its new generation, with no id built on the way.
+     *
+     * **Each bitmap belongs to the caller, who may change it:** it is the combine's own accumulator or a bitmap built
+     * for the range's edge, never a chunk the cache holds or another operand's. The reads, the budget, the metric and the
+     * errors are the id reads' own.
+     */
+    intersectChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
+    /** {@link union}, as chunks: see {@link intersectChunks}. */
+    unionChunks(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<CombinedChunk>;
+    /** {@link andNot}, as chunks: see {@link intersectChunks}. */
+    andNotChunks(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<CombinedChunk>;
     /**
      * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
      * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
@@ -197,6 +244,10 @@ export declare class SegmentEngine {
     private combinePlan;
     private combine;
     private combineBatches;
+    /** The chunk read: the plan every combine shares, then each key's result, cut to the range at its edges. */
+    private combineChunks;
+    /** An edge chunk's bitmap cut to the window, as a new bitmap: the remainders `edgeIds` would yield. */
+    private cutToWindow;
     /**
      * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
      * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
@@ -207,6 +258,37 @@ export declare class SegmentEngine {
      * used as-is — one clone per key, not one per operand.
      */
     private combineChunk;
+    /**
+     * One operand's chunk at `key`: from its stream when it has one, else a read of that chunk on its own. Not `async`,
+     * so a stream's take is made before anything is awaited: concurrent takes of one stream must be made in key order.
+     */
+    private operandChunk;
+    /**
+     * `keys` of `seg` as a stream of coalesced ranges, or `undefined` when the read is per key: a source with no
+     * `getChunks`, a segment with no generation, or nothing to read. Nothing is requested, and no cache lookup made, until
+     * the first chunk is asked for (see {@link StreamedChunks}).
+     */
+    private openStreamed;
+    /**
+     * Open the stream of `streamed`, at its first uncached chunk `from`: over that chunk and the later keys the cache does
+     * not hold. Each later key is counted as a cache hit or miss here, so a key is counted once whichever way it is served.
+     */
+    private startStream;
+    private closeStreamed;
+    /**
+     * The decoded chunk at `chunkKey` of a streamed read: from the cache while the stream is unopened and the cache holds
+     * it; else the next chunk of its stream, or, for a chunk the cache held when the stream opened, the cached instance
+     * while it is still there, and a read of that chunk alone once it is not.
+     */
+    private streamedChunk;
+    /**
+     * The version a streamed chunk is cached under, or `null` for one not to cache: the version its read reports when the
+     * source names versions as its lookups do (`currentVersion`), and otherwise the one the read planned under, since a
+     * lookup by generation number would never find a chunk cached under a stream's version.
+     */
+    private streamedVersion;
+    /** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
+    private chunkCacheKey;
     /** The segment's chunk keys, ascending — a shape read off the index, no payload. Keys are untrusted (invariant 5). */
     private chunkKeys;
     /**
@@ -297,6 +379,16 @@ export declare class SegmentEngine {
      * one): callers read it or clone it, never mutate it.
      */
     private storageChunk;
-    /** One request for one chunk, decoded, range-checked and cached. */
+    /**
+     * A chunk's stored bytes decoded under the size cap (invariant 5) and range-checked: the one place a chunk becomes a
+     * bitmap, whether its bytes came from a request of its own or from a stream of coalesced ranges.
+     */
+    private decodeChunk;
+    /**
+     * One chunk, decoded, range-checked and cached. A source with `getChunks` is read through it, as a one-key stream, so
+     * a `storage.get` is reported only for a request that was sent (a chunk the source holds in memory sends none);
+     * the chunk is cached under the version it came from, as a stream's chunk is. A source without it is read by
+     * `getChunk`, one event a call.
+     */
     private fetchChunk;
 }

package/dist/core/item-pull.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Pulls the items of an async iterator one at a time so that the code relaying them never holds the one it just
+ * handed on. A binding of a generator's body (`for await (const item of …)`, `yield*`) is kept in the generator's
+ * frame across its next `yield`, so the relay would hold the previous item, and whatever buffer it is a view into,
+ * for as long as it waits for the next. Here the item sits in the pull between {@link advance} and {@link take}, and
+ * `take` clears it before it returns.
+ *
+ * ```ts
+ * const pull = new ItemPull(source);
+ * try {
+ *   while (await pull.advance()) yield pull.take((item) => wrap(item));
+ * } finally {
+ *   await pull.close();
+ * }
+ * ```
+ *
+ * Internal to core: not exported from any entry point.
+ */
+export declare class ItemPull<A> {
+    private readonly iterator;
+    private item;
+    constructor(source: AsyncIterable<A>);
+    /** Waits for the next item and keeps it inside; `false` once the source has ended. A failure of the source throws. */
+    advance(): Promise<boolean>;
+    /** Hands over the item {@link advance} fetched, through `map`, and forgets it. */
+    take<B>(map: (item: A) => B): B;
+    /** Ends the source early (a consumer that stopped), as leaving a `for await` does. */
+    close(): Promise<void>;
+}

package/dist/core/load-input.d.ts

@@ -10,6 +10,7 @@
  * All of it runs before the load's first request, so a malformed input costs no round trip and writes nothing.
  */
 import type { CodecBitmap, CodecInterface } from './codec.js';
+import type { CombinedChunk } from './engine.js';
 /**
  * Anything that serializes itself to portable Roaring: `roaring`'s `RoaringBitmap32` is one. Typed by shape, so
  * no codec's own class appears in a signature. A bitmap that can also say how large its serialization is
@@ -44,6 +45,14 @@ export declare class DecodedLoadInput {
     readonly bitmap: CodecBitmap;
     constructor(bitmap: CodecBitmap);
 }
+/**
+ * A combine's result as the bitmaps of its chunks, ascending by key: written as they are, never id by id. Made only by
+ * {@link loadSegmentChunks}, never by {@link prepareLoadInput}, so no input a load is handed can be one.
+ */
+export declare class ChunkLoadInput {
+    readonly chunks: AsyncIterable<CombinedChunk>;
+    constructor(chunks: AsyncIterable<CombinedChunk>);
+}
 /**
  * Check a load's input and decode a bitmap input, before the load makes any request.
  *

package/dist/core/load.d.ts

@@ -20,7 +20,7 @@
  * object, so it stays, and collection takes it like any other generation once one above it is current.
  */
 import { type IAuditSink } from './audit.js';
-import { type CodecInterface } from './codec.js';
+import { type CodecBitmap, type CodecInterface } from './codec.js';
 import type { Clock, Rng } from './determinism.js';
 import type { IKeystore } from './crypto.js';
 import { type LoadInput } from './load-input.js';
@@ -174,3 +174,22 @@ export interface LoadResult {
  * interesting cases (a guard tripped, a racing writer won) are operational facts rather than faults.
  */
 export declare function loadSegment(ref: SegmentRef, input: LoadInput, deps: LoadDeps, options?: LoadOptions): Promise<LoadResult>;
+/**
+ * {@link loadSegment} for the result of a combine, as the chunks it is made of: each `{ chunkKey, bitmap }` is the
+ * bitmap of one 16-bit chunk's remainders, and they ascend by key. The chunks are written as they are, so no id is built
+ * for a value. What the `*Into` verbs call; the unwired form of them, as {@link loadSegment} is of `store.load`.
+ *
+ * Every chunk is checked before anything is written. Its bitmap must be one the `codec` made (`codec.owns`, so a codec
+ * without it refuses every chunk), its key an integer in `[0, 65535]` above the one before, and its values 16-bit;
+ * anything else is a {@link ValidationError}. An empty bitmap is left out, as no empty chunk is stored.
+ *
+ * **The load consumes the bitmaps it is given.** Writing a chunk may re-encode its bitmap in place for size, which
+ * changes its representation and never its members, so pass bitmaps you made for this call and do not reuse them. A
+ * bitmap the chunk cache holds, or one a caller still reads, is not one to pass.
+ *
+ * Everything else is `loadSegment`'s, the options, the result and the refusals included.
+ */
+export declare function loadSegmentChunks(ref: SegmentRef, chunks: AsyncIterable<{
+    readonly chunkKey: number;
+    readonly bitmap: CodecBitmap;
+}>, deps: LoadDeps, options?: LoadOptions): Promise<LoadResult>;

package/dist/core/metrics.d.ts

@@ -26,19 +26,26 @@ export type MetricEvent = {
     readonly kind: 'storage.get';
     readonly namespace?: string;
     readonly segment: string;
-    /** One event per request, however many callers were waiting on it. Bytes returned (0 if the chunk was absent — a GET still happened). */
+    /**
+     * One event per request for chunks: a range request of a combine or `iterate`, which carries every chunk the read
+     * needs from a stretch of the object, or the one chunk of a read made on its own, however many callers were waiting
+     * on it. Bytes returned: the range's, the gaps between its chunks included (0 if a single chunk was absent — a GET
+     * still happened).
+     */
     readonly bytes: number;
     /**
-     * Elapsed wall time of the read — includes any transient-retry backoff on the storage call. From the
-     * injected clock, clamped to ≥ 0 (0 under the no-wait test clock).
+     * Elapsed wall time of the request — includes any transient-retry backoff on the storage call. From the
+     * injected clock, clamped to ≥ 0 (0 under the no-wait test clock). For a range of a stream, the time from the
+     * range being asked for to its answer, which the source measures.
      */
     readonly ms: number;
 } | {
     /**
      * One decoded-chunk cache lookup (emitted only when a cache is configured). `hit: false` means the lookup found
-     * no cached chunk. A caller that missed while another caller's read of the same chunk was open waits on that
-     * read and adds no `storage.get`, so the misses can outnumber the `storage.get` events; the `storage.get`
-     * count is the number of requests.
+     * no cached chunk. A combine or `iterate` looks up every chunk it needs when it opens its streams. A caller that
+     * missed while another caller's read of the same chunk was open waits on that read and adds no `storage.get`, and a
+     * range carries several chunks, so the misses can outnumber the `storage.get` events; the `storage.get` count is
+     * the number of requests for chunks.
      */
     readonly kind: 'cache';
     readonly hit: boolean;

package/dist/core/pinned-storage-source.d.ts

@@ -21,7 +21,7 @@
  * on some other handle. That is why the routing is by segment identity here rather than by which object the
  * caller happened to start from.
  */
-import type { ChunkRef, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { ChunkRef, ChunkRead, ReadChunksOptions, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
 import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
 /**
  * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
@@ -64,6 +64,12 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
      */
     private pinFor;
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    /**
+     * A pinned segment's chunks come from the pinned generation alone, under the version a pinned read keys by
+     * ({@link currentVersion}); another segment's go to the live source, which answers with its own.
+     */
+    getChunks(ref: SegmentRef, keys: readonly number[], options?: ReadChunksOptions): AsyncIterable<ChunkRead>;
+    private pinnedChunks;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;