@cloudbitmaps/core 0.13.0 → 0.15.0

package/dist/core/crbm/reader.d.ts

@@ -1,6 +1,6 @@
 import type { BlobReader } from '../blob.js';
 import type { CrbmCrypto } from '../crypto.js';
-import type { GenerationMetadata } from '../ports.js';
+import type { ChunkRead, GenerationMetadata, ReadChunksOptions } from '../ports.js';
 /**
  * A parsed index as parallel typed arrays, one slot per entry, ascending by key. Typed arrays hold each field at
  * its own width with no per-entry object, so the retained heap is the arrays' byte length and nothing else.
@@ -58,6 +58,11 @@ export declare const fingerprintFor: (size: number, footerCrc: number) => string
  * checking that its target is what the row says.
  */
 export declare function footerSaysEncrypted(blob: BlobReader): Promise<boolean>;
+/**
+ * {@link CrbmReader.open} for a reader that keeps a copy of the chunk region when the tail read returned the whole
+ * object and that region is at most `keepChunkBytesUpTo` bytes. Not exported from a package entry.
+ */
+export declare function openCrbmReaderKeeping(blob: BlobReader, options: CrbmReaderOptions, keepChunkBytesUpTo: number | undefined): Promise<CrbmReader>;
 export declare class CrbmReader {
     private readonly blob;
     private readonly objectSize;
@@ -82,6 +87,14 @@ export declare class CrbmReader {
     readonly metadata: GenerationMetadata | undefined;
     /** What that metadata adds to {@link retainedBytes}; 0 when there is none. */
     private readonly metadataWeight;
+    /** Where the chunk payloads end: the extension block's start when there is one, else the index's. */
+    private readonly payloadEnd;
+    /**
+     * A copy of the chunk region, `[PAYLOAD_START, payloadEnd)`, when `open()` kept one (see
+     * {@link openCrbmReaderKeeping}); `undefined` otherwise. Never handed out: a read of it is a copy or is
+     * consumed before it returns.
+     */
+    private readonly kept;
     private constructor();
     /**
      * Which object this reader opened, as its size and its footer's CRC. The footer covers the index's CRC, so two
@@ -100,8 +113,9 @@ export declare class CrbmReader {
     /**
      * What this reader holds, the weight the storage reader cache bounds on: its parsed index
      * ({@link retainedIndexBytes}), plus a weight for the metadata it decoded, from its canonical length and its number
-     * of keys. Equal to {@link retainedIndexBytes} for a generation with no metadata. A cache heuristic, not a contract:
-     * how the metadata is weighed may change in any release.
+     * of keys, plus the chunk bytes kept from the open, when it kept any. Equal to {@link retainedIndexBytes} for a
+     * generation with no metadata and nothing kept. A cache heuristic, not a contract: how the metadata is weighed may
+     * change in any release.
      */
     get retainedBytes(): number;
     /** Per-chunk cardinality (`chunkKey → count`) from the parsed index — no payload reads. */
@@ -130,7 +144,74 @@ export declare class CrbmReader {
      * not mutate it.
      */
     getChunk(chunkKey: number): Promise<Uint8Array | null>;
+    /**
+     * `[offset, offset + length)` of the object from the kept chunk region, or `undefined` when it is not wholly in it
+     * (or nothing was kept). The bounds checks are defence on untrusted bytes: the index's validation at open keeps every
+     * extent inside the region, so they do not fire for a reader that opened.
+     *
+     * The range is returned as a copy, so nothing a caller or an AEAD does with it changes the next read.
+     */
+    private keptRange;
+    /**
+     * Check one chunk's stored bytes against the index, and open them: the CRC32C first, then, on an encrypted object,
+     * the AEAD with the chunk's own associated data. The one check every read of a chunk goes through, whether the
+     * bytes came from a range of their own or from a slice of a larger one.
+     */
+    private openChunk;
+    /**
+     * The next needed chunk of the range in `cur`, checked and opened, as a stream hands it out; the range is let go of
+     * when this is its last chunk.
+     */
+    private nextChunk;
+    /**
+     * Read several chunks as a stream, one storage request per merged range ({@link planChunkReads}) rather than one per
+     * chunk. `chunkKeys` are in ascending order, a key may repeat, and anything else is a {@link ValidationError}. One
+     * item comes out per position of `chunkKeys`, in order: the chunk's bytes, or `null` where this generation has none,
+     * as {@link getChunk} answers for an absent key. A key given twice is read once and answered at both positions with
+     * the same view.
+     *
+     * The whole key list is planned once, up front, against this one generation's index. The ranges then run through a
+     * window of `options.concurrency` reads ({@link MAX_RANGES_IN_FLIGHT} by default) that counts a range until the
+     * consumer has taken it, so a stream never holds more than that many ranges, in flight or landed, however many keys
+     * it was given or how slowly it is read; with `options.ramp` the window opens 1, 2, 4 … wide. A range is at most 1 MiB
+     * unless it is one chunk, which is at most the payload cap, so the bound is that many times the per-chunk cap
+     * (1 MiB, and 28 bytes more for an encrypted chunk). Nothing is read until the first item is asked for, and a
+     * consumer that stops launches no more: ranges already in flight finish and are dropped, and a failure of one of
+     * those is never raised.
+     *
+     * Every chunk is held to the check a read of it alone gets: its slice of the range must match its index entry's
+     * CRC32C, and on an encrypted object it opens under its own associated data, so a chunk moved to another place in
+     * the object, or another key's bytes, is refused. The bytes in a gap between needed chunks are never parsed. A
+     * range that comes back shorter than asked is an {@link IntegrityError}, and a chunk that fails its check ends the
+     * stream with that error when its turn comes, as a read of it alone would fail.
+     *
+     * `options.readRange`, when given, runs each range read, so a caller can retry one request without repeating the
+     * others; `options.now` times each request for `options.onRequest`, which hears of every range request that was sent, once, when
+     * it settles: taken, abandoned after the consumer stopped, or failed (a reader has no clock of its own, so `ms` is 0
+     * without one).
+     * A plain chunk is a writable view into the range it was read in: copy a chunk to keep it.
+     *
+     * A stream that fails (a range that errors, a chunk that fails its check) raises at once and sends nothing further:
+     * the ranges it still had in flight finish in the background, are not retried and never raise. Its `settled()`
+     * resolves when those have finished, for a caller that is about to read again and does not want them beside its next
+     * window (the source awaits it before a heal, and only then). It waits as long as the slowest of them takes to
+     * answer, which only the store's own read timeout bounds, so a read that never answers delays a heal and never an
+     * error.
+     */
+    readChunks(chunkKeys: readonly number[], options?: {
+        readonly concurrency?: number;
+        readonly ramp?: boolean | number;
+        readonly readRange?: <T>(read: () => Promise<T>) => Promise<T>;
+        readonly now?: () => number;
+        readonly onRequest?: ReadChunksOptions['onRequest'];
+    }): ChunkStream;
+    private streamChunks;
 }
+/** What {@link CrbmReader.readChunks} returns: the stream, and a way to wait for the reads it left in flight. */
+export type ChunkStream = AsyncGenerator<Omit<ChunkRead, 'version'>> & {
+    /** Resolves, never rejects, once every range the stream launched and did not hand out has finished. */
+    settled(): Promise<void>;
+};
 /**
  * Read the sections of an extension block, its bytes before the trailer, once their CRC has passed. Each section is
  * `u8 type ‖ u32 length ‖ bytes`, in strictly ascending type order, and together they fill the region exactly. Type 1

package/dist/core/crbm-storage-source.d.ts

@@ -1,8 +1,8 @@
 import { type IAuditSink } from './audit.js';
 import type { Yielder } from './cooperative.js';
 import type { Clock, Rng } from './determinism.js';
-import type { CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
-import type { ChunkRef, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
+import type { Aead, CrbmCrypto, IKeystore, WrappedDek } from './crypto.js';
+import type { ChunkRef, ChunkRead, ReadChunksOptions, StorageChunkSource, GenerationMetadata, GenerationSummary, GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, RegistrySummary, SegmentRef, SegmentSize, Token } from './ports.js';
 import { CrbmReader } from './crbm/reader.js';
 import type { CrbmReaderOptions } from './crbm/reader.js';
 import type { CodecBitmap, CodecInterface } from './codec.js';
@@ -55,12 +55,16 @@ export interface CrbmStorageChunkSourceOptions extends CrbmReaderOptions {
      */
     readonly maxOpenSegments?: number;
     /**
-     * Aggregate byte ceiling on the parsed `.crbm` indices resident in the reader cache (default 64 MiB) — the
+     * Aggregate byte ceiling on what the reader cache holds (default 64 MiB): the parsed `.crbm` indices, and the chunk bytes a
+     * reader keeps when its first read returned a whole small object (see below) — the
      * **second half of that memory bound**. `maxOpenSegments` alone bounds by *count*, but a wide/dense segment's
      * parsed index can reach about 1.3 MB, so 1024 wide indices could pin over a GB and blow a small heap (e.g. a 128 MB
      * Lambda) while the count is nominally "in bounds". This caps the summed {@link CrbmReader.retainedBytes} (the
-     * parsed index, and any metadata a generation carries) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
-     * of the count/byte bounds binds first. Lower it for memory-tight deployments with wide segments; a single
+     * parsed index, any metadata a generation carries, and any kept chunk bytes) across cached readers; the least-recently-used reader is evicted once the total would exceed it — whichever
+     * of the count/byte bounds binds first. A reader whose open read the whole object keeps a copy of its chunk region when
+     * that region is at most this ceiling divided by {@link maxOpenSegments} (64 KiB by default) — its fair share — so a small generation's
+     * chunks are served from memory with no further request, and a bigger one is read by range. Only a source with a timed
+     * pointer refresh ({@link currentGenTtlMs} above 0, a registry and a clock) keeps chunk bytes. Lower it for memory-tight deployments with wide segments; a single
      * segment whose index alone exceeds the budget is still cached (it can't be shrunk) but nothing else alongside.
      */
     readonly maxOpenIndexBytes?: number;
@@ -122,6 +126,8 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
     private readonly keystore;
     private readonly requireEncryption;
     private readonly readerOptions;
+    /** The most chunk bytes a reader keeps from its open: the reader cache's byte bound over its count bound, per reader; 0 without a timed pointer refresh. */
+    private readonly keepChunkBytesUpTo;
     private readonly clock;
     private readonly currentGenTtlMs;
     constructor(driver: IStorageDriver, options?: CrbmStorageChunkSourceOptions);
@@ -317,6 +323,22 @@ export declare class CrbmStorageChunkSource implements StorageChunkSource {
     /** Build the per-generation decryption context for an encrypted segment (undefined for cleartext). */
     private cryptoForRead;
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    /**
+     * Several chunks as a stream, in as few storage requests as {@link CrbmReader.readChunks} can make them (see
+     * {@link StorageChunkSource.getChunks}). The stream is served from one resolved generation, and every chunk says
+     * which version it came from. A generation swept, or an object replaced under its number, while the stream runs
+     * re-resolves the segment and carries on with the keys not yet yielded from whatever is current then, so each chunk
+     * is whole and from one generation and the stream may describe two; a second failure with no chunk yielded in between
+     * propagates. `options.retry` runs the resolution, each range request and the check that a failed read's object was
+     * replaced, so a transient fault repeats that step alone. Nothing is resolved until the first chunk is asked for.
+     */
+    getChunks(ref: SegmentRef, keys: readonly number[], options?: ReadChunksOptions): AsyncGenerator<ChunkRead>;
+    /**
+     * {@link getChunks} of a specific generation, the pinned shape read. It yields the chunks without a version: the
+     * caller that pinned the generation knows it. It never moves to another generation; `held` as for
+     * {@link getChunkAt}.
+     */
+    getChunksAt(ref: SegmentRef, generation: number, keys: readonly number[], held?: PinnedObject, options?: ReadChunksOptions): AsyncGenerator<Omit<ChunkRead, 'version'>>;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
@@ -681,6 +703,28 @@ export declare function bulkLoadCrbmGeneration(driver: IStorageDriver, key: GenK
      */
     row?: RegistryRecord | null;
 }): Promise<BulkLoadResult>;
+/**
+ * What a load has already started when it calls {@link bulkLoadAhead}: its number, and the segment's data key, each
+ * still in flight so their round trips overlap the encoding. Both are joined where the write needs them, and neither
+ * may be left to reject unobserved.
+ */
+export interface LoadAhead {
+    /** The number the generation takes (see {@link nextLoadGeneration}); joined just before the object is written. */
+    readonly generation: PromiseLike<number>;
+    /**
+     * The key a caller already asked the keystore to unwrap, with the wrappings it asked about. Used only when the
+     * row the write reads after the ids carries exactly these wrappings; otherwise the write unwraps for itself.
+     */
+    readonly unwrapped?: {
+        readonly wrapped: readonly WrappedDek[];
+        readonly aead: PromiseLike<Aead>;
+    };
+}
+/**
+ * {@link bulkLoadCrbmGeneration} for a load whose number, and perhaps its key, are still being fetched: the same
+ * options, with the same meaning, and the same writes. The number is joined only where the object is written.
+ */
+export declare function bulkLoadAhead(driver: IStorageDriver, ref: SegmentRef, ahead: LoadAhead, ids: Iterable<number> | AsyncIterable<number> | DecodedLoadInput, options?: NonNullable<Parameters<typeof bulkLoadCrbmGeneration>[3]>): Promise<BulkLoadResult>;
 /**
  * Open a {@link CrbmReader} on one generation over the storage driver's range/tail reads (decrypting if `crypto`).
  * The reader the write paths use to re-read what they wrote, and the erasure rewrite uses to stream the old

package/dist/core/engine.d.ts

@@ -4,6 +4,13 @@ import type { Clock } from './determinism.js';
 import type { BoundedLru } from './lru.js';
 import type { IMetricsSink } from './metrics.js';
 import type { GenerationMetadata, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+/**
+ * Range requests a stream holds ahead of a combine or `iterate` by default (chunk keys, on a source that reads chunk by
+ * chunk, and for `count`'s fallback) — bounds memory and concurrent reads (invariant 6). A read that does not fit
+ * one range of a source's stream takes about one sequential round trip per this many ranges, so this, not the network,
+ * is what sets how long a long read takes.
+ */
+export declare const DEFAULT_INTERSECT_CONCURRENCY = 32;
 export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
@@ -49,11 +56,15 @@ export interface IdRange {
 /** Options common to the chunk-aligned combines. */
 export interface CombineOptions extends IdRange {
     /**
-     * Max chunk keys resolved concurrently — bounds the Storage footprint (about `concurrency × operands` chunks in
-     * flight). A positive integer; default 32. The window opens 8 keys wide (or `concurrency`, if lower) and widens
-     * as keys are taken, so a read that stops early fetches no more than a few keys ahead; one that runs to the end
-     * spends nearly all of its round trips at the full width. A read that stops early has requested up to
-     * `concurrency` keys per operand past the last one it used.
+     * How far ahead each operand is read: a positive integer, default 32. The library's own stores read each operand as
+     * one stream of coalesced ranges (chunks that sit near each other are one request), and `concurrency` is how many
+     * range requests a stream holds ahead of the read, in flight or landed and not yet used. So about
+     * `concurrency × operands` requests are in flight at most, and, since a range is at most 1 MiB (one chunk, when a chunk
+     * is larger), about `concurrency × operands` MiB are held, whatever the segment's size. The stream opens 4 ranges wide
+     * (or `concurrency`, if lower) and widens as ranges are taken, so a read that stops early has asked for little past
+     * where it stopped, and a read that fits one range makes one request however many chunks it needs. A source that
+     * reads chunk by chunk (a custom one) is read as before: `concurrency` chunk keys at once, opening 8 wide. A running
+     * stream holds its operand's reader outside the reader cache's bounds until it ends or the segment moves on.
      */
     readonly concurrency?: number;
     /** Override the store's per-op budget for this call (`false` lifts it). */
@@ -83,6 +94,11 @@ export declare class SegmentEngine {
      * promise per distinct key in flight.
      */
     private readonly openReads;
+    /**
+     * The chunk streams open right now. An {@link invalidate} of a segment marks its streams, which then cache no chunk
+     * they deliver, as a read whose entry an invalidation dropped does not (streams hold no `openReads` entry).
+     */
+    private readonly openStreams;
     constructor(deps: EngineDeps);
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
@@ -117,9 +133,17 @@ export declare class SegmentEngine {
     /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
     private chunkWindow;
     /**
-     * Every id, ascending, reading ahead through a window of up to 32 chunk fetches that opens 1, 2, 4 … 32 wide, so a
-     * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
-     * chunks the range overlaps (see {@link IdRange}).
+     * The chunks of `seg` at `chunkKeys`, taken in ascending key order by a read that may stop at any point: one stream of
+     * coalesced ranges when the source has `getChunks`, opening 1, 2, 4 … ranges wide up to {@link DEFAULT_INTERSECT_CONCURRENCY},
+     * else the chunk-by-chunk window of that many reads. `close` stops what is left of it.
+     */
+    private chunkSequence;
+    /**
+     * Every id, ascending, reading the chunks as one stream of coalesced ranges (chunks that sit near each other are one
+     * request) that opens 1, 2, 4 … 32 ranges wide, so a read that stops early has asked for only a few ranges past the
+     * one it stopped in (each at most 1 MiB); with `range`, only the ids in `(after, through]`, fetching only the
+     * chunks the range overlaps (see {@link IdRange}). A source that reads chunk by chunk is read through a window of up to
+     * 32 chunk fetches that opens 1, 2, 4 … 32 wide.
      *
      * Two generators, not one with a branch: a second `yield` site in the full read's generator grows its frame, and
      * every id of a read that asked for no range paid for it (measured at about 5% per id). The full read keeps the
@@ -128,6 +152,13 @@ export declare class SegmentEngine {
     iterate(seg: SegmentRef, range?: IdRange): AsyncGenerator<number>;
     private iterateAll;
     private iterateRange;
+    /**
+     * {@link iterate}, one `Uint32Array` per chunk instead of one id at a time: the same chunks, in the same order, with
+     * the same ids (a range cut at its edges the same way), and no array for a chunk that holds none of them. The
+     * reads, the budget charge and the window are the per-id read's. Each array is at most 65,536 ids (256 KiB) and is
+     * the caller's to keep.
+     */
+    iterateBatches(seg: SegmentRef, range?: IdRange): AsyncGenerator<Uint32Array>;
     /**
      * `segs[0] ∩ segs[1] ∩ …`, minus every id in `options.exclude` — streamed ascending.
      *
@@ -142,11 +173,11 @@ export declare class SegmentEngine {
      *
      * Chunk-skipping intersection. Aligns each segment's chunk-key set (read from the `.crbm` index, no payload),
      * keeps only keys present in **all** operands (a key missing from any operand can't contribute → its Storage
-     * chunks are never fetched — the core saving), then for each surviving key fetches the operands' chunks in
-     * parallel and hands them to the codec for the AND, streaming results through a bounded in-flight window.
+     * chunks are never fetched — the core saving), then reads each operand's surviving chunks as one stream of coalesced
+     * ranges and hands them, key by key, to the codec for the AND, streaming results through a bounded window.
      *
-     * **Memory:** the Storage payload footprint is bounded by the window (`concurrency × operands × chunk`), not by
-     * segment size — that's the Lambda-friendly property.
+     * **Memory:** the Storage payload footprint is bounded by the window (`concurrency × operands × chunk`, a range being
+     * at most a chunk's cap of 1 MiB), not by segment size — that's the Lambda-friendly property.
      *
      * Generation-consistent within the call (normal case): each operand's current generation is resolved **once**
      * up front (before the fan-out) and threaded into every chunk read, so a concurrent load can't corrupt or tear
@@ -157,7 +188,9 @@ export declare class SegmentEngine {
      * waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
      * (`maxOpenSegments`), whose re-read re-resolves fresh; a sweep deleting the generation it was reading, which
      * heals the read forward; and an invalidation, which this store's own `load`, `rollback` and `eraseSubject` make
-     * and `invalidate()` makes on request. Still whole/immutable per read, never torn.
+     * and `invalidate()` makes on request. Still whole/immutable per read, never torn. An operand's stream resolves its
+     * segment again before it serves each chunk, so each of those moves a running read as it would a read of one chunk, and
+     * the ranges it had requested of the earlier generation are dropped, not served.
      */
     intersect(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<number>;
     /**
@@ -176,7 +209,20 @@ export declare class SegmentEngine {
      * the cost scales with the segment being filtered rather than with the size of the suppression list.
      */
     andNot(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<number>;
+    /** {@link intersect}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    intersectBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link union}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    unionBatches(segs: readonly SegmentRef[], options?: CombineOptions): AsyncGenerator<Uint32Array>;
+    /** {@link andNot}, one `Uint32Array` per chunk — see {@link iterateBatches}. */
+    andNotBatches(seg: SegmentRef, excludes: readonly SegmentRef[], options?: Omit<CombineOptions, 'exclude'>): AsyncGenerator<Uint32Array>;
+    /**
+     * Everything a combine decides before it reads a chunk: validation, each operand's keys and generation, the
+     * absent-operand refusal, key alignment, the budget and the metric. `null` when the range is empty. The per-id
+     * read and the batch read share it, so the two cannot disagree about what is read or charged.
+     */
+    private combinePlan;
     private combine;
+    private combineBatches;
     /**
      * The AND (or OR) of one chunk key across all operands, minus the excludes that hold it, or `null` if empty.
      * Operand chunks are fetched **in parallel** (the spec's parallel byte-range reads).
@@ -187,6 +233,32 @@ export declare class SegmentEngine {
      * used as-is — one clone per key, not one per operand.
      */
     private combineChunk;
+    /**
+     * One operand's chunk at `key`: from its stream when it has one, else a read of that chunk on its own. Not `async`,
+     * so a stream's take is made before anything is awaited: concurrent takes of one stream must be made in key order.
+     */
+    private operandChunk;
+    /**
+     * Open `keys` of `seg` as a stream of coalesced ranges, or `undefined` when the read is per key: a source with no
+     * `getChunks`, a segment with no generation, or nothing to read. The chunks already decoded in the cache are taken out
+     * of the key list first and served from there, so a repeat read asks the source for nothing it holds. Nothing is
+     * requested until the first chunk is asked for.
+     */
+    private openStreamed;
+    private closeStreamed;
+    /**
+     * The decoded chunk at `chunkKey` of a streamed read: the next chunk of its stream, or, for a chunk the cache held
+     * when the read opened, the cached instance while it is still there, and a read of that chunk alone once it is not.
+     */
+    private streamedChunk;
+    /**
+     * The version a streamed chunk is cached under, or `null` for one not to cache: the version its read reports when the
+     * source names versions as its lookups do (`currentVersion`), and otherwise the one the read planned under, since a
+     * lookup by generation number would never find a chunk cached under a stream's version.
+     */
+    private streamedVersion;
+    /** The cache key of a chunk: by the version it was read under, or by segment and key alone for a source with none. */
+    private chunkCacheKey;
     /** The segment's chunk keys, ascending — a shape read off the index, no payload. Keys are untrusted (invariant 5). */
     private chunkKeys;
     /**
@@ -277,6 +349,16 @@ export declare class SegmentEngine {
      * one): callers read it or clone it, never mutate it.
      */
     private storageChunk;
-    /** One request for one chunk, decoded, range-checked and cached. */
+    /**
+     * A chunk's stored bytes decoded under the size cap (invariant 5) and range-checked: the one place a chunk becomes a
+     * bitmap, whether its bytes came from a request of its own or from a stream of coalesced ranges.
+     */
+    private decodeChunk;
+    /**
+     * One chunk, decoded, range-checked and cached. A source with `getChunks` is read through it, as a one-key stream, so
+     * a `storage.get` is reported only for a request that was sent (a chunk the source holds in memory sends none);
+     * the chunk is cached under the version it came from, as a stream's chunk is. A source without it is read by
+     * `getChunk`, one event a call.
+     */
     private fetchChunk;
 }

package/dist/core/item-pull.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Pulls the items of an async iterator one at a time so that the code relaying them never holds the one it just
+ * handed on. A binding of a generator's body (`for await (const item of …)`, `yield*`) is kept in the generator's
+ * frame across its next `yield`, so the relay would hold the previous item, and whatever buffer it is a view into,
+ * for as long as it waits for the next. Here the item sits in the pull between {@link advance} and {@link take}, and
+ * `take` clears it before it returns.
+ *
+ * ```ts
+ * const pull = new ItemPull(source);
+ * try {
+ *   while (await pull.advance()) yield pull.take((item) => wrap(item));
+ * } finally {
+ *   await pull.close();
+ * }
+ * ```
+ *
+ * Internal to core: not exported from any entry point.
+ */
+export declare class ItemPull<A> {
+    private readonly iterator;
+    private item;
+    constructor(source: AsyncIterable<A>);
+    /** Waits for the next item and keeps it inside; `false` once the source has ended. A failure of the source throws. */
+    advance(): Promise<boolean>;
+    /** Hands over the item {@link advance} fetched, through `map`, and forgets it. */
+    take<B>(map: (item: A) => B): B;
+    /** Ends the source early (a consumer that stopped), as leaving a `for await` does. */
+    close(): Promise<void>;
+}

package/dist/core/metrics.d.ts

@@ -26,19 +26,26 @@ export type MetricEvent = {
     readonly kind: 'storage.get';
     readonly namespace?: string;
     readonly segment: string;
-    /** One event per request, however many callers were waiting on it. Bytes returned (0 if the chunk was absent — a GET still happened). */
+    /**
+     * One event per request for chunks: a range request of a combine or `iterate`, which carries every chunk the read
+     * needs from a stretch of the object, or the one chunk of a read made on its own, however many callers were waiting
+     * on it. Bytes returned: the range's, the gaps between its chunks included (0 if a single chunk was absent — a GET
+     * still happened).
+     */
     readonly bytes: number;
     /**
-     * Elapsed wall time of the read — includes any transient-retry backoff on the storage call. From the
-     * injected clock, clamped to ≥ 0 (0 under the no-wait test clock).
+     * Elapsed wall time of the request — includes any transient-retry backoff on the storage call. From the
+     * injected clock, clamped to ≥ 0 (0 under the no-wait test clock). For a range of a stream, the time from the
+     * range being asked for to its answer, which the source measures.
      */
     readonly ms: number;
 } | {
     /**
      * One decoded-chunk cache lookup (emitted only when a cache is configured). `hit: false` means the lookup found
-     * no cached chunk. A caller that missed while another caller's read of the same chunk was open waits on that
-     * read and adds no `storage.get`, so the misses can outnumber the `storage.get` events; the `storage.get`
-     * count is the number of requests.
+     * no cached chunk. A combine or `iterate` looks up every chunk it needs when it opens its streams. A caller that
+     * missed while another caller's read of the same chunk was open waits on that read and adds no `storage.get`, and a
+     * range carries several chunks, so the misses can outnumber the `storage.get` events; the `storage.get` count is
+     * the number of requests for chunks.
      */
     readonly kind: 'cache';
     readonly hit: boolean;

package/dist/core/pinned-storage-source.d.ts

@@ -21,7 +21,7 @@
  * on some other handle. That is why the routing is by segment identity here rather than by which object the
  * caller happened to start from.
  */
-import type { ChunkRef, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { ChunkRef, ChunkRead, ReadChunksOptions, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
 import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
 /**
  * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
@@ -64,6 +64,12 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
      */
     private pinFor;
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
+    /**
+     * A pinned segment's chunks come from the pinned generation alone, under the version a pinned read keys by
+     * ({@link currentVersion}); another segment's go to the live source, which answers with its own.
+     */
+    getChunks(ref: SegmentRef, keys: readonly number[], options?: ReadChunksOptions): AsyncIterable<ChunkRead>;
+    private pinnedChunks;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;

package/dist/core/ports.d.ts

@@ -40,11 +40,80 @@ export interface GenKey extends SegmentRef {
 export interface SegmentSize {
     readonly sizeBytes: number;
 }
+/**
+ * One chunk of a {@link StorageChunkSource.getChunks} stream: the key asked for, its bytes (`null` where the generation
+ * holds none), and the version of the generation the bytes were read from, or `null` when the segment has no
+ * generation. The version is what {@link StorageChunkSource.currentVersion} reports for that generation, so a caller
+ * can key what it keeps by the generation the bytes came from rather than the one it resolved earlier.
+ */
+export interface ChunkRead {
+    readonly key: number;
+    readonly bytes: Uint8Array | null;
+    readonly version: string | null;
+}
+/** Options of {@link StorageChunkSource.getChunks}. */
+export interface ReadChunksOptions {
+    /**
+     * Runs one storage request, and may run it again if it fails. A source passes each request it makes through
+     * this, so a caller that retries transient faults repeats the one request that failed, not the ones that landed.
+     * Absent, each request runs once.
+     */
+    readonly retry?: <T>(request: () => Promise<T>) => Promise<T>;
+    /**
+     * How many range requests the stream holds ahead of its consumer, in flight or landed and not yet taken: a positive
+     * integer, 32 by default. It bounds the requests in flight, and the memory the stream holds, at this many times the
+     * largest range (1 MiB, 1 MiB + 28 B for one encrypted chunk).
+     */
+    readonly concurrency?: number;
+    /**
+     * Open the window 1, 2, 4 … ranges wide up to `concurrency` instead of at full width, so a reader that stops early
+     * has asked for little; a number starts it that wide instead of at 1 (a positive integer), and it doubles from there.
+     */
+    readonly ramp?: boolean | number;
+    /**
+     * Called once for every range request the stream sends, when it settles: a request the consumer took, one that was
+     * still in flight when the consumer stopped, and one that failed. `bytes` is what the request moved, the gaps between
+     * the chunks it carried included (0 when it failed), and `ms` how long it took, retries included (0 when the source
+     * has no clock). It is how a caller counts what it is billed for, since several chunks can share a request and
+     * a stopped stream leaves some in flight. A throw from it is ignored.
+     */
+    readonly onRequest?: (request: {
+        readonly bytes: number;
+        readonly ms: number;
+    }) => void;
+}
 /** Per-chunk read view of the immutable Storage tier (implemented by the `.crbm` reader). */
 export interface StorageChunkSource {
     /** Read-only bytes for the chunk, or `null` if absent. Callers must not mutate the buffer. */
     getChunk(ref: ChunkRef): Promise<Uint8Array | null>;
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
+    /**
+     * Optional: several chunks of one segment as a stream, read **from one generation** with fewer storage requests than
+     * one per chunk where the chunks sit near each other in the object. `keys` are chunk keys in ascending order (a key
+     * may repeat; any other order is a `ValidationError`). The stream yields one {@link ChunkRead} per position of
+     * `keys`, in that order, as the requests that carry them land; a key the generation does not hold yields `null`
+     * bytes, as {@link StorageChunkSource.getChunk} answers. Each chunk is checked as a read of it alone is.
+     *
+     * **Nothing is resolved or read until the first chunk is asked for**, and the source holds at most
+     * {@link ReadChunksOptions.concurrency} ranges at a time (in flight, or landed and not yet taken), however many keys
+     * there are: a consumer that is slow holds the stream back, and one that stops early (`break`, or `return()`) stops
+     * the reads, so the requests it had already started (at most that many) finish and are dropped, and are not retried.
+     * A stream that fails raises at once and sends nothing further, and the requests it left in flight finish in the
+     * background and never raise. A plain chunk may be a view into a buffer of up to 1 MiB that it shares with its
+     * neighbours (and the same view at two positions for a key asked twice), so do not write to it, and copy a chunk to
+     * keep it.
+     *
+     * One stream reads one generation. If that generation is swept, or its object replaced, while the stream runs, a
+     * source may re-resolve the segment and continue with the keys not yet yielded from the generation that is current
+     * then; each chunk says which version it came from, and each is whole and verified either way. Before it does, it
+     * waits for the requests the failed stream left in flight, so a heal never opens a second window beside them: a
+     * request that never answers delays a heal, as it would hang a read of one chunk, until the driver's read timeout (if
+     * it has one) ends it, and never delays an error. A source that reads a pinned generation never heals.
+     *
+     * A source that cannot read a range of an object omits this, and a caller reads chunk by chunk; the `.crbm`
+     * source, which is what the stores the library ships read through, implements it.
+     */
+    getChunks?(ref: SegmentRef, keys: readonly number[], options?: ReadChunksOptions): AsyncIterable<ChunkRead>;
     /**
      * Optional: the current generation's grounded size, cheaply (from the already-parsed `.crbm` index — no
      * payload reads), or `null` if the segment has no Storage generation. Powers the grounded `costReport()`.

package/dist/core/validate.d.ts

@@ -42,5 +42,10 @@ export declare function isReservedNamespace(namespace: string | undefined): bool
 export declare function validateUserNamespace(namespace: string): void;
 /** Validate a segment ref an application hands in: the name rules, and a namespace outside the reserved prefix. */
 export declare function validateUserRef(ref: SegmentRef): void;
+/**
+ * Validate the keys of a `getChunks` stream: in ascending order, a key may repeat. Checked before anything is
+ * resolved or read, so a caller's mistake costs no request.
+ */
+export declare function validateChunkKeyOrder(keys: readonly number[]): void;
 /** Validate a chunk ref: the segment/namespace rules plus `chunkKey` ∈ `[0, 65535]` (a u16). A driver-boundary check. */
 export declare function validateChunkRef(ref: ChunkRef): void;

package/dist/driver-kit.js

@@ -28,7 +28,7 @@ import {
   validateRegistryPatch,
   validateSegmentRef,
   webCryptoEntropy
-} from "./chunk-5W3XS2IU.js";
+} from "./chunk-B7IGS3W6.js";
 
 // src/drivers/_shared/object-registry-keys.ts
 var REGISTRY_SUFFIX = ".reg";
@@ -104,7 +104,7 @@ function parseRegistryKey(prefix, objectKey) {
 var MAX_ROW_BYTES = 1 * 1024 * 1024;
 var MAX_DELETE_ATTEMPTS = 8;
 var MAX_READ_ATTEMPTS = 8;
-var LIST_READ_CONCURRENCY = 16;
+var LIST_READ_CONCURRENCY = 48;
 var ObjectVersionRaced = class extends Error {
   constructor(objectKey) {
     super(`registry object was overwritten mid-read: ${objectKey}`);