@cloudbitmaps/core 0.11.1 → 0.12.0

package/dist/core/crypto.d.ts

@@ -62,19 +62,23 @@ export interface IKeystore {
 }
 /**
  * What the encrypting codec needs: the {@link Aead} for this segment's DEK plus a way to build the AAD for a
- * given scope (`'index'` or a chunkKey). Built per (segment, generation) by the storage-source bridge so the
- * codec itself stays segment-agnostic.
+ * given scope (a chunkKey, `'index'`, or `'metadata'` for the generation's metadata section). Built per
+ * (segment, generation) by the storage-source bridge so the codec itself stays segment-agnostic.
  */
 export interface CrbmCrypto {
     readonly aead: Aead;
-    aadFor(scope: number | 'index'): Uint8Array;
+    aadFor(scope: number | 'index' | 'metadata'): Uint8Array;
 }
 /**
- * Build the AEAD associated-data binding a chunk (or the index) to its exact location:
- * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`. Length-prefixed
- * so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the authenticated label.
+ * Build the AEAD associated-data binding a chunk, the index, the metadata section or a registry row's sealed summary
+ * to its exact location:
+ * `v1 ‖ len(namespace) ‖ namespace ‖ len(segment) ‖ segment ‖ generation ‖ scope ‖ chunkKey`, where scope is
+ * `0` for a chunk, `1` for the index, `2` for the metadata and `3` for the row's summary, and chunkKey is `0` for
+ * the last three.
+ * Length-prefixed so distinct `(namespace, segment)` pairs can never collide. Pure (no crypto) — just the
+ * authenticated label.
  */
 export declare function aadFor(ref: {
     readonly namespace?: string;
     readonly segment: string;
-}, generation: number, scope: number | 'index'): Uint8Array;
+}, generation: number, scope: number | 'index' | 'metadata' | 'summary'): Uint8Array;

package/dist/core/determinism.d.ts

@@ -40,3 +40,10 @@ export interface Rng {
     /** A float in the half-open interval [0, 1). */
     next(): number;
 }
+/**
+ * Injected entropy: `length` cryptographically strong random bytes, for an identity that must never repeat (a
+ * registry row's incarnation id, and the random part every registry write adds to its token). Deliberately not an
+ * {@link Rng}: an `Rng` is seedable for simulation, and a seeded source hands every process the same "random"
+ * identity. Production wiring uses the platform's Web Crypto; a test injects a deterministic source to replay a run.
+ */
+export type Entropy = (length: number) => Uint8Array;

package/dist/core/engine.d.ts

@@ -3,7 +3,7 @@ import type { Budget, BudgetOption } from './budget.js';
 import type { Clock } from './determinism.js';
 import type { BoundedLru } from './lru.js';
 import type { IMetricsSink } from './metrics.js';
-import type { StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { GenerationMetadata, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
 export interface EngineDeps {
     readonly storage: StorageChunkSource;
     /** Optional cache of decoded (immutable) Storage chunks. */
@@ -74,23 +74,38 @@ export declare class SegmentEngine {
     /** Membership: one chunk lookup — the cache, else one Storage fetch of that chunk. */
     has(seg: SegmentRef, id: number): Promise<boolean>;
     /**
-     * Cardinality. When the Storage source can serve per-chunk cardinality from its `.crbm` index (the `.crbm`
-     * source can), the count is summed from the index with **zero payload reads**. A source without that
-     * capability (the in-memory source) falls back to fetching every chunk.
+     * Cardinality. When the Storage source can describe the current generation (the `.crbm` source can), the count is
+     * that description's: the registry row's summary when the row has one it can use, which is one request when cold
+     * and no read of the object, and otherwise the sum of the object's index. Either way, **zero payload reads**. A
+     * source that can only serve per-chunk cardinality sums those, and one with neither (the in-memory source) falls back
+     * to fetching every chunk.
      *
-     * The index sum is trusted, not confirmed: the reader checked the index for internal consistency when it opened
+     * The row's summary and the index sum are trusted, not confirmed: the source holds the summary against the object
+     * whenever it opens it, and the reader checked the index for internal consistency when it opened
      * the object, and a corrupt index that is still internally consistent yields a wrong count. `iterate()` and the
      * combines decode the payloads.
      */
     count(seg: SegmentRef): Promise<number>;
+    /**
+     * What the segment's current generation is: its number, its id count and its metadata, from one resolution, so the
+     * three cannot straddle a publish. A source with no summary answers from `count()` and the generation it resolves.
+     */
+    stat(seg: SegmentRef): Promise<{
+        generation: number | null;
+        cardinality: number;
+        metadata?: GenerationMetadata;
+    }>;
     /** Whether the Storage source can measure segment size (for grounded cost); false ⇒ storage isn't grounded. */
     get supportsStorageSize(): boolean;
     /** How often the Storage source re-reads a pointer while reading (cost reporting); undefined if it does not say. */
     get pointerRefreshMs(): number | undefined;
     /** Grounded Storage size of a segment's current generation (cost reporting), or null if it has no generation. */
     segmentSize(seg: SegmentRef): Promise<SegmentSize | null>;
+    /** The chunks of `seg` at `chunkKeys`, read through a window of {@link DEFAULT_INTERSECT_CONCURRENCY}. */
+    private chunkWindow;
     /**
-     * Every id, ascending, one chunk at a time; with `range`, only the ids in `(after, through]`, fetching only the
+     * Every id, ascending, reading ahead through a window of up to 8 chunk fetches that opens 1, 2, 4, 8 wide, so a
+     * read that stops early has fetched only a handful of chunks past the one it stopped in; with `range`, only the ids in `(after, through]`, fetching only the
      * chunks the range overlaps (see {@link IdRange}).
      *
      * Two generators, not one with a branch: a second `yield` site in the full read's generator grows its frame, and
@@ -124,7 +139,7 @@ export declare class SegmentEngine {
      * up front (before the fan-out) and threaded into every chunk read, so a concurrent load can't corrupt or tear
      * the result — every chunk read is a whole, checksum-verified, immutable generation. The edge a *long* call can
      * hit is invariant 3's: if it straddles a mid-call `cache.genTtlMs` boundary and a load has published, an
-     * operand's not-yet-read chunks may re-resolve forward to the newer generation (a generation hop within one long
+     * operand's not-yet-requested chunks may re-resolve forward to the newer generation (a generation hop within one long
      * call) — the call never crashes or returns a torn object, but may mix generations. Three things hop it without
      * waiting for the TTL, so a shorter call can meet them too: **the reader cache evicting an operand mid-call**
      * (`maxOpenSegments`), whose re-read re-resolves fresh; a sweep deleting the generation it was reading, which

package/dist/core/erase-id.d.ts

@@ -9,6 +9,12 @@
  * (`keep: 0`), so the bit is **physically gone from the bucket when this returns**. Constant memory: one chunk in
  * flight, never the whole segment.
  *
+ * The rewrite is the same generation without one id, so it keeps everything else: the new object carries the source's
+ * metadata as it is, and the row's summary of it, built from what was written, counts one id fewer and holds the same
+ * metadata. Metadata is not scanned for the id. On an encrypted segment, a source object with no metadata block whose
+ * row's sealed summary of it has metadata is rewritten with the row's: the block's presence is not authenticated, so
+ * the authenticated copy decides, and the erasure still goes through.
+ *
  * **`erased: true` is a claim about every generation of the segment, not only the one it replaced.** A rollback
  * leaves generations above the pointer that were once current and can be made current again, so a holder can sit
  * above the pointer as well as below it, and there can be several. Before it reports `erased: true` the call
@@ -69,8 +75,10 @@
 import { type IAuditSink } from './audit.js';
 import type { CodecInterface } from './codec.js';
 import type { Yielder } from './cooperative.js';
+import type { Rng } from './determinism.js';
 import type { IKeystore } from './crypto.js';
 import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
+import { type ReadRetry } from './retry.js';
 /** What {@link eraseIdFromSegment} needs: the objects, the pointer, the codec, and the key material if encrypted. */
 export interface EraseIdDeps {
     readonly storage: IStorageDriver;
@@ -90,8 +98,20 @@ export interface EraseIdDeps {
     readonly requireEncryption?: boolean;
     /** Supplying a clock that can yield makes the rewrite cooperative, as it makes a load. */
     readonly clock?: Yielder;
+    /**
+     * The random source that spreads the waits between the rewrite's fresh compare-and-swaps. Absent, the read retry's
+     * source is used if it has one, and otherwise each wait is its bound.
+     */
+    readonly rng?: Rng;
     /** Per-chunk decode ceiling (invariant 5); defaults to 1 MiB. */
     readonly maxBitmapBytes?: number;
+    /**
+     * The store's read retry, for the reads the rewrite makes along the way: the generation it rewrites and each of its
+     * chunks, the read-back that verifies the generation it wrote, and any other generation it checks for the id. A
+     * transient fault on one is run again under it rather than failing the erasure. Absent, each read is made once.
+     * The deletes are not retried, and the rewrite's registry write is settled as a load's is.
+     */
+    readonly readRetry?: ReadRetry;
 }
 export interface EraseIdResult {
     readonly segment: string;

package/dist/core/erasure.d.ts

@@ -159,10 +159,14 @@ export interface DropResult {
  *    cleaned up by re-running; a torn pointer costs correctness and is not self-healing. Given the choice, leak
  *    bytes — but say so: whatever survives the sweep is reported in {@link DropResult.generationsRemaining}.
  *
- * **Why step 2 sweeps more than once.** A load that was already writing its object when the tombstone landed
- * still finishes the write — its publish is then refused, but the object survives, and it holds the complete
- * set. A single list-then-delete misses it entirely. The re-sweep converges because the tombstone *is* a hard
- * fence on **publishing**, so only already-in-flight writes can appear and they are finite.
+ * **Why step 2 sweeps more than once.** A load that had read the row before the tombstone landed, whether it was
+ * already writing its object or still consuming its ids, still finishes the write — its publish is then refused,
+ * but the object exists, and it holds the complete set. A single list-then-delete misses it entirely. The re-sweep
+ * converges because the tombstone *is* a hard fence on **publishing**, so only loads already under way can write
+ * and they are finite. One that writes after the last pass deletes its own object when its publish is refused, as a
+ * refused load does under a `destroyed` row; only one whose process stops in between, or whose publish fails without
+ * a definite answer (a lost response, a timeout), leaves it behind, for a re-run of the drop, and
+ * `generationsRemaining` cannot report an object written after this call returned.
  *
  * **When "reads as empty" starts being true.** Not instantly, for a store that has already read this segment: a
  * resolved generation is cached and decoded chunks sit in the cache, so an in-flight reader can answer from

package/dist/core/generation-gc.d.ts

@@ -1,4 +1,4 @@
-import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
+import type { GenKey, IStorageDriver, IRegistryDriver, RegistryRecord, SegmentRef } from './ports.js';
 /** The two ports generation bookkeeping needs: the objects, and the pointer that says which one is current. */
 export interface GenerationDeps {
     readonly storage: IStorageDriver;
@@ -6,7 +6,9 @@ export interface GenerationDeps {
 }
 /**
  * The generation number a writer should use for the segment's **next** object: one above the highest generation
- * the registry points at *or* that is present in Storage — whichever is higher.
+ * the registry points at *or* that is present in Storage — whichever is higher. The erasure rewrite numbers this
+ * way; a load numbers with {@link nextLoadGeneration}, which lists as this does when its existence check finds the
+ * number taken or cannot answer.
  *
  * Both are consulted because they can disagree. A load that wrote its object and crashed before publishing leaves
  * an object *above* `currentGen`; a writer that consulted only the pointer would pick that same number and hit the
@@ -17,6 +19,46 @@ export interface GenerationDeps {
  * A segment with no row and no objects starts at 0. An admin/write-path helper, never on the read path.
  */
 export declare function nextGeneration(ref: SegmentRef, deps: GenerationDeps): Promise<number>;
+/** The number a load takes, and whether one existence check proved it free (so nothing is above the pointer). */
+export interface LoadNumber {
+    readonly generation: number;
+    /**
+     * `true` when the number is `currentGen + 1` and the check found no object holding it. Collection by name rests
+     * on it: the row named `generation - 1` as current, so every number the window evicts is below it, and nothing
+     * the listing would have kept is a name the pass takes. `false` when the listing numbered the load.
+     */
+    readonly checked: boolean;
+}
+/**
+ * The generation number a load takes, from the row it has already read: `currentGen + 1` (0 with no row or no
+ * pointer yet) when no object holds that number, which one existence check proves (a zero-byte `getTail`, one
+ * metadata request on every shipped driver); otherwise one above the pointer and above everything listed, as
+ * {@link nextGeneration} numbers. The check finding the number taken (a crashed load's orphan, the object of a
+ * load still in flight, or the generations a rollback left above the pointer) and the check failing in any way
+ * other than "not found" both take the listing, so the listing stays the authority whenever the check cannot
+ * prove the number free. A failed check falls back silently: nothing records it, and what it costs is the listing,
+ * one more PUT-class request on S3, so a fault that makes every check fail shows only in the bill.
+ *
+ * A load can therefore take a number **below** an object already in the bucket: an orphan at `currentGen + 2`
+ * with `currentGen + 1` free is numbered under, not past. That is as safe as a listing's number. A load never
+ * takes a number an object holds, and a put that races onto one fails write-once, so the load reports
+ * `superseded`. The orphan above stays unpublished; the load whose check meets it numbers past it by the listing,
+ * and collection takes it once a generation above it is current. Like a listing's, the number can be one whose
+ * object was deleted (an erasure removes the generations above a rolled-back pointer that held the id), which is
+ * why nothing identifies a generation by its number alone: the reader cache, the chunk cache and a pin all key on
+ * the number and the row's token, and the token has moved on.
+ */
+export declare function nextLoadGeneration(ref: SegmentRef, deps: GenerationDeps, row: RegistryRecord | null): Promise<LoadNumber>;
+/**
+ * How often a load that could collect by name lists instead: every generation divisible by this. A name-only pass
+ * takes one generation a load and never looks at the rest, so what it misses (a refused load's object that stayed
+ * because the row had changed, extras left by a `keep` that shrank, an object a rollback or an erasure left
+ * stranded below the pointer) waits for a listing. The number is a generation's, not a clock's, so the choice needs
+ * no state and no time, and a segment that loads cleanly lists once in this many loads at most. The bound is in
+ * generation numbers: a rollback moves the pointer down, and the loads that then take the numbers above it count
+ * from there.
+ */
+export declare const LIST_COLLECTION_CADENCE = 16;
 /**
  * Garbage-collect superseded Storage generations for a segment: everything strictly below `currentGen`, keeping
  * the most recent `keep` of them as a grace window, so a read still fetching from a just-superseded
@@ -42,8 +84,8 @@ export declare function nextGeneration(ref: SegmentRef, deps: GenerationDeps): P
  * an empty array is not: `eraseIdFromSegment` reads the returned list as the physical half of its erasure
  * receipt, and would otherwise report `erased: true` over bytes still in the bucket. Re-run it.
  *
- * That state is reachable in practice: a `dropSegment` whose Storage sweep threw part-way, or a load that was
- * already writing its object when the tombstone landed and finished the write afterwards. `dropSegment` re-sweeps
+ * That state is reachable in practice: a `dropSegment` whose Storage sweep threw part-way, or a load that had read
+ * the row before the tombstone landed, wrote its object afterwards, and stopped before its refusal deleted it. `dropSegment` re-sweeps
  * and reports whatever it could not reclaim in `generationsRemaining`, but a drop that was never re-run leaves a
  * residual, and this is what eventually collects it from the retention sweep.
  *
@@ -54,3 +96,59 @@ export declare function nextGeneration(ref: SegmentRef, deps: GenerationDeps): P
 export declare function gcOrphanGenerations(ref: SegmentRef, deps: GenerationDeps, options?: {
     keep?: number;
 }): Promise<number[]>;
+/**
+ * Collect after a load's publish: by name when `byName` (the load numbered its generation with one existence check,
+ * the object its row named was there when the guard looked, and its caller does not ask for a listing) and `keep` is
+ * at most one, and by listing otherwise ({@link gcOrphanGenerations}). A load lists in three cases: `keep` of 2 or
+ * more, which a name cannot serve because the window then counts generations that may be absent (a refused
+ * neighbour's number is a gap, and by name would take one the window promised to keep); a number the listing chose,
+ * so something sits above the pointer or the check could not answer, or a current object the load's check found gone; and
+ * every {@link LIST_COLLECTION_CADENCE}th generation, which collects what a name-only pass leaves. It makes no
+ * request when `keep` is at least the generation published, since no more generations than that exist below it. A
+ * load onto a `destroyed` row is refused at its publish and never reaches here; a drop that lands after the publish
+ * leaves this pass collecting the tombstone's generations, one by name or all of them by listing, and the drop's own
+ * sweep takes the rest.
+ */
+export declare function collectAfterLoad(ref: SegmentRef, deps: GenerationDeps, options: {
+    generation: number;
+    keep: number;
+    byName: boolean;
+    /**
+     * The object the load's row named as current, which the caller did not open. Before it collects by name a generation
+     * the window keeps (a `keep` of 1), the pass looks for this object with one zero-byte read, and lists instead
+     * unless that finds it: a name-only pass is safe only while the object the row named is in the bucket.
+     */
+    proveCurrent?: GenKey;
+}): Promise<number[]>;
+/**
+ * Collect, with no listing, the one generation a publish of `generation` pushed out of a grace window of `keep`
+ * (0 or 1): `generation - keep - 1`, when that is a number at all. Returns that name, whether or not an object was
+ * there: a delete of an absent object succeeds on every backend and says nothing, so the name returned is the one
+ * asked for, not one found, and a generation a concurrent pass, an erasure or a lifecycle rule had already taken is
+ * listed all the same. A list is not a receipt (invariant 4), and neither is this.
+ *
+ * The caller has published `generation` having numbered it with one existence check, so the row named
+ * `generation - 1` and every generation the window newly evicts is below it. With `keep` of 0 or 1 that name is
+ * the one a listing pass would also take, or one it left to a later pass: never a generation the listing keeps.
+ * That rests on the object the row named being in the bucket, and a fault can break it (a lifecycle rule or a
+ * partial restore removing it, or an erasure deleting the object of a load whose publish then landed;
+ * `checkConsistency` reports the state). The caller, which learns that from its guard's read of the current
+ * generation or, when the guard took the size from the row's summary and opened nothing, from one zero-byte read of
+ * it (see {@link collectAfterLoad}), lists instead; a load that made neither (`allowEmpty` without `minRetained`)
+ * cannot tell, and deletes by name the older generation a listing would have kept as the window. `keep` of 2 or more is refused,
+ * because there the window counts the generations that exist, which a name cannot know.
+ *
+ * Collection never touches the current generation, and the row is re-proved before the delete as every pass does.
+ * A row that is gone, or a pointer that has fallen below `generation` (a rollback, or a name purged and re-created
+ * with fewer loads), deletes nothing and returns `[]`: the publish already landed, nothing here protects it or
+ * needs the result, and a throw would hand the caller a failure for a load that took effect. A pointer that has
+ * moved above `generation` does not stop it: a publish landing in the meantime leaves routine collection working.
+ * A fault, a registry read or a delete that throws, still propagates, with the pointer at `generation`. The bound
+ * is `generation`, the pointer this load's own publish set, so the lower-of-two-pointers rule of a listing pass
+ * holds with one observation in place of two. Nothing is deleted when `generation` has nothing below its window,
+ * and no request is made.
+ */
+export declare function collectByName(ref: SegmentRef, deps: GenerationDeps, options: {
+    generation: number;
+    keep: number;
+}): Promise<number[]>;

package/dist/core/load-input.d.ts

@@ -0,0 +1,55 @@
+/**
+ * What a load accepts: ids, or a whole bitmap as `{ serialized }` bytes or as a `{ bitmap }` that serializes to
+ * them.
+ *
+ * Every bitmap input takes one path. The bytes are size-capped, checked structurally and decoded by the codec's
+ * safe deserializer into a bitmap of the codec's own, which then writes its chunks straight from its containers:
+ * no id passes through JavaScript. `{ bitmap }` is only a spelling of `{ serialized }`, serialized once, here, so
+ * a caller's bitmap is never trusted as a structure and a change made to it after the call is not loaded.
+ *
+ * All of it runs before the load's first request, so a malformed input costs no round trip and writes nothing.
+ */
+import type { CodecBitmap, CodecInterface } from './codec.js';
+/**
+ * Anything that serializes itself to portable Roaring: `roaring`'s `RoaringBitmap32` is one. Typed by shape, so
+ * no codec's own class appears in a signature. A bitmap that can also say how large its serialization is
+ * (`getSerializationSizeInBytes`) is asked first, so one over the cap is refused before it is serialized.
+ */
+export interface PortableBitmap {
+    serialize(format: 'portable'): Uint8Array;
+    getSerializationSizeInBytes?(format: 'portable'): number;
+}
+/**
+ * What a load writes as the segment's new contents.
+ *
+ * - **ids**: an iterable or async iterable of integers in `[0, 2^32)`, consumed lazily. Not a `Uint8Array`,
+ *   `Uint8ClampedArray` or `Buffer`, which a load refuses, since each byte would load as an id.
+ * - **`{ serialized }`**: one bitmap in the portable Roaring format, checked before anything is written.
+ * - **`{ bitmap }`**: an object with `serialize('portable')`, loaded as `{ serialized: bitmap.serialize('portable') }`.
+ */
+export type LoadInput = Iterable<number> | AsyncIterable<number> | {
+    readonly serialized: Uint8Array;
+} | {
+    readonly bitmap: PortableBitmap;
+};
+/**
+ * No canonical 32-bit Roaring bitmap serializes to more than this. Under the run cookie, the larger of the two
+ * headers, it is a 4-byte cookie, 8,192 bytes of run flags and 8 bytes of header for each of the 65,536
+ * containers, and no container's body is over 8,192 bytes: a bitset is 8,192, and a run container is kept only
+ * where it is smaller. 537,403,396 bytes, about 512.5 MiB.
+ */
+export declare const MAX_SERIALIZED_LOAD_BYTES: number;
+/** A bitmap input, decoded into the codec's own bitmap: written from its chunks, never id by id. */
+export declare class DecodedLoadInput {
+    readonly bitmap: CodecBitmap;
+    constructor(bitmap: CodecBitmap);
+}
+/**
+ * Check a load's input and decode a bitmap input, before the load makes any request.
+ *
+ * @returns ids unchanged, to be consumed lazily by the write, or the decoded bitmap.
+ * @throws {ValidationError} for a byte array passed as ids; bytes that are malformed, over the cap, or followed by
+ * more bytes; a `{ bitmap }` with no `serialize` method, or whose `serialize('portable')` returns no `Uint8Array`;
+ * and anything that is none of the three inputs. An error the caller's own `serialize` throws propagates as it is.
+ */
+export declare function prepareLoadInput(input: LoadInput, codec: CodecInterface): Iterable<number> | AsyncIterable<number> | DecodedLoadInput;

package/dist/core/load.d.ts

@@ -15,15 +15,17 @@
  *
  * Refusing therefore has to clean up after itself. The object is already durable at that point, and it sits
  * ABOVE `currentGen`, where generation collection deliberately never looks (it deletes strictly below the
- * pointer), so a refused load deletes its own object before returning, while the segment's row is unchanged or
- * gone. Once another write has changed the row, the object's number may name a re-created segment's live object,
- * so it stays, and collection takes it like any other generation once one above it is current.
+ * pointer), so a refused load deletes its own object before returning, while the segment's row is unchanged, gone
+ * or `destroyed`. Once another write has changed the row, the object's number may name a re-created segment's live
+ * object, so it stays, and collection takes it like any other generation once one above it is current.
  */
 import { type IAuditSink } from './audit.js';
 import { type CodecInterface } from './codec.js';
-import type { Clock } from './determinism.js';
+import type { Clock, Rng } from './determinism.js';
 import type { IKeystore } from './crypto.js';
-import type { IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
+import { type LoadInput } from './load-input.js';
+import { type ReadRetry } from './retry.js';
+import type { GenerationMetadata, IStorageDriver, IRegistryDriver, SegmentRef } from './ports.js';
 /** What {@link loadSegment} needs: the objects, the pointer, the codec, and key material if encrypted. */
 export interface LoadDeps {
     readonly storage: IStorageDriver;
@@ -32,6 +34,27 @@ export interface LoadDeps {
     readonly keystore?: IKeystore;
     readonly requireEncryption?: boolean;
     readonly clock?: Clock;
+    /**
+     * The random source that spreads the waits between a publish's fresh compare-and-swaps (a random time under each
+     * bound). Absent, the read retry's source is used if it has one, and otherwise each wait is its bound.
+     */
+    readonly rng?: Rng;
+    /**
+     * The store's read retry, for the read the guard makes of the current generation: a transient fault there is run
+     * again under it rather than failing the load. Absent, the read is made once. It does not govern the write: a
+     * registry write that gets no answer is settled by reading the row, and sent again only as a bounded fresh
+     * compare-and-swap.
+     */
+    readonly readRetry?: ReadRetry;
+    /**
+     * Collect by listing the segment's objects after the publish, whatever `keep` is. Absent, a load that keeps at
+     * most one generation and found nothing above its pointer deletes by name the one generation its publish pushed
+     * out of the window, and lists every sixteenth generation to take what that leaves; the generations older than
+     * the window wait for that listing. Set it for a caller whose `keep` promises every generation below the new
+     * one beyond the window is gone when the call returns: the `*Into` verbs, whose `keep` is how an operator clears
+     * a destination that earlier materialisations kept in full.
+     */
+    readonly collectByListing?: boolean;
 }
 /**
  * What a plausible result looks like, so an implausible one is refused instead of published.
@@ -71,8 +94,27 @@ export interface LoadOptions {
      * collected. Keep at least one generation for every one that can be written above the pinned one while your
      * longest pinned job runs, on every writer that loads the segment — see "Generations and `keep`" in the loading
      * guide.
+     *
+     * With `keep` of 0 or 1, a load that found nothing above the pointer collects without listing: it deletes the one
+     * generation its publish pushed out of the window, and lists the segment's objects on every sixteenth generation to
+     * take whatever that pass leaves, such as the generations an earlier, wider `keep` held. With `keep` of 2 or more
+     * it lists on every load, and so does one whose check found the current generation's object gone: the guard's read of
+     * it, or, when the guard took the size from the row's summary, one zero-byte read made before a `keep` of 1 takes a
+     * name. A `keep` at least the generation published collects nothing and asks for nothing.
      */
     readonly keep?: number;
+    /**
+     * Small, immutable metadata of your own for this generation: what it was computed from, such as a definition's
+     * version, the time its data landed, or a run id. A flat object of string keys and string or finite-number values,
+     * at most 1,024 bytes as canonical JSON (keys sorted by UTF-16 code unit, no whitespace), and no key longer than 128
+     * bytes. It is stored in the generation's object, and written to the segment's row by the write that makes the
+     * generation current, so whoever sees this generation as current sees its metadata. It never changes once written:
+     * a new generation is how it changes. `undefined` and the empty object store nothing. Anything else that breaks a
+     * rule (a boolean, `null`, an array, nesting, `NaN`, a key that is not well-formed text) throws `ValidationError`
+     * before a request is made. **Never put a subject's id in it**: an erasure rewrites the ids and carries the metadata
+     * over as it is. Sealed under the segment's key when the segment is encrypted.
+     */
+    readonly metadata?: GenerationMetadata;
     readonly audit?: IAuditSink;
 }
 /** Why a load did not become current. */
@@ -82,13 +124,14 @@ export type LoadRefusal = 'empty' | 'min-cardinality' | 'min-retained'
  * put refused this one and it wrote nothing (`size: 0`); or the segment's registry row changed while the load was
  * writing: another load published, a retention change, a rollback or an erasure wrote the row, or the row was
  * deleted. Once another write has changed the row, the object stays in the bucket, because its generation number
- * may by then name another incarnation's live object; once the row is gone, the object is deleted.
+ * may by then name another incarnation's live object; once the row is `destroyed` (dropped or crypto-shredded),
+ * the object is deleted, and once the row is gone, so is an object its footer proves this load's own.
  */
  | 'superseded';
 export interface LoadResult {
     /**
      * The generation written. Present even when refused: a refusal deletes that object while the segment's row is
-     * unchanged or gone, and leaves it in the bucket once another write has changed the row. A load that lost its
+     * unchanged or `destroyed`, or gone once its footer proves it this load's own, and leaves it in the bucket once another write has changed the row. A load that lost its
      * generation number to another wrote nothing, and reports `size: 0`.
      */
     readonly generation: number;
@@ -108,14 +151,26 @@ export interface LoadResult {
      * whatever happens next.
      */
     readonly cardinalityBefore: number | null;
-    /** The superseded generations collected after publishing. Empty when nothing was published. */
+    /**
+     * The superseded generations collected after publishing. Empty when nothing was published.
+     *
+     * When collection deleted by name (see {@link LoadOptions.keep}) this is the name it deleted, and that generation
+     * may have been gone already: a delete of an absent object succeeds on every backend and does not say so, so the
+     * list names what the pass asked the bucket to delete, not what it found there. A listing pass names the
+     * generations it found and deleted. Neither is a receipt, since a concurrent collector may have taken a generation
+     * first.
+     */
     readonly collected: readonly number[];
 }
 /**
- * Replace a segment's contents with `ids`, as one immutable generation, and make it current.
+ * Replace a segment's contents with `input`, as one immutable generation, and make it current.
+ *
+ * `input` is ids, or a whole bitmap as `{ serialized }` portable Roaring bytes or a `{ bitmap }` with
+ * `serialize('portable')` ({@link LoadInput}). A bitmap input is checked, decoded and serialized before the first
+ * request, and is written from its own chunks; the generation is byte for byte the one its ids would write.
  *
  * Returns what was written and whether it became current. A refusal is **not** an exception: `published: false`
  * with a `reason` is a normal outcome a caller branches on, in the same shape as a successful load, because the
  * interesting cases (a guard tripped, a racing writer won) are operational facts rather than faults.
  */
-export declare function loadSegment(ref: SegmentRef, ids: Iterable<number> | AsyncIterable<number>, deps: LoadDeps, options?: LoadOptions): Promise<LoadResult>;
+export declare function loadSegment(ref: SegmentRef, input: LoadInput, deps: LoadDeps, options?: LoadOptions): Promise<LoadResult>;

package/dist/core/metadata.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Generation metadata: a small, flat record of strings and finite numbers attached to one generation.
+ *
+ * The rules live here once, for every boundary that meets a value: a caller's input, refused with
+ * `ValidationError`, and bytes read back from a tier, refused with `IntegrityError`. The caller passes the error
+ * to raise. Pure: no I/O.
+ */
+import type { GenerationMetadata } from './ports.js';
+/** Cap on the canonical JSON of one generation's metadata, in UTF-8 bytes (braces and quotes included). */
+export declare const MAX_METADATA_BYTES = 1024;
+/** Cap on one metadata key, in UTF-8 bytes. */
+export declare const MAX_METADATA_KEY_BYTES = 128;
+/**
+ * Validate `value` as generation metadata (`GenerationMetadata` in the ports) and return its canonical JSON: RFC 8785
+ * (the JSON Canonicalization Scheme) for a flat object of strings and finite numbers. Keys are sorted by UTF-16 code
+ * unit, strings are escaped as `JSON.stringify` escapes them, numbers are written as ECMAScript's
+ * `Number.prototype.toString` writes them (so `-0` is `0`, and `1e21` is `1e+21`), and there is no whitespace. Key
+ * order therefore never changes the bytes, and `tests/golden/metadata-canonical.json` holds vectors a port can check
+ * itself against. Both caps are in UTF-8 bytes. The container is an object whose prototype is
+ * `Object.prototype` or `null`, holding only enumerable data properties with string keys. `fail` raises the error the
+ * boundary calls for. The empty object passes and returns `{}`: whether it may be stored is the caller's rule.
+ */
+export declare function canonicalMetadataJson(value: unknown, fail: (message: string) => never): string;
+/**
+ * The bytes a generation stores for `value`: its canonical JSON in UTF-8, or `undefined` when it has none (`undefined`
+ * or the empty object, which store nothing). Synchronous, so what is stored is what the caller passed at the call,
+ * whatever it does with its object afterwards. `fail` raises the boundary's error.
+ */
+export declare function metadataBytes(value: unknown, fail: (message: string) => never): Uint8Array | undefined;
+/**
+ * A caller's metadata, checked against the rules and copied, so what is stored is what the caller passed at the call,
+ * whatever it does with its object afterwards: the frozen record in canonical key order, or `undefined` when there is
+ * none (`undefined`, or the empty object, which store nothing). Synchronous. `fail` raises the boundary's error.
+ */
+export declare function copiedMetadata(value: unknown, fail: (message: string) => never): GenerationMetadata | undefined;
+/**
+ * Read metadata back from bytes a tier holds, which are untrusted. They must be at most {@link MAX_METADATA_BYTES},
+ * valid UTF-8, and exactly the canonical JSON of a non-empty record, so a stored copy has one spelling: whitespace, a
+ * key out of order or listed twice, and a number or an escape `JSON.stringify` would write another way are all
+ * refused. Returns a frozen record of its own. `fail` raises the boundary's error.
+ */
+export declare function metadataFromBytes(bytes: Uint8Array, fail: (message: string) => never): GenerationMetadata;

package/dist/core/pinned-storage-source.d.ts

@@ -21,7 +21,7 @@
  * on some other handle. That is why the routing is by segment identity here rather than by which object the
  * caller happened to start from.
  */
-import type { ChunkRef, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
+import type { ChunkRef, GenerationSummary, StorageChunkSource, SegmentRef, SegmentSize } from './ports.js';
 import type { CrbmStorageChunkSource } from './crbm-storage-source.js';
 /**
  * What a pin holds for one segment: the generation, the version identifying those exact bytes, and the pinned
@@ -67,6 +67,7 @@ export declare class PinnedStorageChunkSource implements StorageChunkSource {
     listChunkKeys(ref: SegmentRef): Promise<number[]>;
     sizeOf(ref: SegmentRef): Promise<SegmentSize | null>;
     cardinalities(ref: SegmentRef): Promise<ReadonlyMap<number, number> | null>;
+    summary(ref: SegmentRef): Promise<GenerationSummary | null>;
     currentGeneration(ref: SegmentRef): Promise<number | null>;
     /**
      * A pinned segment reports the version captured when it was pinned, marked as a pin's, so its decoded chunks